Been a minute and I've currently found some motivation to work on animations and make the code run faster plus also mapping things finally.

Found out the ESP32 only has a 64 byte hardware buffer for I2C and that I need to use endTransmission after 63 bytes, stole some adafruit code for I2C oled displays and put it in the library I'm currently using for the IS31FL3733.

Here's a snippet of the I2C write section.

Wire.beginTransmission(i2c_addr);
Wire.write((byte) reg_addr);

uint16_t rounds = 1;
uint16_t bytesOut = 1;
uint8_t *ptr = buffer;

#if defined(ESP32)
    while (count--) {
        if (bytesOut >= 64) {
            Wire.endTransmission();
            Wire.beginTransmission(i2c_addr);

            Wire.write((byte) reg_addr + 63*rounds);
            bytesOut = 1;
            rounds++;
        }
        Wire.write((byte)(*ptr++));
        bytesOut++;
    }
    Wire.endTransmission();
    return 1;

#else
    Wire.write((byte) reg_addr);
    for (uint8_t i = 0; i < count; i++) {
        Wire.write((byte)(buffer[i]));
    }
    Wire.endTransmission();
    return 1;
#endif