Speed up JPEG decoding on ESP32-P4 and ESP32-S31

ports/espressif/CMakeLists.txt

@@ -5,7 +5,7 @@ cmake_minimum_required(VERSION 3.16)
 set(ENV{IDF_PATH} ${CMAKE_SOURCE_DIR}/esp-idf)
 
 # The component list here determines what options we get in menuconfig and what the ninja file can build.
-set(COMPONENTS bt driver esp_driver_dac esp_driver_gpio esp_driver_gptimer esp_driver_i2c esp_driver_i2s esp_driver_ledc esp_driver_pcnt esp_driver_rmt esp_driver_sdmmc esp_driver_spi esp_driver_touch_sens esp_driver_tsens esp_driver_uart esp_eth esp-tls esp_adc esp_adc_cal esp_event esp_netif esp_psram esp_security esp_wifi esptool_py freertos log lwip main mbedtls mdns pthread soc ulp wpa_supplicant esp-camera esp_lcd vfs esp_vfs_console esp_stdio sdmmc usb)
+set(COMPONENTS bt driver esp_driver_dac esp_driver_gpio esp_driver_gptimer esp_driver_i2c esp_driver_i2s esp_driver_jpeg esp_driver_ledc esp_driver_pcnt esp_driver_rmt esp_driver_sdmmc esp_driver_spi esp_driver_touch_sens esp_driver_tsens esp_driver_uart esp_eth esp-tls esp_adc esp_adc_cal esp_event esp_netif esp_psram esp_security esp_wifi esptool_py freertos log lwip main mbedtls mdns pthread soc ulp wpa_supplicant esp-camera esp_lcd vfs esp_vfs_console esp_stdio sdmmc usb)
 set(EXTRA_COMPONENT_DIRS "esp-protocols/components/mdns" "esp-camera")
 
 include($ENV{IDF_PATH}/tools/cmake/project.cmake)

ports/espressif/Makefile

@@ -65,6 +65,7 @@ INC += \
 	-isystem esp-idf/components/esp_driver_gptimer/include \
 	-isystem esp-idf/components/esp_driver_i2c/include \
 	-isystem esp-idf/components/esp_driver_i2s/include \
+	-isystem esp-idf/components/esp_driver_jpeg/include \
 	-isystem esp-idf/components/esp_driver_ledc/include \
 	-isystem esp-idf/components/esp_driver_parlio/include \
 	-isystem esp-idf/components/esp_driver_pcnt/include \
@@ -119,6 +120,8 @@ INC += \
 	-isystem esp-idf/components/esp_hal_i2c/include \
 	-isystem esp-idf/components/esp_hal_i2s/include \
 	-isystem esp-idf/components/esp_hal_i2s/$(IDF_TARGET)/include \
+	-isystem esp-idf/components/esp_hal_jpeg/include \
+	-isystem esp-idf/components/esp_hal_jpeg/$(IDF_TARGET)/include \
 	-isystem esp-idf/components/esp_hal_ledc/include \
 	-isystem esp-idf/components/esp_hal_pcnt/include \
 	-isystem esp-idf/components/esp_hal_rmt/include \
@@ -485,6 +488,9 @@ CFLAGS += \
 	-isystem esp-idf/components/esp_hw_support/ldo/include \
 	-isystem esp-idf/components/esp_driver_touch_sens/hw_ver3/include
 
+# JPEG engine DMA writes into GC blocks, so make them one 64-byte cache line.
+CFLAGS += -DMICROPY_BYTES_PER_GC_BLOCK=64
+
 ifeq ($(CIRCUITPY_ESP32P4_REV),3)
 IDF_TARGET_ROM_FLOAT_LD = esp32p4.rom.rvfp.ld
 LDFLAGS += \
@@ -521,6 +527,9 @@ CHIP_COMPONENTS = \
 else ifeq ($(IDF_TARGET),esp32s31)
 CFLAGS += -isystem esp-idf/components/esp_hw_support/ldo/include
 
+# JPEG engine DMA writes into GC blocks, so make them one 64-byte cache line.
+CFLAGS += -DMICROPY_BYTES_PER_GC_BLOCK=64
+
 IDF_TARGET_ROM_FLOAT_LD = esp32s31.rom.rvfp.ld
 LDFLAGS += \
 	-Tesp32s31.rom.heap.ld \
@@ -815,6 +824,11 @@ SRC_ULP := \
 SRC_C += $(SRC_ULP)
 endif
 
+ifneq ($(CIRCUITPY_JPEGIO_HW),0)
+SRC_C += common-hal/jpegio/JpegDecoder.c
+CHIP_COMPONENTS += esp_driver_jpeg esp_hal_jpeg
+endif
+
 ifneq ($(CIRCUITPY_NEOPIXEL_WRITE),0)
 CHIP_COMPONENTS += esp_driver_rmt
 endif

ports/espressif/common-hal/jpegio/JpegDecoder.c

@@ -0,0 +1,219 @@
+// This file is part of the CircuitPython project: https://circuitpython.org
+//
+// SPDX-FileCopyrightText: Copyright (c) 2026 Mikey Sklar
+//
+// SPDX-License-Identifier: MIT
+
+// Hardware JPEG decode for jpegio, on chips with SOC_JPEG_DECODE_SUPPORTED.
+// The engine decodes a whole baseline JPEG to RGB565 with no scaling, so only
+// decode() calls with scale=0 into a 16-bit Bitmap take this path. Anything the
+// hardware does not handle returns false and jpegio falls back to TJpgDec.
+
+#include "py/runtime.h"
+
+#include "shared-bindings/bitmaptools/__init__.h"
+#include "shared-module/displayio/Bitmap.h"
+#include "shared-module/jpegio/JpegDecoder.h"
+
+#include "driver/jpeg_decode.h"
+#include "esp_cache.h"
+#include "esp_heap_caps.h"
+#include "esp_memory_utils.h"
+
+// A bad image blocks the engine for at most this long.
+#define JPEG_DECODE_TIMEOUT_MS (1000)
+
+void jpegio_jpegdecoder_hw_init(jpegio_jpegdecoder_obj_t *self) {
+    jpeg_decode_engine_cfg_t engine_config = {
+        .timeout_ms = JPEG_DECODE_TIMEOUT_MS,
+    };
+    jpeg_decoder_handle_t engine;
+    if (jpeg_new_decoder_engine(&engine_config, &engine) != ESP_OK) {
+        engine = NULL;
+    }
+    self->hw_decoder = engine;
+}
+
+void jpegio_jpegdecoder_hw_deinit(jpegio_jpegdecoder_obj_t *self) {
+    if (self->hw_decoder != NULL) {
+        jpeg_del_decoder_engine(self->hw_decoder);
+        self->hw_decoder = NULL;
+    }
+}
+
+// GC blocks start on a cache line, so the engine can write Bitmap pixels directly.
+#ifdef CONFIG_CACHE_L2_CACHE_LINE_SIZE
+_Static_assert(MICROPY_BYTES_PER_GC_BLOCK % CONFIG_CACHE_L2_CACHE_LINE_SIZE == 0, "GC block is not a cache line");
+#endif
+#ifdef CONFIG_CACHE_L1_DCACHE_LINE_SIZE
+_Static_assert(MICROPY_BYTES_PER_GC_BLOCK % CONFIG_CACHE_L1_DCACHE_LINE_SIZE == 0, "GC block is not a cache line");
+#endif
+
+// True if the hardware may write `size` bytes at `buf` directly: the start and
+// length must both be whole cache lines.
+static bool output_aligned(const void *buf, size_t size) {
+    size_t align = esp_cache_get_line_size_by_addr(buf);
+    if (align < 4) {
+        align = 4;
+    }
+    return ((uintptr_t)buf & (align - 1)) == 0 && (size & (align - 1)) == 0;
+}
+
+// These return false so decode() uses TJpgDec, or raise ValueError if
+// software decoding is not allowed.
+static bool use_software(jpegio_jpegdecoder_obj_t *self, mp_rom_error_text_t msg) {
+    if (!self->software_ok) {
+        mp_raise_ValueError(msg);
+    }
+    return false;
+}
+
+static bool use_software_for_arg(jpegio_jpegdecoder_obj_t *self, qstr arg_name) {
+    if (!self->software_ok) {
+        mp_raise_ValueError_varg(MP_ERROR_TEXT("Invalid %q"), arg_name);
+    }
+    return false;
+}
+
+bool jpegio_jpegdecoder_hw_decode(jpegio_jpegdecoder_obj_t *self, const mp_buffer_info_t *jpeg, int scale) {
+    if (self->hw_decoder == NULL) {
+        return use_software(self, MP_ERROR_TEXT("Operation or feature not supported"));
+    }
+    displayio_bitmap_t *bitmap = self->dest;
+    if (scale != 0) {
+        return use_software_for_arg(self, MP_QSTR_scale);
+    }
+    if (bitmap->bits_per_value != 16 || bitmap->read_only) {
+        return use_software_for_arg(self, MP_QSTR_bitmap);
+    }
+    // Skip indices are slower on this path than in TJpgDec.
+    if (!self->skip_source_index_none) {
+        return use_software_for_arg(self, MP_QSTR_skip_source_index);
+    }
+    if (!self->skip_dest_index_none) {
+        return use_software_for_arg(self, MP_QSTR_skip_dest_index);
+    }
+    // The engine reads the input by DMA, so it must be in DMA-capable memory.
+    if (!esp_ptr_dma_capable(jpeg->buf) && !esp_ptr_dma_ext_capable(jpeg->buf)) {
+        return use_software_for_arg(self, MP_QSTR_data_source);
+    }
+
+    jpeg_decode_picture_info_t info;
+    if (jpeg_decoder_get_info(jpeg->buf, jpeg->len, &info) != ESP_OK) {
+        return use_software(self, MP_ERROR_TEXT("Format not supported"));
+    }
+
+    // The engine pads its output to whole MCUs: 8x8 for 4:4:4, 16x8 for 4:2:2
+    // and 16x16 for 4:2:0. Grayscale is left to TJpgDec.
+    uint32_t mcu_width, mcu_height;
+    switch (info.sample_method) {
+        case JPEG_DOWN_SAMPLING_YUV444:
+            mcu_width = 8;
+            mcu_height = 8;
+            break;
+        case JPEG_DOWN_SAMPLING_YUV422:
+            mcu_width = 16;
+            mcu_height = 8;
+            break;
+        case JPEG_DOWN_SAMPLING_YUV420:
+            mcu_width = 16;
+            mcu_height = 16;
+            break;
+        default:
+            return use_software(self, MP_ERROR_TEXT("Format not supported"));
+    }
+    uint32_t width = info.width;
+    uint32_t height = info.height;
+    // The driver rejects images whose pixel count is not a multiple of 8.
+    if ((width * height) % 8 != 0) {
+        return use_software(self, MP_ERROR_TEXT("Format not supported"));
+    }
+    uint32_t output_width = (width + mcu_width - 1) / mcu_width * mcu_width;
+    uint32_t output_height = (height + mcu_height - 1) / mcu_height * mcu_height;
+    size_t output_size = output_width * output_height * sizeof(uint16_t);
+
+    // Decode straight into the Bitmap when it is exactly the image, unpadded and
+    // cache-line aligned, and the whole image lands at 0,0.
+    bool direct = width == output_width && height == output_height &&
+        bitmap->width == width && bitmap->height == height &&
+        self->x == 0 && self->y == 0 &&
+        self->lim.x1 <= 0 && self->lim.y1 <= 0 &&
+        self->lim.x2 >= (int)width && self->lim.y2 >= (int)height &&
+        output_aligned(bitmap->data, output_size);
+
+    // Otherwise decode into a temporary buffer and copy the visible part out of it.
+    uint8_t *output;
+    size_t output_buffer_size;
+    uint8_t *scratch = NULL;
+    if (direct) {
+        output = (uint8_t *)bitmap->data;
+        output_buffer_size = output_size;
+    } else {
+        jpeg_decode_memory_alloc_cfg_t mem_config = {
+            .buffer_direction = JPEG_DEC_ALLOC_OUTPUT_BUFFER,
+        };
+        scratch = jpeg_alloc_decoder_mem(output_size, &mem_config, &output_buffer_size);
+        if (scratch == NULL) {
+            return use_software(self, MP_ERROR_TEXT("Operation or feature not supported"));
+        }
+        output = scratch;
+    }
+
+    // RGB order with RGB565 makes the engine emit the high byte first, the same
+    // byte-swapped RGB565 that TJpgDec produces here. JFIF files use full range
+    // BT601 YCbCr.
+    jpeg_decode_cfg_t decode_config = {
+        .output_format = JPEG_DECODE_OUT_FORMAT_RGB565,
+        .rgb_order = JPEG_DEC_RGB_ELEMENT_ORDER_RGB,
+        .conv_std = JPEG_YUV_RGB_CONV_STD_BT601,
+        .yuv_range = COLOR_RANGE_FULL,
+    };
+    uint32_t decoded_size;
+    if (jpeg_decoder_process(self->hw_decoder, &decode_config, jpeg->buf, jpeg->len,
+        output, output_buffer_size, &decoded_size) != ESP_OK) {
+        // A failed direct decode may have written part of the Bitmap. TJpgDec
+        // then decodes the whole image over it.
+        heap_caps_free(scratch);
+        return use_software(self, MP_ERROR_TEXT("Format not supported"));
+    }
+
+    if (direct) {
+        displayio_area_t area = { 0, 0, width, height, NULL };
+        displayio_bitmap_set_dirty_area(bitmap, &area);
+        return true;
+    }
+
+    // Copy out the visible part with the same clipping TJpgDec's output callback
+    // applies, treating the whole image as a single block at 0,0.
+    displayio_bitmap_t src = {
+        .width = width,
+        .height = height,
+        .data = (uint32_t *)(void *)output,
+        .stride = output_width / 2, // in units of uint32_t
+        .bits_per_value = 16,
+        .x_shift = 1,
+        .x_mask = 1,
+        .bitmask = 0xffff,
+    };
+    int x = self->x;
+    int y = self->y;
+    int x1 = self->lim.x1;
+    int y1 = self->lim.y1;
+    int x2 = MIN(self->lim.x2, (int)width);
+    int y2 = MIN(self->lim.y2, (int)height);
+    if (x1 < 0) {
+        x += -x1;
+        x1 = 0;
+    }
+    if (y1 < 0) {
+        y += -y1;
+        y1 = 0;
+    }
+    if (x2 > 0 && y2 > 0) {
+        common_hal_bitmaptools_blit(bitmap, &src, x, y, x1, y1, x2, y2,
+            self->skip_source_index, self->skip_source_index_none,
+            self->skip_dest_index, self->skip_dest_index_none);
+    }
+    heap_caps_free(scratch);
+    return true;
+}

ports/espressif/mpconfigport.mk

@@ -404,6 +404,9 @@ CIRCUITPY_ESPCAMERA = 0
 # P4 has MIPI-DSI
 CIRCUITPY_MIPIDSI = 1
 
+# Hardware JPEG decoder
+CIRCUITPY_JPEGIO_HW ?= $(CIRCUITPY_JPEGIO)
+
 #### esp32s31 #########################################################
 else ifeq ($(IDF_TARGET),esp32s31)
 
@@ -438,6 +441,9 @@ CIRCUITPY_ESPULP = 0
 # The S31 has five eFuse key blocks, not six.
 CIRCUITPY_HARDWAREKEY = 0
 
+# Hardware JPEG decoder
+CIRCUITPY_JPEGIO_HW ?= $(CIRCUITPY_JPEGIO)
+
 # No I80 support from the IDF
 CIRCUITPY_PARALLELDISPLAYBUS = 0
 

ports/espressif/supervisor/port.c

@@ -343,9 +343,11 @@ void *port_malloc(size_t size, bool dma_capable) {
     }
 
     void *ptr = NULL;
-    // Try SPIRAM first if available.
+    // Try SPIRAM first if available. Aligned to the GC block size so heap
+    // areas (and with 64-byte GC blocks, every GC allocation in them) are
+    // cache-line aligned, which video DMA destinations require.
     #ifdef CONFIG_SPIRAM
-    ptr = heap_caps_malloc(size, MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM);
+    ptr = heap_caps_aligned_alloc(MICROPY_BYTES_PER_GC_BLOCK, size, MALLOC_CAP_8BIT | MALLOC_CAP_SPIRAM);
     #endif
     if (ptr == NULL) {
         ptr = heap_caps_malloc(size, MALLOC_CAP_8BIT);

py/circuitpy_mpconfig.mk

@@ -426,6 +426,10 @@ CFLAGS += -DCIRCUITPY_IS31FL3741=$(CIRCUITPY_IS31FL3741)
 CIRCUITPY_JPEGIO ?= $(CIRCUITPY_BITMAPTOOLS)
 CFLAGS += -DCIRCUITPY_JPEGIO=$(CIRCUITPY_JPEGIO)
 
+# Port provides a hardware JPEG decode path for jpegio.
+CIRCUITPY_JPEGIO_HW ?= 0
+CFLAGS += -DCIRCUITPY_JPEGIO_HW=$(CIRCUITPY_JPEGIO_HW)
+
 CIRCUITPY_JSON ?= $(CIRCUITPY_FULL_BUILD)
 CFLAGS += -DCIRCUITPY_JSON=$(CIRCUITPY_JSON)
 

shared-bindings/jpegio/JpegDecoder.c

@@ -37,18 +37,25 @@
 //|
 //|     """
 //|
-//|     def __init__(self) -> None:
-//|         """Create a JpegDecoder"""
+//|     def __init__(self, *, software_ok: bool = True) -> None:
+//|         """Create a JpegDecoder
+//|
+//|         :param bool software_ok: If True, `decode` uses the software decoder when the JPEG
+//|           decoder peripheral cannot decode the image. If False, `decode` raises `ValueError`
+//|           instead, including on chips without the peripheral.
+//|         """
 //|         ...
 //|
 static mp_obj_t jpegio_jpegdecoder_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) {
+    enum { ARG_software_ok };
     static const mp_arg_t allowed_args[] = {
+        { MP_QSTR_software_ok, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = true} },
     };
     mp_arg_val_t args[MP_ARRAY_SIZE(allowed_args)];
     mp_arg_parse_all_kw_array(n_args, n_kw, all_args, MP_ARRAY_SIZE(allowed_args), allowed_args, args);
 
-    jpegio_jpegdecoder_obj_t *self = mp_obj_malloc(jpegio_jpegdecoder_obj_t, &jpegio_jpegdecoder_type);
-    common_hal_jpegio_jpegdecoder_construct(self);
+    jpegio_jpegdecoder_obj_t *self = mp_obj_malloc_with_finaliser(jpegio_jpegdecoder_obj_t, &jpegio_jpegdecoder_type);
+    common_hal_jpegio_jpegdecoder_construct(self, args[ARG_software_ok].u_bool);
 
     return MP_OBJ_FROM_PTR(self);
 }
@@ -124,6 +131,14 @@ MP_DEFINE_CONST_FUN_OBJ_2(jpegio_jpegdecoder_open_obj, jpegio_jpegdecoder_open);
 //|         possible to repeatedly ``decode`` the same jpeg data, even if it is to
 //|         select different scales or crop regions from it.
 //|
+//|         On chips with a JPEG decoder peripheral (ESP32-P4 and ESP32-S31), ``decode``
+//|         uses the peripheral for baseline color JPEGs when the data was opened from a
+//|         buffer in DMA-capable memory, ``scale`` is 0, ``bitmap`` holds 16-bit values,
+//|         and no skip index is given. It is fastest when ``bitmap`` is exactly the image
+//|         size and the whole image is decoded at 0,0. Colors may differ slightly from the
+//|         software decoder's. Other images, and data opened from a file or stream, use
+//|         the software decoder, or raise `ValueError` if ``software_ok`` is False.
+//|
 //|         :param Bitmap bitmap: Optional output buffer
 //|         :param int scale: Scale factor from 0 to 3, inclusive.
 //|         :param int x: Horizontal pixel location in bitmap where source_bitmap upper-left
@@ -195,9 +210,18 @@ static mp_obj_t jpegio_jpegdecoder_decode(size_t n_args, const mp_obj_t *pos_arg
 }
 static MP_DEFINE_CONST_FUN_OBJ_KW(jpegio_jpegdecoder_decode_obj, 1, jpegio_jpegdecoder_decode);
 
+// Releases any hardware decoder the object holds when it is collected.
+static mp_obj_t jpegio_jpegdecoder_del(mp_obj_t self_in) {
+    jpegio_jpegdecoder_obj_t *self = MP_OBJ_TO_PTR(self_in);
+    common_hal_jpegio_jpegdecoder_deinit(self);
+    return mp_const_none;
+}
+static MP_DEFINE_CONST_FUN_OBJ_1(jpegio_jpegdecoder_del_obj, jpegio_jpegdecoder_del);
+
 static const mp_rom_map_elem_t jpegio_jpegdecoder_locals_dict_table[] = {
     { MP_ROM_QSTR(MP_QSTR_open), MP_ROM_PTR(&jpegio_jpegdecoder_open_obj) },
     { MP_ROM_QSTR(MP_QSTR_decode), MP_ROM_PTR(&jpegio_jpegdecoder_decode_obj) },
+    { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&jpegio_jpegdecoder_del_obj) },
 };
 static MP_DEFINE_CONST_DICT(jpegio_jpegdecoder_locals_dict, jpegio_jpegdecoder_locals_dict_table);
 

shared-bindings/jpegio/JpegDecoder.h

@@ -15,7 +15,8 @@ extern const mp_obj_type_t jpegio_jpegdecoder_type;
 
 typedef struct jpegio_jpegdecoder_obj jpegio_jpegdecoder_obj_t;
 
-void common_hal_jpegio_jpegdecoder_construct(jpegio_jpegdecoder_obj_t *self);
+void common_hal_jpegio_jpegdecoder_construct(jpegio_jpegdecoder_obj_t *self, bool software_ok);
+void common_hal_jpegio_jpegdecoder_deinit(jpegio_jpegdecoder_obj_t *self);
 void common_hal_jpegio_jpegdecoder_close(jpegio_jpegdecoder_obj_t *self);
 mp_obj_t common_hal_jpegio_jpegdecoder_set_source_buffer(jpegio_jpegdecoder_obj_t *self, mp_obj_t jpeg_data);
 mp_obj_t common_hal_jpegio_jpegdecoder_set_source_file(jpegio_jpegdecoder_obj_t *self, mp_obj_t file_obj);