diff --git a/build_mp.py b/build_mp.py index 804f0f15..f14d564b 100755 --- a/build_mp.py +++ b/build_mp.py @@ -1293,6 +1293,19 @@ def num(v): cursor += num(r[4]) after = rows[idx + 1:] moved = (", ".join(f"'{r[0]}'" for r in after) + " moved to follow it. ") if after else "" + # A table that names its filesystem gives it a size, and growing the app + # pushes that partition past the end of the flash. The filesystem is the + # part that can shrink: cut the last data partition to what is left. + flash_bytes = int(flash.group(1)) * 1024 * 1024 if flash else 0 + if after and flash_bytes and cursor > flash_bytes: + last = after[-1] + if last[1] == "data" and last[2] in ("fat", "littlefs", "spiffs"): + room = flash_bytes - num(last[3]) + if room < 0x10000: + die(f"autosize: the app grew to {hex(size)} and leaves no room for '{last[0]}' in {flash.group(1)} MB") + last[4] = hex(room) + cursor = flash_bytes + moved += f"'{last[0]}' cut to {hex(room)} to fit the flash. " if any(r[1] == "data" and r[2] in ("fat", "littlefs", "spiffs") for r in after): fs = ["# The filesystem's partition moved with them."] else: diff --git a/docs/guides/use-esp-vision-modules.md b/docs/guides/use-esp-vision-modules.md index 6c35037a..f133757f 100644 --- a/docs/guides/use-esp-vision-modules.md +++ b/docs/guides/use-esp-vision-modules.md @@ -78,6 +78,38 @@ These ran on an ESP32-P4 board, each with an example. The arena is what Each example's docstring says where to download its model. +## Detect in a thread + +`espdl` lets go of the interpreter while a model runs, so detection on a +`_thread` leaves the main thread free: a display, a UI or a recording loop +keeps running through each `detect()`, `classify()` or `predict()`. Only the +image preparation before the model and the results after it hold the +interpreter. Use each model object from one thread at a time, and don't call +its `deinit()` while it is detecting. +[`espdl_thread.py`](https://github.com/PyDevices/micropython-pydevices/blob/main/modules/vision/examples/espdl_thread.py) +detects on a thread and measures the main loop's longest pause meanwhile. + +## Faces with landmarks + +`espdl.HumanFaceDetect(msr_path, mnp_path, *, score=0.5, nms=0.5, mnp_score=0.5, mnp_nms=0.5)` +runs ESP-DL's two-stage face detector: MSR finds candidates in the whole +picture, then MNP looks at each one closely and places five key points. +`detect(img)` takes an RGB565 `image.Image` and returns a list of +`(x, y, w, h, score, keypoints)`, where `keypoints` is ten ints, five (x, y) +pairs in ESP-DL's order: left eye, left mouth corner, nose, right eye, right +mouth corner. `set_thresholds()` takes the same four keywords as the +constructor, and the whole detection runs without the interpreter's lock. + +The two model files come from ESP-DL's +[`models/human_face_detect`](https://github.com/espressif/esp-dl/tree/master/models/human_face_detect), +in its `models/p4` or `models/s3` folder: `human_face_detect_msr_s8_v1.espdl` +and `human_face_detect_mnp_s8_v1.espdl`. They are MIT-licensed, by Espressif; +the firmware doesn't carry them, so copy them to the board's filesystem and pass +their paths. Release 0.5.0 of the `espressif/human_face_detect` component is +the one for the ESP-DL this firmware is built with. +[`espdl_face_landmarks.py`](https://github.com/PyDevices/micropython-pydevices/blob/main/modules/vision/examples/espdl_face_landmarks.py) +marks the faces in a photo. + ## Run long scripts as `main.py` An mpftp `exec` that outlasts mpftp's wait resets the board over USB diff --git a/docs/how-it-works/esp-vision.md b/docs/how-it-works/esp-vision.md index 441f0d1b..60f83f1f 100644 --- a/docs/how-it-works/esp-vision.md +++ b/docs/how-it-works/esp-vision.md @@ -14,7 +14,7 @@ The leading dot keeps it out of `--modules` and `all`, because esp-vision's own [`modules/tflite`](https://github.com/PyDevices/micropython-pydevices/tree/main/modules/tflite) and [`modules/vision`](https://github.com/PyDevices/micropython-pydevices/tree/main/modules/vision) are our glue around esp-vision's files: the ESP-IDF components esp-vision uses, at its versions; the include paths and definitions MicroPython's QSTR scan needs (it sees only a module's own paths, -never a linked component's); and four pieces of our own. +never a linked component's); and five pieces of our own. - **imlib compiles inside the module**, not as an ESP-IDF component. As a component it would wait for MicroPython's generated headers while being one of @@ -32,6 +32,12 @@ never a linked component's); and four pieces of our own. those models use is in esp-vision's list already. - **`src/vision_preview_none.c`** stands in for esp-vision's preview, which streams frames to its IDE over its own USB multiplexer. +- **`esp-vision-patches/`** holds two small changes to esp-vision's + `py_espdl.cpp`: `espdl` releases the GIL while a model runs, and its module + table lists `HumanFaceDetect` from **`src/vision_espdl_face.cpp`**. The + build applies them, in order, to a copy of the file in the build tree, so + the pinned checkout stays as it is, and a pin move that breaks one stops the + build with the patch's name. - **`src/vision_h264.c`** is `h264`'s encoder, in place of esp-vision's `platform/h264.c`. It is that file with one addition: the encoder in a P4 before revision 3 reads only packed YUV420 and refuses RGB565, so firmware diff --git a/modules/.esp-vision/ESP_VISION.lock b/modules/.esp-vision/ESP_VISION.lock index b2377182..79a7b626 100644 --- a/modules/.esp-vision/ESP_VISION.lock +++ b/modules/.esp-vision/ESP_VISION.lock @@ -1,5 +1,7 @@ # esp-vision (espressif/esp-vision, Apache-2.0): the source of our tflite # module and, later, its image stack. Pinned by full commit; fetch.sh puts it # at modules/.esp-vision/src. Its files are compiled as they are: our glue -# lives in the module directories that use it (modules/tflite). +# lives in the module directories that use it (modules/tflite, modules/vision), +# and modules/vision/esp-vision-patches changes py_espdl.cpp in a build-tree +# copy, never here. esp-vision https://github.com/espressif/esp-vision.git ecda8004b95e1df275ac2a1f9cdf487d03defc63 diff --git a/modules/h264enc/README.md b/modules/h264enc/README.md index fdb75e15..e6dad000 100644 --- a/modules/h264enc/README.md +++ b/modules/h264enc/README.md @@ -26,7 +26,7 @@ take it too (`--interpreter circuitpython --modules h264enc,ppa`), and ## API -### `h264enc.Encoder(width, height, fps=30, gop=30, bitrate=3_000_000, qp_min=10, qp_max=45, *, canvas_w=0, canvas_h=0, out_size=0)` +### `h264enc.Encoder(width, height, fps=30, gop=30, bitrate=3_000_000, qp_min=10, qp_max=45, *, canvas_w=0, canvas_h=0, out_size=0, pipeline=False)` `width` and `height` are multiples of 16, from 80x80 to 1920x2032. `gop` is the frames from one IDR (key) frame to the next. With `canvas_w` / `canvas_h` @@ -35,6 +35,8 @@ puts a 720x720 panel on a 1280x720 TV. `out_size` caps one access unit. The default holds a keyframe of the whole canvas at its raw size (1.4 MB at 1280x720, in PSRAM), because a busy picture such as a camera's can encode bigger than any fixed guess, and a frame that doesn't fit stops the encoder. +`pipeline=True` overlaps the colour conversion of one frame with the encode +of the one before; see [Pipelined encoding](#pipelined-encoding). The P4 has one H.264 encoder, so one `Encoder` (or one cast) at a time: a second raises `OSError` saying it is busy, and the first carries on. `close()` @@ -44,14 +46,60 @@ gives the encoder back; so does the object being collected, and `with` works. One frame of native-order RGB565 (what displaydev framebuffers hold), at least `width * height * 2` bytes. Returns one access unit: Annex B NAL units, SPS -and PPS included before each IDR. +and PPS included before each IDR. A pipelined encoder returns the previous +frame's access unit instead, and `b""` for the first frame. + +### `flush() -> bytes` + +A pipelined encoder's last access unit, the one `encode()` hasn't returned +yet, or `b""` when there is none. A plain encoder always returns `b""`. ### `keyframe`, `force_idr()`, `set_bitrate(bps)`, `stats()` -`keyframe` says whether the last `encode()` made an IDR frame. `force_idr()` -makes the next one an IDR (a viewer joining late needs one). `set_bitrate()` -takes effect from the next frame. `stats()` is a dict: `frames`, `keyframes`, -`bytes`, `errors`, and the last frame's `ppa_us` and `enc_us`. +`keyframe` says whether the access unit `encode()` (or `flush()`) last +returned is an IDR frame. `force_idr()` makes the next frame the encoder +starts an IDR (a viewer joining late needs one). `set_bitrate()` takes effect +from the next frame. `stats()` is a dict: `frames` and `keyframes` returned so +far, `bytes`, `errors`, the last conversion's `ppa_us` and the last encode's +`enc_us`, and `wall_us`, how long the last `encode()` kept its caller. +`pipeline` says which kind of encoder this is. `close()` returns what +`flush()` would have, or `None` for a plain encoder; leaving a `with` block +discards it, so flush first. + +## Pipelined encoding + +The colour conversion runs on the PPA and the encode on the H.264 engine, two +separate pieces of hardware. A plain encoder runs them one after the other, so +a frame costs their sum: at 1920x1072 about 40 ms to convert and 38 ms to +encode, 12 frames a second. A pipelined encoder converts frame N on the PPA +while frame N-1 is still being encoded, so a frame costs about the longer of +the two. + +```python +enc = h264enc.Encoder(1920, 1072, fps=25, gop=25, pipeline=True) +with open("clip.h264", "wb") as f: + for frame in frames: + f.write(enc.encode(frame)) # the previous frame's access unit + f.write(enc.flush()) # and the last one +enc.close() +``` + +The price is one frame of latency: each `encode()` returns the frame before. +`encode()` has finished reading `frame` when it returns, so you can capture the +next picture into the same buffer straight away. The encoder keeps two YUV +canvases in PSRAM rather than one (3 MB each at 1920x1088) and starts one +small task the first time a pipelined encoder opens, and keeps it. + +It's off by default because the latency changes what `encode()` returns, and +code that pairs each frame with its own access unit (writing timestamps, or +checking `keyframe` straight after) would quietly be off by one. + +The PPA does one operation at a time. Any other `ppa.srm()` call, such as a +preview cropping and scaling each camera frame for a screen, waits while the +encoder's conversion runs, and the conversion waits for it. A preview's PPA time +adds to the conversion's, so with a preview running at 1080p the PPA, not +the encoder, sets the frame rate. `stats()["ppa_us"]` and your own preview's +timing show how close the two together come to the frame period. ## Colour diff --git a/modules/h264enc/src/mod_h264enc.c b/modules/h264enc/src/mod_h264enc.c index 6eb3d24c..d725bcd0 100644 --- a/modules/h264enc/src/mod_h264enc.c +++ b/modules/h264enc/src/mod_h264enc.c @@ -5,6 +5,10 @@ // au = enc.encode(framebuffer) # bytes: Annex B NAL units // enc.keyframe; enc.force_idr(); enc.set_bitrate(n); enc.stats(); enc.close() // +// enc = h264enc.Encoder(1920, 1072, fps=25, pipeline=True) +// au = enc.encode(frame) # the previous frame's access unit (b"" at first) +// au = enc.flush() # the last one +// // ppa's C API (the PPA) converts each frame to the packed YUV420 the encoder reads (the only // raw format the pre-revision-3 P4 takes), placed on a canvas that may be // larger than the frame; esp_h264 encodes it. Limited-range BT.601, which is @@ -13,6 +17,13 @@ // The same session is available to C (h264enc.h): castif's cast task encodes // through it on core 0. The P4 has one encoder, so one session at a time; a // second open gets "busy" and the first carries on. +// +// With pipeline=True the session has two canvases and a worker task. encode() +// converts this frame on the PPA (on the caller's thread, so the frame is free +// again when encode() returns) while the worker encodes the previous one, then +// hands the worker this frame and returns the previous frame's access unit. +// The PPA and the encoder are separate engines, so a frame costs about the +// longer of the two rather than their sum. #include @@ -23,7 +34,12 @@ // IDF's: the IDF headers stay out of it. #ifndef NO_QSTR #include "esp_heap_caps.h" +#include "esp_task.h" #include "esp_timer.h" +#include "freertos/FreeRTOS.h" +#include "freertos/semphr.h" +#include "freertos/task.h" +#include "freertos/idf_additions.h" #include "esp_h264_enc_single_hw.h" #include "esp_h264_alloc.h" #endif @@ -69,7 +85,15 @@ struct h264enc_session { bool enc_open; ppa_mod_client_t *ppa; uint8_t *yuv; + uint8_t *yuv2; // the second canvas, pipeline only uint32_t yuv_len; + bool pipeline; + uint8_t *next; // the canvas the next frame is converted into + bool busy; // the worker has a frame + bool ready; // out holds an access unit nobody has taken yet + int job_err; // the worker's result for the frame it finished + uint32_t job_len; + bool job_idr; uint32_t in_len; // the packed YUV420 canvas the encoder reads uint8_t *out; uint32_t out_cap; @@ -81,12 +105,73 @@ struct h264enc_session { // The one session the hardware can hold. static h264enc_session_t *h264enc_owner; +// --- the pipeline's worker --------------------------------------------------- +// +// One task for the life of the firmware, made by the first pipelined open: +// there is one encoder, so there is never more than one session to serve, and +// a task that is never deleted can't be freed under a session that is closing. + +static TaskHandle_t h264enc_worker_task; +static SemaphoreHandle_t h264enc_go, h264enc_done; +static h264enc_session_t *volatile h264enc_job; + +static int h264enc_run(h264enc_session_t *s, uint8_t *yuv, uint32_t *len, bool *idr); + +static void h264enc_worker(void *arg) { + (void)arg; + for (;;) { + xSemaphoreTake(h264enc_go, portMAX_DELAY); + h264enc_session_t *s = h264enc_job; + s->job_err = h264enc_run(s, s->next == s->yuv ? s->yuv2 : s->yuv, &s->job_len, &s->job_idr); + xSemaphoreGive(h264enc_done); + } +} + +static const char *h264enc_worker_start(void) { + if (h264enc_worker_task != NULL) { + return NULL; + } + if (h264enc_go == NULL) { + h264enc_go = xSemaphoreCreateBinary(); + h264enc_done = xSemaphoreCreateBinary(); + if (h264enc_go == NULL || h264enc_done == NULL) { + return "no memory for the pipeline's semaphores"; + } + } + // The encoder call blocks on its interrupt for nearly all of a frame; the + // stack is for its rate control and its error log. + if (xTaskCreatePinnedToCore(h264enc_worker, "h264enc", 8192, NULL, ESP_TASK_PRIO_MIN + 2, + &h264enc_worker_task, tskNO_AFFINITY) != pdPASS) { + h264enc_worker_task = NULL; + return "no memory for the pipeline's task"; + } + return NULL; +} + +// Waits for the frame with the worker, if any; its access unit is then in out. +static void h264enc_wait(h264enc_session_t *s) { + if (s->busy) { + xSemaphoreTake(h264enc_done, portMAX_DELAY); + s->busy = false; + s->ready = true; + } +} + +// Hands the canvas just converted (s->next) to the worker and flips canvases. +static void h264enc_submit(h264enc_session_t *s) { + s->next = s->next == s->yuv ? s->yuv2 : s->yuv; + s->busy = true; + h264enc_job = s; + xSemaphoreGive(h264enc_go); +} + // --- C API ------------------------------------------------------------------- void h264enc_close(h264enc_session_t *s) { if (s == NULL) { return; } + h264enc_wait(s); if (s->enc_open) { esp_h264_enc_close(s->enc); } @@ -99,6 +184,9 @@ void h264enc_close(h264enc_session_t *s) { if (s->yuv) { esp_h264_free(s->yuv); } + if (s->yuv2) { + esp_h264_free(s->yuv2); + } if (s->out) { esp_h264_free(s->out); } @@ -108,8 +196,33 @@ void h264enc_close(h264enc_session_t *s) { heap_caps_free(s); } +// A canvas whose border is limited-range black (packed U Y Y / V Y Y). +static uint8_t *h264enc_canvas(h264enc_session_t *s) { + uint32_t got; + uint8_t *p = esp_h264_aligned_calloc(128, 1, (s->in_len + 127) & ~127u, &got, ESP_H264_MEM_SPIRAM); + if (p == NULL) { + return NULL; + } + s->yuv_len = got; + for (uint32_t i = 0; i + 2 < s->in_len; i += 3) { + p[i] = 0x80; + p[i + 1] = 0x10; + p[i + 2] = 0x10; + } + return p; +} + +static const char *h264enc_open_pipelined(h264enc_session_t **out, int width, int height, int canvas_w, + int canvas_h, int fps, int gop, int bitrate, int qp_min, int qp_max, uint32_t out_size, bool pipeline); + const char *h264enc_open(h264enc_session_t **out, int width, int height, int canvas_w, int canvas_h, int fps, int gop, int bitrate, int qp_min, int qp_max, uint32_t out_size) { + return h264enc_open_pipelined(out, width, height, canvas_w, canvas_h, fps, gop, bitrate, qp_min, qp_max, + out_size, false); +} + +static const char *h264enc_open_pipelined(h264enc_session_t **out, int width, int height, int canvas_w, + int canvas_h, int fps, int gop, int bitrate, int qp_min, int qp_max, uint32_t out_size, bool pipeline) { *out = NULL; if (h264enc_owner != NULL) { return "busy"; @@ -145,16 +258,18 @@ const char *h264enc_open(h264enc_session_t **out, int width, int height, int can why = "no memory for a ppa client"; goto fail; } - s->yuv = esp_h264_aligned_calloc(128, 1, (s->in_len + 127) & ~127u, &s->yuv_len, ESP_H264_MEM_SPIRAM); + s->yuv = h264enc_canvas(s); if (s->yuv == NULL) { why = "no memory for the YUV canvas"; goto fail; } - // The canvas border stays limited-range black (packed U Y Y / V Y Y). - for (uint32_t i = 0; i + 2 < s->in_len; i += 3) { - s->yuv[i] = 0x80; - s->yuv[i + 1] = 0x10; - s->yuv[i + 2] = 0x10; + s->next = s->yuv; + if (pipeline) { + s->yuv2 = h264enc_canvas(s); + if (s->yuv2 == NULL) { + why = "no memory for the second YUV canvas"; + goto fail; + } } esp_h264_enc_cfg_hw_t cfg = {0}; cfg.pic_type = ESP_H264_RAW_FMT_O_UYY_E_VYY; @@ -196,6 +311,15 @@ const char *h264enc_open(h264enc_session_t **out, int width, int height, int can why = "no memory for the output buffer"; goto fail; } + // After the encoder, whose reference rows want the largest piece of + // internal RAM: the task's stack comes from what is left. + if (pipeline) { + why = h264enc_worker_start(); + if (why != NULL) { + goto fail; + } + s->pipeline = true; + } h264enc_owner = s; *out = s; return NULL; @@ -205,41 +329,58 @@ const char *h264enc_open(h264enc_session_t **out, int width, int height, int can return why; } -int h264enc_encode(h264enc_session_t *s, const uint8_t *rgb565, const uint8_t **data, uint32_t *len, bool *idr) { +// RGB565 to the packed YUV420 the encoder reads, placed centred on the +// canvas; studio range, which is what H.264 decoders assume. +static int h264enc_convert(h264enc_session_t *s, const uint8_t *rgb565, uint8_t *yuv) { int64_t t0 = esp_timer_get_time(); - // RGB565 to the packed YUV420 the encoder reads, placed centred on the - // canvas; studio range, which is what H.264 decoders assume ppa_mod_srm_t op = { .src = rgb565, .src_w = s->w, .src_h = s->h, .src_fmt = H264ENC_PPA_RGB565, - .dst = s->yuv, .dst_len = s->yuv_len, .dst_w = s->cw, .dst_h = s->ch, .dst_fmt = H264ENC_PPA_YUV420, + .dst = yuv, .dst_len = s->yuv_len, .dst_w = s->cw, .dst_h = s->ch, .dst_fmt = H264ENC_PPA_YUV420, .x = ((s->cw - s->w) / 2) & ~1u, .y = ((s->ch - s->h) / 2) & ~1u, .w = s->w, .h = s->h, .yuv_limited = true, }; const char *perr = ppa_mod_srm(s->ppa, &op); - int64_t t1 = esp_timer_get_time(); - s->ppa_us = (uint32_t)(t1 - t0); - if (perr != NULL) { - return -1; - } + s->ppa_us = (uint32_t)(esp_timer_get_time() - t0); + return perr != NULL ? -1 : 0; +} + +// One canvas through the encoder into out. +static int h264enc_run(h264enc_session_t *s, uint8_t *yuv, uint32_t *len, bool *idr) { + int64_t t0 = esp_timer_get_time(); esp_h264_enc_in_frame_t inf = {0}; - inf.raw_data.buffer = s->yuv; + inf.raw_data.buffer = yuv; inf.raw_data.len = s->in_len; inf.pts = s->frames; esp_h264_enc_out_frame_t outf = {0}; outf.raw_data.buffer = s->out; outf.raw_data.len = s->out_cap; esp_h264_err_t err = esp_h264_enc_process(s->enc, &inf, &outf); - s->enc_us = (uint32_t)(esp_timer_get_time() - t1); + s->enc_us = (uint32_t)(esp_timer_get_time() - t0); if (err != ESP_H264_ERR_OK) { return (int)err; } s->frames++; - *data = s->out; *len = outf.length; *idr = outf.frame_type == ESP_H264_FRAME_TYPE_IDR; return 0; } +int h264enc_encode(h264enc_session_t *s, const uint8_t *rgb565, const uint8_t **data, uint32_t *len, bool *idr) { + if (s->pipeline) { + return -2; + } + int err = h264enc_convert(s, rgb565, s->yuv); + if (err != 0) { + return err; + } + err = h264enc_run(s, s->yuv, len, idr); + if (err != 0) { + return err; + } + *data = s->out; + return 0; +} + void h264enc_force_idr(h264enc_session_t *s) { esp_h264_enc_force_idr((esp_h264_enc_param_handle_t)s->param); } @@ -260,6 +401,7 @@ typedef struct _h264enc_encoder_obj_t { h264enc_session_t *s; bool keyframe; // the last encode() was an IDR frame uint32_t frames, keyframes, errors; + uint32_t wall_us; // how long the last encode() kept its caller uint64_t bytes; } h264enc_encoder_obj_t; @@ -271,7 +413,8 @@ static h264enc_session_t *h264enc_get(h264enc_encoder_obj_t *self) { } static mp_obj_t h264enc_encoder_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) { - enum { ARG_width, ARG_height, ARG_fps, ARG_gop, ARG_bitrate, ARG_qp_min, ARG_qp_max, ARG_canvas_w, ARG_canvas_h, ARG_out_size }; + enum { ARG_width, ARG_height, ARG_fps, ARG_gop, ARG_bitrate, ARG_qp_min, ARG_qp_max, ARG_canvas_w, ARG_canvas_h, ARG_out_size, + ARG_pipeline }; static const mp_arg_t allowed[] = { { MP_QSTR_width, MP_ARG_REQUIRED | MP_ARG_INT, {.u_int = 0} }, { MP_QSTR_height, MP_ARG_REQUIRED | MP_ARG_INT, {.u_int = 0} }, @@ -283,13 +426,15 @@ static mp_obj_t h264enc_encoder_make_new(const mp_obj_type_t *type, size_t n_arg { MP_QSTR_canvas_w, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 0} }, { MP_QSTR_canvas_h, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 0} }, { MP_QSTR_out_size, MP_ARG_KW_ONLY | MP_ARG_INT, {.u_int = 0} }, + { MP_QSTR_pipeline, MP_ARG_KW_ONLY | MP_ARG_BOOL, {.u_bool = false} }, }; mp_arg_val_t args[MP_ARRAY_SIZE(allowed)]; mp_arg_parse_all_kw_array(n_args, n_kw, all_args, MP_ARRAY_SIZE(allowed), allowed, args); h264enc_session_t *s; - const char *why = h264enc_open(&s, args[ARG_width].u_int, args[ARG_height].u_int, + const char *why = h264enc_open_pipelined(&s, args[ARG_width].u_int, args[ARG_height].u_int, args[ARG_canvas_w].u_int, args[ARG_canvas_h].u_int, args[ARG_fps].u_int, args[ARG_gop].u_int, - args[ARG_bitrate].u_int, args[ARG_qp_min].u_int, args[ARG_qp_max].u_int, (uint32_t)args[ARG_out_size].u_int); + args[ARG_bitrate].u_int, args[ARG_qp_min].u_int, args[ARG_qp_max].u_int, (uint32_t)args[ARG_out_size].u_int, + args[ARG_pipeline].u_bool); if (why != NULL) { if (strcmp(why, "busy") == 0) { mp_raise_msg(&mp_type_OSError, MP_ERROR_TEXT("h264enc: the H.264 encoder is busy (a cast or another Encoder has it)")); @@ -300,11 +445,36 @@ static mp_obj_t h264enc_encoder_make_new(const mp_obj_type_t *type, size_t n_arg self->s = s; self->keyframe = false; self->frames = self->keyframes = self->errors = 0; + self->wall_us = 0; self->bytes = 0; return MP_OBJ_FROM_PTR(self); } -// encode(frame) -> bytes: one access unit, Annex B. +// Counts an access unit handed to Python, and makes its bytes. +static mp_obj_t h264enc_encoder_take(h264enc_encoder_obj_t *self, const uint8_t *data, uint32_t len, bool idr) { + self->keyframe = idr; + self->frames++; + self->keyframes += idr; + self->bytes += len; + return mp_obj_new_bytes(data, len); +} + +// The pipeline's finished access unit, or b"" when there is none; raises if +// the encoder failed on it. +static mp_obj_t h264enc_encoder_collect(h264enc_encoder_obj_t *self, h264enc_session_t *s) { + if (!s->ready) { + return mp_const_empty_bytes; + } + s->ready = false; + if (s->job_err != 0) { + self->errors++; + mp_raise_msg_varg(&mp_type_OSError, MP_ERROR_TEXT("h264enc: encode failed (%d)"), s->job_err); + } + return h264enc_encoder_take(self, s->out, s->job_len, s->job_idr); +} + +// encode(frame) -> bytes: one access unit, Annex B. Pipelined, the previous +// frame's (b"" for the first frame). static mp_obj_t h264enc_encoder_encode(mp_obj_t self_in, mp_obj_t frame) { h264enc_encoder_obj_t *self = MP_OBJ_TO_PTR(self_in); h264enc_session_t *s = h264enc_get(self); @@ -314,6 +484,40 @@ static mp_obj_t h264enc_encoder_encode(mp_obj_t self_in, mp_obj_t frame) { mp_raise_msg_varg(&mp_type_ValueError, MP_ERROR_TEXT("h264enc: a %dx%d frame is %d bytes, got %d"), s->w, s->h, s->w * s->h * 2, (int)buf.len); } + int64_t t0 = esp_timer_get_time(); + if (s->pipeline) { + // Convert this frame while the worker encodes the last one, then wait + // for it. The GIL is out for both, as below. + MP_THREAD_GIL_EXIT(); + int perr = h264enc_convert(s, buf.buf, s->next); + h264enc_wait(s); + MP_THREAD_GIL_ENTER(); + if (perr != 0) { + // Nothing goes to the worker, so frames still come out one for one: + // the access unit waiting now comes out of the next encode(). + self->wall_us = (uint32_t)(esp_timer_get_time() - t0); + self->errors++; + mp_raise_msg(&mp_type_OSError, MP_ERROR_TEXT("h264enc: the PPA could not convert the frame")); + } + // The access unit is copied out before the worker may write the next. + mp_obj_t au = mp_const_empty_bytes; + int jerr = 0; + if (s->ready) { + s->ready = false; + jerr = s->job_err; + if (jerr == 0) { + au = h264enc_encoder_take(self, s->out, s->job_len, s->job_idr); + } else { + self->errors++; + } + } + h264enc_submit(s); + self->wall_us = (uint32_t)(esp_timer_get_time() - t0); + if (jerr != 0) { + mp_raise_msg_varg(&mp_type_OSError, MP_ERROR_TEXT("h264enc: encode failed (%d)"), jerr); + } + return au; + } const uint8_t *data; uint32_t len; bool idr; @@ -324,18 +528,26 @@ static mp_obj_t h264enc_encoder_encode(mp_obj_t self_in, mp_obj_t frame) { MP_THREAD_GIL_EXIT(); int err = h264enc_encode(s, buf.buf, &data, &len, &idr); MP_THREAD_GIL_ENTER(); + self->wall_us = (uint32_t)(esp_timer_get_time() - t0); if (err != 0) { self->errors++; mp_raise_msg_varg(&mp_type_OSError, MP_ERROR_TEXT("h264enc: encode failed (%d)"), err); } - self->keyframe = idr; - self->frames++; - self->keyframes += idr; - self->bytes += len; - return mp_obj_new_bytes(data, len); + return h264enc_encoder_take(self, data, len, idr); } static MP_DEFINE_CONST_FUN_OBJ_2(h264enc_encoder_encode_obj, h264enc_encoder_encode); +// flush() -> bytes: the frame still in the pipeline, or b"". +static mp_obj_t h264enc_encoder_flush(mp_obj_t self_in) { + h264enc_encoder_obj_t *self = MP_OBJ_TO_PTR(self_in); + h264enc_session_t *s = h264enc_get(self); + MP_THREAD_GIL_EXIT(); + h264enc_wait(s); + MP_THREAD_GIL_ENTER(); + return h264enc_encoder_collect(self, s); +} +static MP_DEFINE_CONST_FUN_OBJ_1(h264enc_encoder_flush_obj, h264enc_encoder_flush); + static mp_obj_t h264enc_encoder_force_idr(mp_obj_t self_in) { h264enc_force_idr(h264enc_get(MP_OBJ_TO_PTR(self_in))); return mp_const_none; @@ -363,22 +575,51 @@ static mp_obj_t h264enc_encoder_stats(mp_obj_t self_in) { PUT(MP_QSTR_errors, self->errors); PUT(MP_QSTR_ppa_us, s->ppa_us); PUT(MP_QSTR_enc_us, s->enc_us); + PUT(MP_QSTR_wall_us, self->wall_us); #undef PUT return d; } static MP_DEFINE_CONST_FUN_OBJ_1(h264enc_encoder_stats_obj, h264enc_encoder_stats); -static mp_obj_t h264enc_encoder_close(mp_obj_t self_in) { - h264enc_encoder_obj_t *self = MP_OBJ_TO_PTR(self_in); +static void h264enc_encoder_release(h264enc_encoder_obj_t *self) { h264enc_close(self->s); self->s = NULL; - return mp_const_none; +} + +// close() -> bytes or None: pipelined, the last frame's access unit (b"" when +// there is none), as flush() would have returned it. +static mp_obj_t h264enc_encoder_close(mp_obj_t self_in) { + h264enc_encoder_obj_t *self = MP_OBJ_TO_PTR(self_in); + if (self->s == NULL) { + return mp_const_none; + } + mp_obj_t last = mp_const_none; + if (self->s->pipeline) { + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + last = h264enc_encoder_flush(self_in); + nlr_pop(); + } else { + h264enc_encoder_release(self); + nlr_jump(nlr.ret_val); + } + } + h264enc_encoder_release(self); + return last; } static MP_DEFINE_CONST_FUN_OBJ_1(h264enc_encoder_close_obj, h264enc_encoder_close); +// The finaliser and __exit__ give the encoder back without allocating: the +// collector can't make bytes, and __exit__ must not return a true value. +static mp_obj_t h264enc_encoder_del(mp_obj_t self_in) { + h264enc_encoder_release(MP_OBJ_TO_PTR(self_in)); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(h264enc_encoder_del_obj, h264enc_encoder_del); + static mp_obj_t h264enc_encoder_exit(size_t n_args, const mp_obj_t *args) { (void)n_args; - return h264enc_encoder_close(args[0]); + return h264enc_encoder_del(args[0]); } static MP_DEFINE_CONST_FUN_OBJ_VAR_BETWEEN(h264enc_encoder_exit_obj, 4, 4, h264enc_encoder_exit); @@ -389,6 +630,8 @@ static void h264enc_encoder_attr(mp_obj_t self_in, qstr attr, mp_obj_t *dest) { h264enc_encoder_obj_t *self = MP_OBJ_TO_PTR(self_in); if (attr == MP_QSTR_keyframe) { dest[0] = mp_obj_new_bool(self->keyframe); + } else if (attr == MP_QSTR_pipeline) { + dest[0] = mp_obj_new_bool(h264enc_get(self)->pipeline); } else if (attr == MP_QSTR_width || attr == MP_QSTR_height) { h264enc_session_t *s = h264enc_get(self); dest[0] = MP_OBJ_NEW_SMALL_INT(attr == MP_QSTR_width ? s->w : s->h); @@ -403,7 +646,8 @@ static const mp_rom_map_elem_t h264enc_encoder_locals_dict_table[] = { { MP_ROM_QSTR(MP_QSTR_set_bitrate), MP_ROM_PTR(&h264enc_encoder_set_bitrate_obj) }, { MP_ROM_QSTR(MP_QSTR_stats), MP_ROM_PTR(&h264enc_encoder_stats_obj) }, { MP_ROM_QSTR(MP_QSTR_close), MP_ROM_PTR(&h264enc_encoder_close_obj) }, - { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&h264enc_encoder_close_obj) }, + { MP_ROM_QSTR(MP_QSTR_flush), MP_ROM_PTR(&h264enc_encoder_flush_obj) }, + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&h264enc_encoder_del_obj) }, { MP_ROM_QSTR(MP_QSTR___enter__), MP_ROM_PTR(&mp_identity_obj) }, { MP_ROM_QSTR(MP_QSTR___exit__), MP_ROM_PTR(&h264enc_encoder_exit_obj) }, }; diff --git a/modules/h264enc/tests/hardware/test_h264enc.py b/modules/h264enc/tests/hardware/test_h264enc.py index d11af3bb..fc7ded48 100644 --- a/modules/h264enc/tests/hardware/test_h264enc.py +++ b/modules/h264enc/tests/hardware/test_h264enc.py @@ -11,6 +11,11 @@ while this one is open raises OSError "busy" without disturbing it. check_h264_ffmpeg.py then has ffmpeg decode the file on the PC. +Then the same frames through a pipelined Encoder: the first encode() returns +b"", every later one the frame before, flush() the last; the access units +match the plain Encoder's byte for byte up to the forced IDR; and the time a +frame keeps its caller (wall_us) is printed beside the plain one's. + run on the board with h264_frames.py beside it (mpftp probe; see the README), MicroPython or CircuitPython, with a /jt folder for the stream """ @@ -18,6 +23,11 @@ import sys import time +try: + from hashlib import sha256 +except ImportError: + sha256 = None + import h264enc try: @@ -54,6 +64,8 @@ def main(): keys = [] total = 0 enc_us = 0 + wall_us = 0 + digests = [] with open(OUT + "/h264enc.h264", "wb") as f: for k in range(N): frames.frame(bg, k, buf) @@ -66,6 +78,8 @@ def main(): keys.append(k) total += len(au) enc_us += enc.stats()["enc_us"] + enc.stats()["ppa_us"] + wall_us += enc.stats()["wall_us"] + digests.append(sha256(au).digest() if sha256 else len(au)) f.write(au) st = enc.stats() print("frames", st["frames"], "keyframes at", keys, "bytes", total, @@ -85,7 +99,36 @@ def main(): except ValueError: pass print("after close(): a new Encoder opens; the closed one refuses encode()") + pipelined(bg, buf, digests, wall_us // N) print("h264enc board test passed, %d frames, %.1f s" % (N, ticks_diff(ticks_ms(), t0) / 1000)) +def pipelined(bg, buf, digests, plain_wall_us): + # The same frames, pipelined, without the forced IDR: up to frame 44 the + # two streams must be the same bytes. + enc = h264enc.Encoder(frames.W, frames.H, 30, 30, 3_000_000, pipeline=True) + assert enc.pipeline + aus = [] + wall_us = 0 + for k in range(N): + frames.frame(bg, k, buf) + au = enc.encode(buf) + wall_us += enc.stats()["wall_us"] + if k == 0: + assert au == b"", "the first pipelined encode() returned %d bytes" % len(au) + else: + assert len(au) > 0 and au[:3] in (b"\x00\x00\x00", b"\x00\x00\x01"), au[:8] + aus.append(au) + aus.append(enc.flush()) + assert enc.flush() == b"", "a second flush() returned bytes" + st = enc.stats() + assert len(aus) == N and st["frames"] == N and st["errors"] == 0, (len(aus), st) + for k in range(45): + got = sha256(aus[k]).digest() if sha256 else len(aus[k]) + assert got == digests[k], "pipelined frame %d differs from the plain encode" % k + assert enc.close() == b"" + print("pipelined: %d frames, the same bytes as the plain Encoder up to frame 44" % N) + print("mean wall per frame: plain %d us, pipelined %d us" % (plain_wall_us, wall_us // N)) + + main() diff --git a/modules/vision/esp-vision-patches/0001-espdl-release-the-GIL-during-inference.patch b/modules/vision/esp-vision-patches/0001-espdl-release-the-GIL-during-inference.patch new file mode 100644 index 00000000..e12c8fbb --- /dev/null +++ b/modules/vision/esp-vision-patches/0001-espdl-release-the-GIL-during-inference.patch @@ -0,0 +1,73 @@ +--- a/modules/py_espdl.cpp ++++ b/modules/py_espdl.cpp +@@ -109,6 +109,16 @@ + float score_thr; + } espdl_imagenet_cls_obj_t; + ++// Inference runs without the GIL, so other threads keep running while the ++// model works. It touches only the model's own buffers; use one model object ++// from one thread at a time, and don't deinit() it while detect() runs. ++static void espdl_run(dl::Model *model) ++{ ++ MP_THREAD_GIL_EXIT(); ++ model->run(); ++ MP_THREAD_GIL_ENTER(); ++} ++ + static void espdl_free_model_data(espdl_model_data_t *model_data) + { + if (model_data->data == nullptr) { +@@ -627,7 +637,7 @@ + dl::image::img_t dl_img = espdl_make_dl_image(img); + + self->image_preprocessor->preprocess(dl_img); +- self->model->run(); ++ espdl_run(self->model); + + return espdl_tensor_map_info_to_dict(self->model->get_outputs(), true); + } +@@ -830,7 +840,7 @@ + dl::image::img_t dl_img = espdl_make_dl_image(img); + + self->image_preprocessor->preprocess(dl_img); +- self->model->run(); ++ espdl_run(self->model); + self->postprocessor->clear_result(); + self->postprocessor->postprocess(); + +@@ -979,7 +989,7 @@ + dl::image::img_t dl_img = espdl_make_dl_image(img); + + self->image_preprocessor->preprocess(dl_img); +- self->model->run(); ++ espdl_run(self->model); + self->postprocessor->clear_result(); + self->postprocessor->postprocess(); + +@@ -1139,7 +1149,7 @@ + dl::image::img_t dl_img = espdl_make_dl_image(img); + + self->image_preprocessor->preprocess(dl_img); +- self->model->run(); ++ espdl_run(self->model); + self->postprocessor->clear_result(); + self->postprocessor->postprocess(); + +@@ -1296,7 +1306,7 @@ + dl::image::img_t dl_img = espdl_make_dl_image(img); + + self->image_preprocessor->preprocess(dl_img); +- self->model->run(); ++ espdl_run(self->model); + self->postprocessor->clear_result(); + self->postprocessor->postprocess(); + +@@ -1447,7 +1457,7 @@ + dl::image::img_t dl_img = espdl_make_dl_image(img); + + self->image_preprocessor->preprocess(dl_img); +- self->model->run(); ++ espdl_run(self->model); + + std::vector &results = self->postprocessor->postprocess(); + mp_obj_t classifications = mp_obj_new_list(0, nullptr); diff --git a/modules/vision/esp-vision-patches/0002-espdl-HumanFaceDetect.patch b/modules/vision/esp-vision-patches/0002-espdl-HumanFaceDetect.patch new file mode 100644 index 00000000..1e90cd63 --- /dev/null +++ b/modules/vision/esp-vision-patches/0002-espdl-HumanFaceDetect.patch @@ -0,0 +1,19 @@ +--- a/modules/py_espdl.cpp ++++ b/modules/py_espdl.cpp +@@ -43,6 +43,8 @@ + extern const mp_obj_type_t espdl_yolo11n_pose_type; + extern const mp_obj_type_t espdl_imagenet_cls_type; + extern const mp_obj_type_t espdl_model_type; ++// The face detector with landmarks, in its own file beside this one. ++extern "C" const mp_obj_type_t espdl_human_face_detect_type; + + typedef struct { + uint8_t *data; +@@ -1538,6 +1540,7 @@ + {MP_ROM_QSTR(MP_QSTR_YOLO11), MP_ROM_PTR(&espdl_yolo11_type)}, + {MP_ROM_QSTR(MP_QSTR_YOLO11nPose), MP_ROM_PTR(&espdl_yolo11n_pose_type)}, + {MP_ROM_QSTR(MP_QSTR_ImageNetCls), MP_ROM_PTR(&espdl_imagenet_cls_type)}, ++ {MP_ROM_QSTR(MP_QSTR_HumanFaceDetect), MP_ROM_PTR(&espdl_human_face_detect_type)}, + {MP_ROM_QSTR(MP_QSTR_load_model), MP_ROM_PTR(&espdl_load_model_obj)}, + }; + static MP_DEFINE_CONST_DICT(espdl_module_globals, espdl_module_globals_table); diff --git a/modules/vision/examples/espdl_face_landmarks.py b/modules/vision/examples/espdl_face_landmarks.py new file mode 100644 index 00000000..ded75236 --- /dev/null +++ b/modules/vision/examples/espdl_face_landmarks.py @@ -0,0 +1,59 @@ +# SPDX-FileCopyrightText: 2026 Brad Barnett +# +# SPDX-License-Identifier: MIT +"""Find faces in a photo and mark their eyes, nose and mouth. + +Runs ESP-DL's two-stage face detector (HumanFaceDetect) on a photo. Each +face comes back as a box, a score and five key points: the eyes, the tip +of the nose and the corners of the mouth. The program prints them, draws +the boxes and points on the photo and saves it as a JPEG for you to look at. + +Needs: an ESP32-S3 or ESP32-P4 board with firmware built with vision, a +photo with a face in it, and the two models of ESP-DL's human_face_detect +(MIT licence; the files for your chip, from its models/p4 or models/s3 +folder), copied to the board: + + mpftp put face.jpg /face.jpg + mpftp put human_face_detect_msr_s8_v1.espdl /models/human_face_detect_msr_s8_v1.espdl + mpftp put human_face_detect_mnp_s8_v1.espdl /models/human_face_detect_mnp_s8_v1.espdl + +Run it: + + mpftp run espdl_face_landmarks.py + mpftp get /faces.jpg faces.jpg +""" + +import time + +import espdl +import image + +# Change these to match your setup. +PHOTO = "/face.jpg" +MSR_MODEL = "/models/human_face_detect_msr_s8_v1.espdl" +MNP_MODEL = "/models/human_face_detect_mnp_s8_v1.espdl" +FILENAME = "/faces.jpg" + +# Load the photo as RGB565, which the detector takes, and both models. +img = image.Image(PHOTO).to_rgb565(copy=True) +detector = espdl.HumanFaceDetect(MSR_MODEL, MNP_MODEL) + +# Detect, timing the call. +start = time.ticks_ms() +faces = detector.detect(img) +print("%d faces in %d ms" % (len(faces), time.ticks_diff(time.ticks_ms(), start))) + +# Print each face, and draw its box in green and its five points in red. +for x, y, w, h, score, points in faces: + print("face at (%d, %d), %dx%d, score %.2f" % (x, y, w, h, score)) + print(" points:", [(points[i], points[i + 1]) for i in range(0, len(points), 2)]) + img.draw_rectangle(x, y, w, h, color=(0, 255, 0), thickness=2) + for i in range(0, len(points), 2): + img.draw_circle(points[i], points[i + 1], 3, color=(255, 0, 0), fill=True) + +# Save the marked photo as a JPEG. +jpeg = img.to_jpeg(quality=85, copy=True) +with open(FILENAME, "wb") as f: + f.write(jpeg.bytearray()) +print("Saved", FILENAME) +detector.deinit() diff --git a/modules/vision/examples/espdl_thread.py b/modules/vision/examples/espdl_thread.py new file mode 100644 index 00000000..7f64b81a --- /dev/null +++ b/modules/vision/examples/espdl_thread.py @@ -0,0 +1,70 @@ +# SPDX-FileCopyrightText: 2026 Brad Barnett +# +# SPDX-License-Identifier: MIT +"""Run ESP-DL detection in a thread while the main loop keeps going. + +Starts a thread that runs the pedestrian detector on a photo ten times, +while the main thread does nothing but count its own loop and note the +longest pause between two turns. espdl lets go of the interpreter while a +model runs, so the main loop keeps turning during each detection: the +longest pause comes out well under the time one detection takes. A +display, a UI or a recording loop on the main thread gets the same. + +Needs: an ESP32-S3 or ESP32-P4 board with firmware built with vision, and +the photo and model espdl_pedestrian_test.py uses, copied to the board: + + mpftp put pedestrian.jpg /pedestrian.jpg + mpftp put pedestrian_detect_pico.espdl /pedestrian.espdl + +Run it: + + mpftp run espdl_thread.py +""" + +import _thread +import time + +import espdl +import image + +# Change these to match your setup. +PHOTO = "/pedestrian.jpg" +MODEL = "/pedestrian.espdl" +RUNS = 10 +THREAD_STACK = 32 * 1024 # bytes; the model runs on the thread's own stack + +# Load the photo as RGB565, which the detector takes, and the model. +img = image.Image(PHOTO).to_rgb565(copy=True) +detector = espdl.PedestrianDetect(MODEL) +results = [] + + +def detect_loop(): + # Detect RUNS times, then hand back the last count and the total time. + start = time.ticks_ms() + found = [] + for _ in range(RUNS): + found = detector.detect(img) + results.append((len(found), time.ticks_diff(time.ticks_ms(), start))) + + +# Start the detector on its own thread. +_thread.stack_size(THREAD_STACK) +_thread.start_new_thread(detect_loop, ()) + +# Count turns of the main loop until the thread is done, and keep the +# longest gap between two turns. +turns = 0 +longest_us = 0 +last = time.ticks_us() +while not results: + now = time.ticks_us() + longest_us = max(longest_us, time.ticks_diff(now, last)) + last = now + turns += 1 + +# Report the main loop's turns beside what the detector did meanwhile. +count, total_ms = results[0] +print("%d detections, %d ms each, %d people in the photo" % (RUNS, total_ms // RUNS, count)) +print("main loop: %d turns meanwhile, longest pause %.1f ms" % (turns, longest_us / 1000)) +detector.deinit() diff --git a/modules/vision/micropython.cmake b/modules/vision/micropython.cmake index 2f4f298e..dc28a0dc 100644 --- a/modules/vision/micropython.cmake +++ b/modules/vision/micropython.cmake @@ -2,8 +2,10 @@ # SPIKE. ESP32-P4 and ESP32-S3 only; every other port and chip skips it. # Make ports never look here: there is no micropython.mk. # -# Ours: components/vision_deps (the ESP-IDF components it needs) and -# src/vision_preview_none.c (preview without esp-vision's USB multiplexer). +# Ours: components/vision_deps (the ESP-IDF components it needs), +# src/vision_preview_none.c (preview without esp-vision's USB multiplexer), +# src/vision_espdl_face.cpp (espdl.HumanFaceDetect) and esp-vision-patches/ (two small +# changes to esp-vision's py_espdl.cpp, applied to a copy in the build tree). # Everything else is esp-vision's file as it is, imlib (OpenMV's, MIT) # included: compiled here, in MicroPython's own target, because it includes # MicroPython's generated headers, and esp-vision's components/imlib @@ -61,9 +63,32 @@ if(ESP_PLATFORM AND (IDF_TARGET STREQUAL "esp32p4" OR IDF_TARGET STREQUAL "esp32 list(APPEND _libs idf::espressif__esp_h264) endif() - # espdl: ESP-DL inference, with the model classes esp-vision ships. + # espdl: ESP-DL inference, with the model classes esp-vision ships. Its + # file gets our patches (esp-vision-patches/, in order) as a copy in the build + # tree, so the pinned checkout, which other trees may share, stays as it + # is: inference without the GIL, and HumanFaceDetect from + # src/vision_espdl_face.cpp in the module's table. idf_component_get_property(_dl espressif__esp-dl COMPONENT_DIR) - list(APPEND _src ${EV}/modules/py_espdl.cpp) + set(_espdl_dir ${CMAKE_BINARY_DIR}/vision) + set(_espdl ${_espdl_dir}/py_espdl.cpp) + file(MAKE_DIRECTORY ${_espdl_dir}) + file(GLOB _espdl_patches ${CMAKE_CURRENT_LIST_DIR}/esp-vision-patches/*.patch) + list(SORT _espdl_patches) + configure_file(${EV}/modules/py_espdl.cpp ${_espdl}.new COPYONLY) + foreach(_p ${_espdl_patches}) + execute_process( + COMMAND patch --quiet --forward --no-backup-if-mismatch --reject-file=- ${_espdl}.new ${_p} + RESULT_VARIABLE _rc) + if(NOT _rc EQUAL 0) + message(FATAL_ERROR "vision: ${_p} does not apply to esp-vision's py_espdl.cpp") + endif() + endforeach() + # Only a changed result touches the copy, so a reconfigure doesn't + # recompile it. + execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different ${_espdl}.new ${_espdl}) + file(REMOVE ${_espdl}.new) + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${_espdl_patches}) + list(APPEND _src ${_espdl} ${CMAKE_CURRENT_LIST_DIR}/src/vision_espdl_face.cpp) foreach(_d dl dl/tool/include dl/tensor/include dl/base dl/base/isa dl/math/include dl/model/include dl/module/include fbs_loader/include vision/detect vision/image vision/image/isa vision/recognition vision/classification audio/common diff --git a/modules/vision/src/vision_espdl_face.cpp b/modules/vision/src/vision_espdl_face.cpp new file mode 100644 index 00000000..be9ba70e --- /dev/null +++ b/modules/vision/src/vision_espdl_face.cpp @@ -0,0 +1,335 @@ +// SPDX-License-Identifier: MIT +// espdl.HumanFaceDetect: faces with five landmarks, from ESP-DL's two-stage +// human_face_detect models (MSR finds candidates, MNP refines each one and +// places the eyes, nose and mouth corners). +// +// det = espdl.HumanFaceDetect("/models/human_face_detect_msr_s8_v1.espdl", +// "/models/human_face_detect_mnp_s8_v1.espdl") +// for x, y, w, h, score, keypoints in det.detect(img): # keypoints: 10 ints +// +// The stages follow ESP-DL's models/human_face_detect (MIT, 0.5.0, the release +// for the esp-dl this firmware pins): the same anchors, thresholds, BGR input +// and square crops for MNP. Both models are read from the filesystem, as the +// other espdl classes read theirs. The whole detection runs without the GIL. + +#include +#include +#include +#include + +// esp-dl's headers use FreeRTOS without including it. +#include "freertos/FreeRTOS.h" +#include "freertos/idf_additions.h" +#include "freertos/semphr.h" +#include "freertos/task.h" + +#include "dl_detect_define.hpp" +#include "dl_detect_mnp_postprocessor.hpp" +#include "dl_detect_msr_postprocessor.hpp" +#include "dl_image_preprocessor.hpp" +#include "dl_model_base.hpp" +#include "esp_heap_caps.h" +#include "fbs_model.hpp" + +extern "C" { +#include "extmod/vfs.h" +#include "py/mperrno.h" +#include "py/obj.h" +#include "py/runtime.h" +#include "py/stream.h" +#include "py_image.h" +} + +extern "C" const mp_obj_type_t espdl_human_face_detect_type; + +namespace { + +// One stage: a model loaded from a file, its preprocessor and postprocessor. +struct face_stage_t { + uint8_t *data; + dl::Model *model; + dl::image::ImagePreprocessor *pre; + dl::detect::DetectPostprocessor *post; +}; + +typedef struct { + mp_obj_base_t base; + face_stage_t msr, mnp; +} face_obj_t; + +void face_stage_free(face_stage_t *st) { + delete st->post; + st->post = nullptr; + delete st->pre; + st->pre = nullptr; + delete st->model; + st->model = nullptr; + if (st->data != nullptr) { + heap_caps_free(st->data); + st->data = nullptr; + } +} + +void face_free(face_obj_t *self) { + face_stage_free(&self->mnp); + face_stage_free(&self->msr); +} + +// The model file in one 16-byte-aligned block (PSRAM first), as esp-dl wants +// a model it reads in place. +uint8_t *face_read_file(mp_obj_t path) { + mp_obj_t args[2] = { path, MP_OBJ_NEW_QSTR(MP_QSTR_rb) }; + mp_obj_t file = mp_vfs_open(2, args, (mp_map_t *)&mp_const_empty_map); + int err = 0; + mp_off_t size = mp_stream_seek(file, 0, MP_SEEK_END, &err); + if (size > 0) { + mp_stream_seek(file, 0, MP_SEEK_SET, &err); + } + if (size <= 0 || err != 0) { + mp_stream_close(file); + mp_raise_msg(&mp_type_ValueError, MP_ERROR_TEXT("HumanFaceDetect: empty or unreadable model file")); + } + size_t aligned = ((size_t)size + 15u) & ~15u; + uint8_t *data = (uint8_t *)heap_caps_aligned_alloc(16, aligned, MALLOC_CAP_SPIRAM | MALLOC_CAP_8BIT); + if (data == nullptr) { + data = (uint8_t *)heap_caps_aligned_alloc(16, aligned, MALLOC_CAP_DEFAULT); + } + if (data == nullptr) { + mp_stream_close(file); + mp_raise_msg(&mp_type_MemoryError, MP_ERROR_TEXT("HumanFaceDetect: no memory for the model")); + } + mp_uint_t got = mp_stream_read_exactly(file, data, (mp_uint_t)size, &err); + mp_stream_close(file); + if (err != 0 || got != (mp_uint_t)size) { + heap_caps_free(data); + mp_raise_msg(&mp_type_OSError, MP_ERROR_TEXT("HumanFaceDetect: short read of the model file")); + } + return data; +} + +// Loads one stage's model; NULL on success, else why not. The postprocessor +// is made by the caller, since the two stages differ there. +const char *face_stage_load(face_stage_t *st) { + st->model = new (std::nothrow) dl::Model((const char *)st->data, fbs::MODEL_LOCATION_IN_FLASH_RODATA); + if (st->model == nullptr) { + return "no memory for the model"; + } + if (st->model->get_fbs_model() == nullptr) { + return "the model file did not load (is it an .espdl model for this chip?)"; + } + st->model->minimize(); + // The models take BGR, as ESP-DL's own HumanFaceDetect feeds them. + st->pre = new (std::nothrow) dl::image::ImagePreprocessor(st->model, + std::array {0.0f, 0.0f, 0.0f}, std::array {1.0f, 1.0f, 1.0f}, true); + return st->pre == nullptr ? "no memory for the preprocessor" : nullptr; +} + +image_t *face_get_image(mp_obj_t img_in, dl::image::img_t *out) { + if (!mp_obj_is_type(img_in, &py_image_type)) { + mp_raise_TypeError(MP_ERROR_TEXT("expected image.Image")); + } + image_t *img = (image_t *)py_image_cobj(img_in); + if (img == nullptr || img->pixels == nullptr || img->w <= 0 || img->h <= 0) { + mp_raise_msg(&mp_type_ValueError, MP_ERROR_TEXT("invalid image")); + } + if (img->pixfmt != PIXFORMAT_RGB565) { + mp_raise_msg(&mp_type_ValueError, MP_ERROR_TEXT("HumanFaceDetect takes an RGB565 image")); + } + out->data = img->pixels; + out->width = (uint16_t)img->w; + out->height = (uint16_t)img->h; + out->pix_type = dl::image::DL_IMAGE_PIX_TYPE_RGB565LE; + return img; +} + +face_obj_t *face_get(mp_obj_t self_in) { + face_obj_t *self = (face_obj_t *)MP_OBJ_TO_PTR(self_in); + if (self->msr.post == nullptr || self->mnp.post == nullptr) { + mp_raise_msg(&mp_type_OSError, MP_ERROR_TEXT("HumanFaceDetect object is deinitialized")); + } + return self; +} + +// Both stages, as ESP-DL's MSRMNP::run does it: MSR over the whole picture, +// then MNP on a square around each candidate, then one NMS over MNP's faces. +std::list &face_run(face_obj_t *self, const dl::image::img_t &img) { + self->msr.pre->preprocess(img); + self->msr.model->run(); + self->msr.post->clear_result(); + self->msr.post->postprocess(); + std::list &candidates = self->msr.post->get_result(img.width, img.height); + + self->mnp.post->clear_result(); + for (dl::detect::result_t &c : candidates) { + int cx = (c.box[0] + c.box[2]) >> 1; + int cy = (c.box[1] + c.box[3]) >> 1; + int side = DL_MAX(c.box[2] - c.box[0], c.box[3] - c.box[1]); + c.box[0] = cx - (side >> 1); + c.box[1] = cy - (side >> 1); + c.box[2] = c.box[0] + side; + c.box[3] = c.box[1] + side; + c.limit_box(img.width, img.height); + self->mnp.pre->preprocess(img, c.box); + self->mnp.model->run(); + self->mnp.post->postprocess(); + } + self->mnp.post->nms(); + return self->mnp.post->get_result(img.width, img.height); +} + +float face_float_arg(mp_obj_t obj, float dflt) { + return obj == MP_OBJ_NULL ? dflt : mp_obj_get_float_to_f(obj); +} + +} // namespace + +// HumanFaceDetect(msr_path, mnp_path, *, score=0.5, nms=0.5, mnp_score=0.5, mnp_nms=0.5) +static mp_obj_t face_make_new(const mp_obj_type_t *type, size_t n_args, size_t n_kw, const mp_obj_t *all_args) { + enum { ARG_msr, ARG_mnp, ARG_score, ARG_nms, ARG_mnp_score, ARG_mnp_nms }; + static const mp_arg_t allowed[] = { + { MP_QSTR_msr_path, MP_ARG_REQUIRED | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} }, + { MP_QSTR_mnp_path, MP_ARG_REQUIRED | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} }, + { MP_QSTR_score, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} }, + { MP_QSTR_nms, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} }, + { MP_QSTR_mnp_score, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} }, + { MP_QSTR_mnp_nms, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} }, + }; + mp_arg_val_t args[MP_ARRAY_SIZE(allowed)]; + mp_arg_parse_all_kw_array(n_args, n_kw, all_args, MP_ARRAY_SIZE(allowed), allowed, args); + float score = face_float_arg(args[ARG_score].u_obj, 0.5f); + float nms = face_float_arg(args[ARG_nms].u_obj, 0.5f); + float mnp_score = face_float_arg(args[ARG_mnp_score].u_obj, 0.5f); + float mnp_nms = face_float_arg(args[ARG_mnp_nms].u_obj, 0.5f); + + face_obj_t *self = mp_obj_malloc_with_finaliser(face_obj_t, type); + self->msr = face_stage_t{nullptr, nullptr, nullptr, nullptr}; + self->mnp = face_stage_t{nullptr, nullptr, nullptr, nullptr}; + + self->msr.data = face_read_file(args[ARG_msr].u_obj); + nlr_buf_t nlr; + if (nlr_push(&nlr) == 0) { + self->mnp.data = face_read_file(args[ARG_mnp].u_obj); + nlr_pop(); + } else { + face_free(self); + nlr_jump(nlr.ret_val); + } + + const char *why = face_stage_load(&self->msr); + if (why == nullptr) { + // ESP-DL's anchors for the MSR stage: two strides, two box sizes each. + self->msr.post = new (std::nothrow) dl::detect::MSRPostprocessor(self->msr.model, self->msr.pre, + score, nms, 10, {{8, 8, 9, 9, {{16, 16}, {32, 32}}}, {16, 16, 9, 9, {{64, 64}, {128, 128}}}}); + if (self->msr.post == nullptr) { + why = "no memory for the postprocessor"; + } + } + if (why == nullptr) { + why = face_stage_load(&self->mnp); + } + if (why == nullptr) { + // MNP sees one 48x48 crop per candidate. + self->mnp.post = new (std::nothrow) dl::detect::MNPPostprocessor(self->mnp.model, self->mnp.pre, + mnp_score, mnp_nms, 10, {{1, 1, 0, 0, {{48, 48}}}}); + if (self->mnp.post == nullptr) { + why = "no memory for the postprocessor"; + } + } + if (why != nullptr) { + face_free(self); + mp_raise_msg_varg(&mp_type_OSError, MP_ERROR_TEXT("HumanFaceDetect: %s"), why); + } + return MP_OBJ_FROM_PTR(self); +} + +// detect(img) -> [(x, y, w, h, score, keypoints)], keypoints a tuple of ten +// ints, five x, y pairs in ESP-DL's order: left eye, left mouth corner, nose, +// right eye, right mouth corner. +static mp_obj_t face_detect(mp_obj_t self_in, mp_obj_t img_in) { + face_obj_t *self = face_get(self_in); + dl::image::img_t img; + face_get_image(img_in, &img); + // img_in stays referenced from this frame, so its pixels outlive the run. + MP_THREAD_GIL_EXIT(); + std::list &faces = face_run(self, img); + MP_THREAD_GIL_ENTER(); + + mp_obj_t out = mp_obj_new_list(0, nullptr); + for (const dl::detect::result_t &r : faces) { + if (r.box.size() < 4) { + continue; + } + int w = r.box[2] - r.box[0] + 1; + int h = r.box[3] - r.box[1] + 1; + if (w <= 0 || h <= 0) { + continue; + } + mp_obj_t kp[10]; + size_t nkp = r.keypoint.size() < 10 ? r.keypoint.size() : 10; + for (size_t i = 0; i < nkp; i++) { + kp[i] = mp_obj_new_int(r.keypoint[i]); + } + mp_obj_t item[6] = { + mp_obj_new_int(r.box[0]), + mp_obj_new_int(r.box[1]), + mp_obj_new_int(w), + mp_obj_new_int(h), + mp_obj_new_float(r.score), + mp_obj_new_tuple(nkp, kp), + }; + mp_obj_list_append(out, mp_obj_new_tuple(6, item)); + } + return out; +} +static MP_DEFINE_CONST_FUN_OBJ_2(face_detect_obj, face_detect); + +// set_thresholds(*, score=, nms=, mnp_score=, mnp_nms=) -> self +static mp_obj_t face_set_thresholds(size_t n_args, const mp_obj_t *pos_args, mp_map_t *kw_args) { + enum { ARG_self, ARG_score, ARG_nms, ARG_mnp_score, ARG_mnp_nms }; + static const mp_arg_t allowed[] = { + { MP_QSTR_self, MP_ARG_REQUIRED | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} }, + { MP_QSTR_score, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} }, + { MP_QSTR_nms, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} }, + { MP_QSTR_mnp_score, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} }, + { MP_QSTR_mnp_nms, MP_ARG_KW_ONLY | MP_ARG_OBJ, {.u_obj = MP_OBJ_NULL} }, + }; + mp_arg_val_t args[MP_ARRAY_SIZE(allowed)]; + mp_arg_parse_all(n_args, pos_args, kw_args, MP_ARRAY_SIZE(allowed), allowed, args); + face_obj_t *self = face_get(args[ARG_self].u_obj); + if (args[ARG_score].u_obj != MP_OBJ_NULL) { + self->msr.post->set_score_thr(mp_obj_get_float_to_f(args[ARG_score].u_obj)); + } + if (args[ARG_nms].u_obj != MP_OBJ_NULL) { + self->msr.post->set_nms_thr(mp_obj_get_float_to_f(args[ARG_nms].u_obj)); + } + if (args[ARG_mnp_score].u_obj != MP_OBJ_NULL) { + self->mnp.post->set_score_thr(mp_obj_get_float_to_f(args[ARG_mnp_score].u_obj)); + } + if (args[ARG_mnp_nms].u_obj != MP_OBJ_NULL) { + self->mnp.post->set_nms_thr(mp_obj_get_float_to_f(args[ARG_mnp_nms].u_obj)); + } + return args[ARG_self].u_obj; +} +static MP_DEFINE_CONST_FUN_OBJ_KW(face_set_thresholds_obj, 1, face_set_thresholds); + +static mp_obj_t face_deinit(mp_obj_t self_in) { + face_free((face_obj_t *)MP_OBJ_TO_PTR(self_in)); + return mp_const_none; +} +static MP_DEFINE_CONST_FUN_OBJ_1(face_deinit_obj, face_deinit); + +static const mp_rom_map_elem_t face_locals_table[] = { + { MP_ROM_QSTR(MP_QSTR___del__), MP_ROM_PTR(&face_deinit_obj) }, + { MP_ROM_QSTR(MP_QSTR_deinit), MP_ROM_PTR(&face_deinit_obj) }, + { MP_ROM_QSTR(MP_QSTR_detect), MP_ROM_PTR(&face_detect_obj) }, + { MP_ROM_QSTR(MP_QSTR_set_thresholds), MP_ROM_PTR(&face_set_thresholds_obj) }, +}; +static MP_DEFINE_CONST_DICT(face_locals_dict, face_locals_table); + +MP_DEFINE_CONST_OBJ_TYPE( + espdl_human_face_detect_type, + MP_QSTR_HumanFaceDetect, + MP_TYPE_FLAG_NONE, + make_new, reinterpret_cast(face_make_new), + locals_dict, &face_locals_dict + ); diff --git a/variants/esp32/ESP32_GENERIC_P4/P4X_EYE/partitions.csv b/variants/esp32/ESP32_GENERIC_P4/P4X_EYE/partitions.csv index 0c853b15..1ffd01cd 100644 --- a/variants/esp32/ESP32_GENERIC_P4/P4X_EYE/partitions.csv +++ b/variants/esp32/ESP32_GENERIC_P4/P4X_EYE/partitions.csv @@ -1,8 +1,8 @@ -# Espressif ESP32-P4X-EYE, 16 MB flash. A 7 MB app slot: the whole stack with -# vision is about 6 MB, and the camera app is frozen in. The rest is the -# filesystem; the SD card holds the photos and videos. +# Espressif ESP32-P4X-EYE, 16 MB flash. A 9 MB app slot: the whole stack with +# vision and the frozen apps is 8.5 MB. The rest is the filesystem; the SD +# card holds the photos and videos. # Name, Type, SubType, Offset, Size, Flags nvs, data, nvs, 0x9000, 0x6000, phy_init, data, phy, 0xf000, 0x1000, -factory, app, factory, 0x10000, 0x700000, -vfs, data, fat, 0x710000, 0x8F0000, +factory, app, factory, 0x10000, 0x900000, +vfs, data, fat, 0x910000, 0x6F0000,