diff --git a/.github/workflows/release-firmware.yml b/.github/workflows/release-firmware.yml index ec398f8..a15e823 100644 --- a/.github/workflows/release-firmware.yml +++ b/.github/workflows/release-firmware.yml @@ -289,9 +289,19 @@ jobs: name: firmware path: firmware + # The notes name every board whose filesystem this set moves, against the + # newest set people could download (a published release, pre-releases too). - name: The tag may go on a draft, and the Release's notes run: | - python3 scripts/release_firmware.py notes firmware/firmware.json --out "$RUNNER_TEMP/notes.md" + prev=$(gh api "repos/$GITHUB_REPOSITORY/releases?per_page=100" \ + --jq '[.[] | select(.draft | not) | select(any(.assets[]; .name == "firmware.json"))][0].tag_name // empty') + previous=() + if [[ -n "$prev" ]]; then + gh release download "$prev" -p firmware.json -D "$RUNNER_TEMP/previous" + previous=(--previous "$RUNNER_TEMP/previous/firmware.json") + echo "comparing with $prev" + fi + python3 scripts/release_firmware.py notes firmware/firmware.json "${previous[@]}" --out "$RUNNER_TEMP/notes.md" echo "TAG=$(python3 -c 'import json; print(json.load(open("firmware/firmware.json"))["tag"])')" >> "$GITHUB_ENV" echo "PRERELEASE=$(python3 -c 'import json; print("--prerelease" if json.load(open("firmware/firmware.json"))["preview"] else "")')" >> "$GITHUB_ENV" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 25d50fe..181ae33 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -93,6 +93,20 @@ inside the flash image, UF2 blocks, the vector table of a GIGA's main image) and every file's sha256 and size, the flash offset, and the pins the set was built from. +### Each board keeps its filesystem + +Every MicroPython esp32 and rp2 cell names the filesystem its boards keep from +one release to the next (`fs`), and builds with `--keep-fs` at that size. An +image that outgrows the flash left beside it fails its build, which says what +size would fit, instead of shipping a set that leaves an updated board with an +empty filesystem. `assemble` checks each filesystem is where its cell says and +records it in `firmware.json`. + +When an image no longer fits, build fewer modules into it, or change the cell's +`fs` on purpose. That moves the filesystem on that board, and the draft's notes +say so: they compare the set with the newest published one and name each board +whose filesystem moved. + ### Build a set Actions > Release firmware > Run workflow. A pull request that touches the diff --git a/build_mp.py b/build_mp.py index c8b1291..c57704b 100755 --- a/build_mp.py +++ b/build_mp.py @@ -1520,6 +1520,102 @@ def fs_size(text): return n if n and n % 4096 == 0 else None +def keep_fs_refused(keep_fs, fits): + """What a build with --keep-fs says when the firmware doesn't fit beside it.""" + say(f"\nThe firmware doesn't fit beside a {keep_fs // 1024} KB filesystem." + + (f" A {fits // 1024} KB one would" if fits else "")) + say((" fit it, but that" if fits else " A smaller one") + " moves the filesystem and a board flashed with it comes up") + say(" with an empty one. (--keep-fs, so nothing was rebuilt: build fewer modules,") + say(" or copy the board's files off and build without --keep-fs.)") + + +# A filesystem partition, by its subtype. MicroPython mounts the one named vfs +# (or ffat); with neither, it makes one of the flash after the last partition. +ESP32_FS_SUBTYPES = ("fat", "littlefs", "spiffs") + + +def esp32_table_rows(src): + rows = [] + for raw in Path(src).read_text().splitlines(): + line = raw.strip() + if line and not line.startswith("#"): + fields = [f.strip() for f in line.split(",")] + rows.append((fields + [""] * 6)[:6]) + return rows + + +def esp32_bytes(v): + v = v.strip().lower() + mult = 1024 if v.endswith("k") else 1024 * 1024 if v.endswith("m") else 1 + return int(v.rstrip("km"), 0) * mult + + +def esp32_used_table(build, port_dir): + """The partition table a configured build in `build` used, or None.""" + try: + sdk = (build / "sdkconfig").read_text() + except OSError: + return None + used = re.search(r'^CONFIG_PARTITION_TABLE_CUSTOM_FILENAME="(.*)"$', sdk, re.M) + if not used: + return None + src = Path(used.group(1)) + return src if src.is_absolute() else port_dir / src + + +def esp32_keep_fs_layout(rows, flash_bytes, fs_bytes): + """(app row, named filesystem row or None, app size) for a table whose + filesystem keeps fs_bytes at the end of the flash. The app partition takes + what the partitions after it leave.""" + apps = [i for i, r in enumerate(rows) if r[1] == "app"] + if len(apps) != 1: + die(f"--keep-fs sizes a table with one app partition; this one has {len(apps)}") + idx = apps[0] + fs = [i for i, r in enumerate(rows) if r[1] == "data" and r[2] in ESP32_FS_SUBTYPES] + if fs and (fs != [len(rows) - 1] or fs[0] < idx): + die("--keep-fs needs the filesystem partition, if the table names one, last and after the app") + named = fs[0] if fs else None + between = sum(esp32_bytes(r[4]) for i, r in enumerate(rows) if i > idx and i != named) + return idx, named, flash_bytes - fs_bytes - esp32_bytes(rows[idx][3]) - between + + +def esp32_keep_fs(src, flash_bytes, fs_bytes, dest): + """The table at src with its app partition sized so the filesystem is the + last fs_bytes of the flash, where a board flashed earlier with the same + size has it.""" + rows = esp32_table_rows(src) + idx, named, size = esp32_keep_fs_layout(rows, flash_bytes, fs_bytes) + if size < 0x10000: + die(f"--keep-fs: a {fs_bytes // 1024} KB filesystem leaves no room for the app in {flash_bytes // 1024 // 1024} MB") + rows[idx][4] = hex(size) + cursor = esp32_bytes(rows[idx][3]) + size + for i in range(idx + 1, len(rows)): + rows[i][3] = hex(cursor) + if i != named: + cursor += esp32_bytes(rows[i][4]) + if named is not None: + rows[named][4] = hex(fs_bytes) + where = "" if named is not None else " (the flash after the last partition)" + body = ["# Name, Type, SubType, Offset, Size, Flags", + f"# Generated by build_mp.py from {src} for --keep-fs: '{rows[idx][0]}' sized to {hex(size)}", + f"# so the filesystem{where} is {fs_bytes // 1024} KB at {hex(flash_bytes - fs_bytes)}."] + body += [", ".join(r).rstrip(", ") for r in rows] + dest.write_text("\n".join(body) + "\n") + return dest + + +def esp32_keep_fs_fits(log_text, table, flash_bytes): + """The largest filesystem, in whole 64 KB, that leaves room for an app the + size the log says it is: 0 if none does, None if the log isn't an app + partition overflow.""" + m = re.search(r"app partition is too small for binary \S+ size (0x[0-9a-fA-F]+)", log_text) + if not m: + return None + _, _, room = esp32_keep_fs_layout(esp32_table_rows(table), flash_bytes, 0) + room -= (int(m.group(1), 16) + 0xFFFF) & ~0xFFFF + return room & ~0xFFFF if room >= 0x10000 else 0 + + def rp2_autosize(log_text): """MICROPY_HW_FLASH_STORAGE_BYTES that leaves the firmware room to fit, from the linker's memory table; None if the log isn't a FLASH overflow.""" @@ -2101,7 +2197,7 @@ def main(): ap.add_argument("--flash", help="esp32 flash size, e.g. 16MB") ap.add_argument("--no-autosize", action="store_true", help="refuse instead of growing the firmware's flash: esp32's app partition, rp2's firmware over its " "filesystem, the Arduino GIGA's over its M4 core's flash") - ap.add_argument("--keep-fs", metavar="SIZE", help="rp2: build beside the filesystem a board already has, its size in bytes or KB " + ap.add_argument("--keep-fs", metavar="SIZE", help="rp2 and esp32: build beside the filesystem a board already has, its size in bytes or KB " "(576K), and refuse instead of moving it when the firmware doesn't fit") ap.add_argument("--coredump", action="store_true", help="esp32: a troubleshooting build that writes a core dump to flash on a crash (standard builds don't)") @@ -2170,14 +2266,16 @@ def main(): die("--flash is for esp32") if args.coredump and port != "esp32": die("--coredump is for esp32") - if args.keep_fs and port != "rp2": - die("--keep-fs is for rp2") + if args.keep_fs and port not in ("rp2", "esp32"): + die("--keep-fs is for rp2 and esp32") keep_fs = fs_size(args.keep_fs) if args.keep_fs else None if args.keep_fs and not keep_fs: die(f"--keep-fs takes a size in whole 4 KB sectors, in bytes or KB (576K), not '{args.keep_fs}'") if args.flash and args.flash.upper() not in FLASH_SIZES: die(f"--flash takes one of {', '.join(FLASH_SIZES)}") flash = args.flash.upper() if args.flash else None + if keep_fs and port == "esp32" and not flash: + die("--keep-fs on esp32 needs --flash: the filesystem is the end of the flash, so its place depends on the flash size") if port == "esp32" and flash is None and not args.port: # Upstream's generic tables assume 4 MB, which most module sets outgrow. flash = choose("Flash size (none keeps the board's own):", list(FLASH_SIZES), allow_none=True) @@ -2310,6 +2408,10 @@ def shell(cmd, log=None): die("--coredump: this board's table has no coredump partition yet, and the dump needs one. " "The ESP32-S3 boards' table has it.") tinyusb = tinyusb_component(mp, chip) + flash_bytes = int(flash[:-2]) * 1024 * 1024 if flash else 0 + kept = build / "partitions-keep-fs.csv" + if keep_fs and table: + table = esp32_keep_fs(table, flash_bytes, keep_fs, kept) for attempt in (1, 2): text, used = esp32_fragment(chip, module_dirs, board, variant, flash, table, args.coredump) frag.write_text(text) @@ -2324,6 +2426,22 @@ def shell(cmd, log=None): say(f" fragment: {u}") shell(make_esp + ["submodules"]) rc, out = shell(make_esp, log=True) + if keep_fs and table != kept: + # The board's own table, found only once a build has + # configured: size it, and build against that. + src = esp32_used_table(build, port_dir) + if not src and rc: + break + if not src: + die("--keep-fs: this build didn't say which partition table it used, so there is none to size") + table = esp32_keep_fs(src, flash_bytes, keep_fs, kept) + say(f"\n--keep-fs: building once more against {table}, sized from the board's {src.name}.\n") + continue + if keep_fs: + fits = esp32_keep_fs_fits(out, table, flash_bytes) if rc else None + if fits is not None: + keep_fs_refused(keep_fs, fits) + break if rc == 0 or attempt == 2: break grown = esp32_autosize(out, build, port_dir, build / "partitions.csv") @@ -2377,10 +2495,7 @@ def shell(cmd, log=None): if not storage: break if keep_fs: - say(f"\nThe firmware doesn't fit beside a {keep_fs // 1024} KB filesystem. A {storage // 1024} KB one would") - say(" fit it, but that moves the filesystem and a board flashed with it comes up") - say(" with an empty one. (--keep-fs, so nothing was rebuilt: build fewer modules,") - say(" or copy the board's files off and build without --keep-fs.)") + keep_fs_refused(keep_fs, storage) break if args.no_autosize: say(f"\nThe firmware doesn't fit. MICROPY_HW_FLASH_STORAGE_BYTES={storage} would fit it.") diff --git a/docs/guides/add-a-board-variant.md b/docs/guides/add-a-board-variant.md index 4d2950a..a6ff612 100644 --- a/docs/guides/add-a-board-variant.md +++ b/docs/guides/add-a-board-variant.md @@ -40,7 +40,8 @@ The partition table is found the same way, the most specific `partitions.csv` winning ([`variants/esp32/partitions.esp32s3.csv`](https://github.com/PyDevices/micropython-pydevices/blob/main/variants/esp32/partitions.esp32s3.csv) adds a coredump partition on the S3, which a standard build leaves unused). Without one, the board's own is used. Either way, autosize grows the app -partition when the image doesn't fit. +partition when the image doesn't fit, unless the build keeps a board's +filesystem with `--keep-fs`. Run the CPU, flash and PSRAM at the fastest clocks the board's chips are rated for. The CPU's rating is the chip's: 240 MHz for an ESP32, S2, S3 or C5, diff --git a/docs/guides/fit-firmware-on-a-small-flash.md b/docs/guides/fit-firmware-on-a-small-flash.md index bc4d442..672c3ae 100644 --- a/docs/guides/fit-firmware-on-a-small-flash.md +++ b/docs/guides/fit-firmware-on-a-small-flash.md @@ -16,10 +16,10 @@ To refuse the growth and stop instead, add `--no-autosize`. ## Keep a board's files through an update -On rp2, autosize works the filesystem's size out again in each new build -folder, so a newer image that is a few KB bigger can come out with a smaller -filesystem than the one already on your board, and flashing it empties the -board. To keep the board's filesystem where it is, ask the board its size: +Autosize works the firmware's share out again in each new build folder, so a +newer image that is a few KB bigger can come out with a smaller filesystem +than the one already on your board, and flashing it empties the board. To keep +the board's filesystem where it is, ask the board its size. On rp2: ```python import rp2 @@ -27,17 +27,31 @@ f = rp2.Flash() print(f.ioctl(4, 0) * f.ioctl(5, 0)) # 589824 is 576 KB ``` +On esp32: + +```python +import esp32 +print(esp32.Partition.find(esp32.Partition.TYPE_DATA, label="vfs")[0].info()[3]) # 2752512 is 2688 KB +``` + and build with that size: ```bash ./build_mp.py --port rp2 --board RPI_PICO --modules displayif,lvgl,pydevices --keep-fs 576K +./build_mp.py --port esp32 --board ESP32_GENERIC_S3 --variant LCD_7 --flash 8MB --modules all --keep-fs 2688K ``` If the firmware fits beside it, the image keeps that filesystem and your files are there after the flash. If it doesn't, the build stops and says what size -would fit, rather than moving it. A rebuild in the same build folder starts -from the size that folder used last, so it keeps the filesystem without -`--keep-fs` unless the image has outgrown it. +would fit, rather than moving it. On esp32 the filesystem is the end of the +flash, so `--keep-fs` needs `--flash`, and the app partition takes the flash +before it. On rp2, a rebuild in the same build folder starts from the size that +folder used last, so it keeps the filesystem without `--keep-fs` unless the +image has outgrown it. + +The firmware page's MicroPython images are built this way, so updating a board +from one release to the next keeps its files. A release that has to move a +board's filesystem says so at the top of its notes. On esp32, autosize works only if the flash size is right: the partition table tool refuses a table larger than the configured flash, so give `--flash`. @@ -60,7 +74,7 @@ compile runs twice. ## When nothing will fit How much fits depends on the flash. On a Pico 2 (4 MB), `all` builds and fits, -using 93 % of its 2560 KB firmware region and leaving a 1536 KB filesystem. On +using 94 % of its 3392 KB firmware region and leaving a 704 KB filesystem. On a Pico (2 MB) it can't: the image is bigger than the whole flash, so the build stops at the first link with the overflow error. Name what your project uses instead: diff --git a/docs/reference/build-command.md b/docs/reference/build-command.md index 3b64b31..033e7de 100644 --- a/docs/reference/build-command.md +++ b/docs/reference/build-command.md @@ -3,7 +3,7 @@ ```text ./build_mp.py [--interpreter {micropython,circuitpython}] [--port PORT] [--board BOARD] [--variant VARIANT] [--modules MODULES] - [--flash FLASH] [--no-autosize] [--coredump] [--clean] + [--flash FLASH] [--no-autosize] [--keep-fs SIZE] [--coredump] [--clean] [make arguments] ``` @@ -22,6 +22,7 @@ optional. Leave out `--modules` and it lists [`modules/`](https://github.com/PyD | `--modules` | A comma list of short names, full paths, or `all`. See [Modules](modules.md). | | `--flash` | esp32 flash size, such as `16MB`. | | `--no-autosize` | esp32, rp2 and the Arduino GIGA: refuse instead of growing the firmware's share of flash. | +| `--keep-fs` | esp32 and rp2: build beside the filesystem a board already has, its size in bytes or KB (`576K`), and stop instead of moving it. See [Keep a board's files through an update](../guides/fit-firmware-on-a-small-flash.md#keep-a-boards-files-through-an-update). | | `--coredump` | esp32: a troubleshooting build that writes a core dump to flash on a crash. Standard builds don't. | | `--clean` | Delete this target's build directory first. | diff --git a/scripts/release_firmware.py b/scripts/release_firmware.py index 0405cc8..015c080 100755 --- a/scripts/release_firmware.py +++ b/scripts/release_firmware.py @@ -24,11 +24,13 @@ beside them in . Exits 1 if any check fails. It never uploads or publishes anything. - release_firmware.py notes [--out FILE] + release_firmware.py notes [--previous FILE] [--out FILE] Check that the set's tag may go on a draft Release and write the Release's notes (to FILE, or stdout). A set built from preview pins only takes a tag that starts with "staged-", so publishing it by - accident would look wrong. This is what the workflow's draft step runs. + accident would look wrong. With --previous, the firmware.json of the + last set people could download, the notes name every board whose + filesystem this set moves. This is what the workflow's draft step runs. release_firmware.py cells List the cells. @@ -36,8 +38,9 @@ The checks in `assemble` are the ones a download would trip over: each file is the one the build wrote (sha256), none is empty or past its flash, a flash image starts like one and carries the app image the build also produced, a UF2 -is whole blocks, no module was built from a dirty tree, and every cell the run -should have built is there. +is whole blocks, no module was built from a dirty tree, every cell the run +should have built is there, and each MicroPython board's filesystem is where +its cell keeps it. """ import argparse import datetime @@ -63,22 +66,29 @@ # `deps` names the cache the workflow keeps its toolchains in. An stm32 cell # also has `base`, where the chip's flash starts: `offset` is an address there, # and the Wi-Fi firmware goes to WIFI_OFFSET in the QSPI flash. +# +# `fs` is the filesystem a MicroPython board keeps from one release to the next, +# the end of its flash, which build_mp.py builds beside (--keep-fs). An image +# that outgrows the flash left to it fails its build instead of shipping a +# filesystem that moved: a board that updated to that would come up with an +# empty one. Changing `fs` moves the filesystem on that board, so it is done on +# purpose, and the notes of the release that ships it say so (`notes --previous`). CELLS = [ dict(id="mp-p4-devkit", label="ESP32-P4-WIFI6-DEV-KIT", interpreter="micropython", port="esp32", board="ESP32_GENERIC_P4", variant="WIFI6_DEV_KIT_H2", flash="16MB", modules="all", - chip="esp32p4", offset=0x2000, kind="esp32", deps="idf"), + chip="esp32p4", offset=0x2000, kind="esp32", deps="idf", fs="10176K"), dict(id="mp-p4-panel", label="ESP32-P4-WIFI6-Touch-LCD-4B", interpreter="micropython", port="esp32", board="ESP32_GENERIC_P4", variant="PRE_REV3_C6_WIFI", flash="32MB", modules="all", - chip="esp32p4", offset=0x2000, kind="esp32", deps="idf"), + chip="esp32p4", offset=0x2000, kind="esp32", deps="idf", fs="26880K"), dict(id="mp-s3-lcd7", label="ESP32-S3-Touch-LCD-7", interpreter="micropython", port="esp32", board="ESP32_GENERIC_S3", variant="LCD_7", flash="8MB", modules="all", - chip="esp32s3", offset=0, kind="esp32", deps="idf"), + chip="esp32s3", offset=0, kind="esp32", deps="idf", fs="2688K"), dict(id="mp-s3-lcd43", label="ESP32-S3-Touch-LCD-4.3", interpreter="micropython", port="esp32", board="ESP32_GENERIC_S3", variant="LCD_4_3", flash="8MB", modules="all", - chip="esp32s3", offset=0, kind="esp32", deps="idf"), + chip="esp32s3", offset=0, kind="esp32", deps="idf", fs="2688K"), dict(id="mp-s3-tembed", label="LilyGO T-Embed S3", interpreter="micropython", port="esp32", board="ESP32_GENERIC_S3", variant="T_EMBED", flash="16MB", modules="all", - chip="esp32s3", offset=0, kind="esp32", deps="idf"), + chip="esp32s3", offset=0, kind="esp32", deps="idf", fs="10880K"), dict(id="cp-p4-devkit", label="ESP32-P4-WIFI6-DEV-KIT", interpreter="circuitpython", port="espressif", board="waveshare_esp32p4_wifi6_dev_kit", variant=None, flash="16MB", modules="audiodsp,pygraphics,pngio,tsmux,ppa,jpegio,cameraif,castif,h264enc,usbhost,pydevices,pdwidgets,palettes,lvgl", @@ -105,10 +115,10 @@ # Off by default until a board is on the bench to run it on. dict(id="mp-pico2", label="Raspberry Pi Pico 2", interpreter="micropython", port="rp2", board="RPI_PICO2", variant=None, flash="4MB", modules="all", - chip="rp2350", offset=None, kind="rp2", deps="none", optional=True), + chip="rp2350", offset=None, kind="rp2", deps="none", fs="704K", optional=True), dict(id="mp-pico-plus2-w", label="Pimoroni Pico Plus 2 W", interpreter="micropython", port="rp2", board="RPI_PICO2_W", variant="PIMORONI_PICO_PLUS2_W", flash="16MB", modules="all", - chip="rp2350", offset=None, kind="rp2", deps="none", optional=True), + chip="rp2350", offset=None, kind="rp2", deps="none", fs="12416K", optional=True), ] # What each kind of cell must hand over, as (role, file in the build dir, required). @@ -166,6 +176,33 @@ def wanted(pico2=False): return [c for c in CELLS if pico2 or not c.get("optional")] +def size_bytes(text): + """A cell's `fs` in bytes ("2688K", "1M"), or None if it isn't whole 4 KB flash sectors.""" + m = re.fullmatch(r"(\d+)([KM]?)", text or "") + n = int(m.group(1)) * {"": 1, "K": 1024, "M": MB}[m.group(2)] if m else 0 + return n if n and n % 4096 == 0 else None + + +def keeps_fs(c): + """Whether a cell's filesystem is ours to keep in place: MicroPython's on + esp32 and rp2, whose layout build_mp.py works out. CircuitPython's boards and + the GIGA's QSPI filesystem have a layout of their own.""" + return c["interpreter"] == "micropython" and c["kind"] in ("esp32", "rp2") + + +def check_cells(): + """Every cell whose filesystem build_mp.py places names the size it keeps.""" + problems = [] + for c in CELLS: + if keeps_fs(c) and not size_bytes(c.get("fs")): + problems.append(f"{c['id']}: names no filesystem to keep (fs={c.get('fs')!r}), so a release " + "could move it and empty every board that updates") + elif not keeps_fs(c) and "fs" in c: + problems.append(f"{c['id']}: fs is for MicroPython's esp32 and rp2 cells") + if problems: + raise Failed("\n".join(problems)) + + def sha256(path): h = hashlib.sha256() with open(path, "rb") as f: @@ -187,6 +224,7 @@ def read_lock(name): def matrix(pico2, only=()): """The cells to build: all the wanted ones, or just the ids in `only`.""" + check_cells() for cell_id in only: cell(cell_id) rows = [{k: c[k] for k in ("id", "label", "kind", "deps", "interpreter")} @@ -245,6 +283,8 @@ def build_args(c): args += ["--variant", c["variant"]] if c["interpreter"] == "micropython" and c["port"] == "esp32": args += ["--flash", c["flash"]] + if c.get("fs"): + args += ["--keep-fs", c["fs"]] args += ["--modules", c["modules"]] return args @@ -261,7 +301,12 @@ def build(cell_id, out): rc = subprocess.run(args, cwd=REPO).returncode seconds = round(time.time() - start) if rc != 0: - raise Failed(f"build_mp.py failed for {c['id']} (exit {rc}) after {seconds} s") + kept = ("" if not c.get("fs") else + f". If it says the firmware doesn't fit beside the {c['fs']} filesystem, the image has outgrown " + "the flash this board keeps for it: build fewer modules into it, or give the cell a smaller fs on " + "purpose, which moves the board's filesystem (the release notes then tell its users to copy their " + "files off first)") + raise Failed(f"build_mp.py failed for {c['id']} (exit {rc}) after {seconds} s{kept}") src = build_dir(c) record = json.loads((src / "pydevices-build.json").read_text()) if not record.get("complete"): @@ -304,6 +349,9 @@ def build(cell_id, out): meta["module_revisions"] = {n: m for n, m in sorted((record.get("module_revisions") or {}).items())} if c["interpreter"] == "circuitpython": meta["circuitpython_patches"] = record.get("circuitpython_patches", []) + if c["kind"] == "rp2" and record.get("rp2_storage_bytes"): + # A UF2 doesn't say where the filesystem starts; the build's CMake cache does. + meta["rp2_storage_bytes"] = record["rp2_storage_bytes"] if c["kind"] == "esp32": args_json = json.loads((src / "flasher_args.json").read_text()) meta["esptool"] = {"chip": args_json.get("extra_esptool_args", {}).get("chip"), @@ -322,6 +370,62 @@ def build(cell_id, out): UF2_MAGICS = (b"UF2\n", b"WQ]\x9e", b"\x30\x6f\xb1\x0a") # start magic 0, 1 and the end magic # Smallest sizes a firmware that carries this interpreter and its modules can have. MIN_BYTES = {"esp32": 1 * MB, "espressif": 1 * MB, "rp2": 512 * 1024, "stm32": 1 * MB} +# Where an RP2040's or RP2350's flash appears, and the UF2 family of a block +# written at an absolute address (the RP2350's errata block past the image). +XIP_BASE = 0x10000000 +UF2_FAMILY_ABSOLUTE = 0xE48BFF57 + + +def esp32_fs(table, flash_bytes): + """(offset, size) of the filesystem MicroPython mounts with this binary + partition table: the data partition named vfs (or ffat), or else the flash + after the last partition, which it makes one of at boot.""" + end, named = 0, None + for at in range(0, len(table) - 31, 32): + if table[at:at + 2] != b"\xaa\x50": + break + ptype, offset, size = table[at + 2], *struct.unpack_from(" XIP_BASE + offset: + problems.append(f"{c['id']}: {data['uf2'].name} writes past {XIP_BASE + offset:#x}, into the filesystem") + if "flash" in data and c["offset"] + data["flash"].stat().st_size > offset: + problems.append(f"{c['id']}: {data['flash'].name} writes past {offset:#x}, into the filesystem") + return problems def check_stm32(c, data, flash_bytes): @@ -417,6 +521,8 @@ def check_cell(d, c): problems.append(f"{c['id']}: {data['uf2'].name} has no UF2 block magic") elif len(blob) // 512 * 256 > flash_bytes: problems.append(f"{c['id']}: {data['uf2'].name} carries more than the {c['flash']} flash holds") + if c.get("fs"): + problems += check_fs(c, meta, data, flash_bytes) for role in ("elf", "map"): if role in data and data[role].stat().st_size < 1024: problems.append(f"{c['id']}: {data[role].name} is only {data[role].stat().st_size} bytes") @@ -503,6 +609,8 @@ def assemble(indir, out, pico2, tag, run_url, now=None): entry["esptool"] = m["esptool"] if "circuitpython_patches" in m: entry["circuitpython_patches"] = m["circuitpython_patches"] + if m.get("fs_offset"): + entry["fs_offset"], entry["fs_bytes"] = m["fs_offset"], m["fs_bytes"] entry["files"] = files builds.append(entry) @@ -520,11 +628,11 @@ def assemble(indir, out, pico2, tag, run_url, now=None): def summary(doc): lines = [f"Firmware set {doc['tag']}" + (" (preview pins: never published)" if doc["preview"] else ""), - "", "| Cell | Build | Flash image | Offset | Build time |", "|---|---|---:|---|---:|"] + "", "| Cell | Build | Flash image | Offset | Filesystem | Build time |", "|---|---|---:|---|---|---:|"] for b in doc["builds"]: main = next(f for f in b["files"] if f["role"] in ("flash", "uf2")) lines.append(f"| {b['id']} | {b['label']} | {main['size'] / MB:.2f} MB | {b.get('flash_offset') or 'UF2'} " - f"| {b['build_seconds'] // 60} min {b['build_seconds'] % 60:02d} s |") + f"| {fs_text(b)} | {b['build_seconds'] // 60} min {b['build_seconds'] % 60:02d} s |") return "\n".join(lines) + "\n" @@ -560,8 +668,25 @@ def write_command(b, f): return f"esptool --chip {b['chip']} write-flash {b['flash_offset']} {f['name']}" -def notes(doc): - """The draft Release's body, for whoever opens it.""" +def kind_of(b): + return "MicroPython" if b["interpreter"] == "micropython" else "CircuitPython-compatible" + + +def fs_text(b): + return f"{b['fs_bytes'] // 1024:,} KB at {b['fs_offset']}" if b.get("fs_offset") else "" + + +def moved(doc, previous): + """The builds whose filesystem isn't where the previous set put it.""" + before = {b["id"]: b for b in (previous or {}).get("builds", []) if b.get("fs_offset")} + return [(b, before[b["id"]]) for b in doc["builds"] if b["id"] in before and b.get("fs_offset") + and (b["fs_offset"], b["fs_bytes"]) != (before[b["id"]]["fs_offset"], before[b["id"]]["fs_bytes"])] + + +def notes(doc, previous=None): + """The draft Release's body, for whoever opens it. `previous` is the + firmware.json of the last set people could download: a board whose + filesystem this set moves is named at the top.""" tag = doc["tag"] lines = [] if doc["preview"]: @@ -569,13 +694,20 @@ def notes(doc): ". This draft stays private until a set is built from tagged versions.", ""] lines += [f"OmniPython firmware {tag}: MicroPython {doc['micropython']}" + (f" and CircuitPython-compatible {doc['circuitpython']}" if doc.get("circuitpython") else "") - + f", built by [this run]({doc['run']}) on {doc['built'][:10]}.", "", - "**Before you flash,** copy your files off the board, `secrets.py` above all. A build can move the " - "filesystem, and after a flash across a layout change MicroPython finds the old data where its " - "filesystem now starts, says it looks corrupted and mounts nothing. So when the partition table " - "changes, or you come from another firmware (MicroPython to CircuitPython or back), run " - "`esptool --chip erase-flash` first, then put your files back.", "", - "| Board | Interpreter | File to flash | How |", "|---|---|---|---|"] + + f", built by [this run]({doc['run']}) on {doc['built'][:10]}.", ""] + shifted = moved(doc, previous) + if shifted: + lines += [f"**This release moves the filesystem** on the boards below. Updating one from {previous['tag']} " + "leaves it with an empty filesystem, so copy its files off first, and put them back after.", ""] + lines += [f"- {b['label']}, {kind_of(b)}: {fs_text(b)}, was {fs_text(old)}" for b, old in shifted] + lines += [""] + lines += ["**Before you flash,** copy your files off the board, `secrets.py` above all. Each MicroPython " + "board's filesystem keeps its size and place from one release to the next, as the table gives it, " + "and a release that moves one says so at the top. After a flash across a layout change MicroPython " + "finds the old data where its filesystem now starts, says it looks corrupted and mounts nothing. " + "So when the filesystem moves, or you come from another firmware (MicroPython to CircuitPython or " + "back), run `esptool --chip erase-flash` first, then put your files back.", "", + "| Board | Interpreter | File to flash | How | Filesystem |", "|---|---|---|---|---|"] for b in doc["builds"]: f = next(x for x in b["files"] if x["role"] in ("flash", "uf2")) how = ("copy it onto the board's BOOTSEL drive" if f["role"] == "uf2" @@ -586,8 +718,7 @@ def notes(doc): # Two images, the Wi-Fi firmware first. names = f"`{wifi['name']}`, then `{f['name']}`" how = f"`{write_command(b, wifi)}`, then `{write_command(b, f)}`" - kind = "MicroPython" if b["interpreter"] == "micropython" else "CircuitPython-compatible" - lines.append(f"| {b['label']} | {kind} {b['interpreter_version']} | {names} | {how} |") + lines.append(f"| {b['label']} | {kind_of(b)} {b['interpreter_version']} | {names} | {how} | {fs_text(b)} |") if any(b["chip"] == "stm32h747" for b in doc["builds"]): lines += ["", "The GIGA's two images are written in its bootloader (`machine.bootloader()` at the REPL, or " @@ -627,6 +758,7 @@ def main(argv=None): a.add_argument("--run-url", default="") n = sub.add_parser("notes") n.add_argument("firmware_json") + n.add_argument("--previous", help="the firmware.json of the last set people could download") n.add_argument("--out") sub.add_parser("cells") args = ap.parse_args(argv) @@ -640,7 +772,11 @@ def main(argv=None): elif args.cmd == "notes": doc = json.loads(Path(args.firmware_json).read_text()) check_draft_tag(doc) - text = notes(doc) + previous = json.loads(Path(args.previous).read_text()) if args.previous else None + text = notes(doc, previous) + for b, old in moved(doc, previous): + print(f"note: {b['id']}'s filesystem moves: {fs_text(b)}, was {fs_text(old)} in {previous['tag']}", + file=sys.stderr) if args.out: Path(args.out).write_text(text) else: diff --git a/tests/test_build_mp.py b/tests/test_build_mp.py index 51d8146..0b40cca 100644 --- a/tests/test_build_mp.py +++ b/tests/test_build_mp.py @@ -820,6 +820,74 @@ def test_a_named_filesystem_partition_keeps_the_minimum(self): self.assertEqual(parts["factory"][1], 0x7D0000) +class Esp32KeepFs(unittest.TestCase): + """--keep-fs on esp32 sizes the app partition so the filesystem is the last + SIZE of the flash, where a board flashed with the same size has it.""" + + TABLE = Esp32Autosize.TABLE + + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.dir = Path(self.tmp.name) + + def tearDown(self): + self.tmp.cleanup() + + def keep(self, fs, mb=8, table=None, src=None): + if src is None: + src = self.dir / "partitions.csv" + src.write_text(table or self.TABLE) + dest = build_mp.esp32_keep_fs(src, mb * 1024 * 1024, fs, self.dir / "kept.csv") + return {r[0]: (int(r[3], 0), int(r[4], 0)) for r in build_mp.esp32_table_rows(dest)} + + def test_the_release_cells_keep_the_tables_their_last_set_had(self): + # The app partitions the release workflow's set of 2026-11-09 shipped with, + # each grown by autosize but the DEV-KIT's. + shipped = {"mp-p4-devkit": 0x600000, "mp-p4-panel": 0x5B0000, "mp-s3-lcd7": 0x540000, + "mp-s3-lcd43": 0x540000, "mp-s3-tembed": 0x540000} + for c in rf.CELLS: + if c["interpreter"] != "micropython" or c["kind"] != "esp32": + continue + table = build_mp.esp32_table(c["chip"], c["board"], c["variant"]) + self.assertIsNotNone(table, f"{c['id']} needs a table build_mp.py knows before it builds") + parts = self.keep(rf.size_bytes(c["fs"]), int(c["flash"][:-2]), src=table) + self.assertEqual(parts["factory"], (0x10000, shipped[c["id"]]), c["id"]) + + def test_a_table_kept_at_its_own_size_is_unchanged(self): + for variant, fs, mb in (("ESP32_GENERIC_P4/P4X_EYE", 0x6F0000, 16), ("ESP32_GENERIC/AUDIO_KIT", 0x20000, 4)): + src = build_mp.VARIANTS_DIR / "esp32" / variant / "partitions.csv" + before = {r[0]: (int(r[3], 0), int(r[4], 0)) for r in build_mp.esp32_table_rows(src)} + self.assertEqual(self.keep(fs, mb, src=src), before, variant) + + def test_partitions_after_the_app_follow_it(self): + parts = self.keep(2688 * 1024, table=self.TABLE + "coredump, data, coredump, 0x200000, 0x10000\n") + self.assertEqual(parts["factory"], (0x10000, 0x540000)) + self.assertEqual(parts["coredump"], (0x550000, 0x10000)) + + def test_a_named_filesystem_is_the_end_of_the_flash(self): + parts = self.keep(0x200000, table=self.TABLE + "vfs, data, fat, 0x200000, 0x200000\n") + self.assertEqual(parts["vfs"], (0x600000, 0x200000)) + self.assertEqual(parts["factory"], (0x10000, 0x5F0000)) + + def test_an_image_that_outgrows_it_is_told_what_fits(self): + table = self.dir / "kept.csv" + self.keep(2688 * 1024, table=self.TABLE + "coredump, data, coredump, 0x200000, 0x10000\n") + log = "Error: app partition is too small for binary micropython.bin size 0x560000:\n" + # 0x560000 of app, the coredump after it: 0x800000 - 0x10000 - 0x560000 - 0x10000 is left. + self.assertEqual(build_mp.esp32_keep_fs_fits(log, table, 0x800000), 0x280000) + self.assertEqual(build_mp.esp32_keep_fs_fits(log.replace("0x560000", "0x7F0000"), table, 0x800000), 0) + self.assertIsNone(build_mp.esp32_keep_fs_fits("error: something else", table, 0x800000)) + + def test_tables_it_cannot_size_are_refused(self): + ota = self.TABLE.replace("factory, app, factory", "ota_0, app, ota_0") + "ota_1, app, ota_1, 0x200000, 0x1F0000\n" + middle = self.TABLE + "vfs, data, fat, 0x200000, 0x100000\ncoredump, data, coredump, 0x300000, 0x10000\n" + for table in (ota, middle): + with self.assertRaises(SystemExit): + self.keep(0x100000, table=table) + with self.assertRaises(SystemExit): + self.keep(0x7F0000) # no room left for the app + + class Stm32M4Slot(unittest.TestCase): """An Arduino GIGA firmware that overflows its 1,280 KB region is built again over the M4 core's flash, without OpenAMP; and no GIGA image may diff --git a/tests/test_release_firmware.py b/tests/test_release_firmware.py index 6710619..46fa1d0 100644 --- a/tests/test_release_firmware.py +++ b/tests/test_release_firmware.py @@ -17,6 +17,7 @@ import tempfile import unittest from pathlib import Path +from unittest import mock sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "scripts")) import release_firmware as rf @@ -29,6 +30,19 @@ def put(d, name, data): return {"file": name, "size": len(data), "sha256": hashlib.sha256(data).hexdigest()} +def table_bin(rows): + """A binary ESP32 partition table: (name, type, subtype, offset, size) rows.""" + return b"".join(struct.pack("<2sBBII16sI", b"\xaa\x50", t, st, off, size, name.encode(), 0) + for name, t, st, off, size in rows) + b"\xff" * 64 + + +def kept_table(c): + """The table a cell's build makes: the app partition up to the filesystem it keeps.""" + flash, fs = int(c["flash"][:-2]) * rf.MB, rf.size_bytes(c["fs"]) + return table_bin([("nvs", 1, 2, 0x9000, 0x6000), ("phy_init", 1, 1, 0xF000, 0x1000), + ("factory", 0, 0, 0x10000, flash - fs - 0x10000)]) + + def fake_cell(root, c, version="v1.30.0-preview"): """A cell directory as `build` leaves it, with files that pass every check.""" d = Path(root) / c["id"] @@ -41,7 +55,7 @@ def fake_cell(root, c, version="v1.30.0-preview"): merged = b"\xff" * boot + b"\xe9" + b"\x00" * (0x10000 - boot - 1) + app + pad files["flash"] = put(d, "flash.bin", merged) files["app"] = put(d, "app.bin", app) - files["partitions"] = put(d, "partitions.bin", b"\xaaP" + b"\x00" * 100) if c["kind"] == "esp32" else None + files["partitions"] = put(d, "partitions.bin", kept_table(c)) if c["kind"] == "esp32" else None if files["partitions"] is None: del files["partitions"] elif c["kind"] == "stm32": @@ -61,6 +75,8 @@ def fake_cell(root, c, version="v1.30.0-preview"): flash_bytes=int(c["flash"][:-2]) * rf.MB, files=files, build_seconds=61, interpreter_version=version if c["interpreter"] == "micropython" else "11.0.0-alpha.1-dirty", module_revisions={"pydevices": {"commit": COMMIT, "dirty": False}}) + if c["kind"] == "rp2" and c.get("fs"): + meta["rp2_storage_bytes"] = rf.size_bytes(c["fs"]) (d / "cell.json").write_text(json.dumps(meta)) return d @@ -97,6 +113,32 @@ def test_build_command(self): self.assertEqual(args[args.index("--variant") + 1], "LCD_7") self.assertEqual(args[args.index("--modules") + 1], "all") + def test_every_micropython_esp32_and_rp2_cell_keeps_its_filesystem(self): + rf.check_cells() + kept = [c["id"] for c in rf.CELLS if c.get("fs")] + self.assertEqual(len(kept), 7) + for c in rf.CELLS: + args = rf.build_args(c) + if c.get("fs"): + self.assertEqual(args[args.index("--keep-fs") + 1], c["fs"], c["id"]) + else: + self.assertNotIn("--keep-fs", args, c["id"]) + + def test_a_cell_that_names_no_filesystem_is_caught(self): + for planted in (None, "2690K", "big"): + cells = [dict(c) for c in rf.CELLS] + cells[2]["fs"] = planted + with mock.patch.object(rf, "CELLS", cells): + with self.assertRaises(rf.Failed) as cm: + rf.matrix(False) + self.assertIn("mp-s3-lcd7: names no filesystem to keep", str(cm.exception)) + + def test_a_filesystem_on_a_cell_that_cannot_keep_one_is_caught(self): + cells = [dict(c, fs="1536K") if c["id"] == "cp-feather-rp2040" else c for c in rf.CELLS] + with mock.patch.object(rf, "CELLS", cells): + with self.assertRaisesRegex(rf.Failed, "cp-feather-rp2040: fs is for"): + rf.check_cells() + def test_circuitpython_names_lvgl_like_micropython(self): c = rf.cell("cp-s3-lcd7") args = rf.build_args(c) @@ -302,6 +344,85 @@ def test_a_stray_directory_is_caught(self): (self.indir / "mp-extra").mkdir() self.fails_with("no cell") + def test_firmware_json_says_where_each_filesystem_is(self): + doc = self.run_assemble() + fs = {b["id"]: (b.get("fs_offset"), b.get("fs_bytes")) for b in doc["builds"]} + self.assertEqual(fs["mp-s3-lcd7"], ("0x560000", 2688 * 1024)) + self.assertEqual(fs["mp-p4-panel"], ("0x5c0000", 26880 * 1024)) + self.assertEqual(fs["cp-s3-lcd7"], (None, None)) + + def rewrite(self, cell_id, role, data): + d = self.indir / cell_id + meta = json.loads((d / "cell.json").read_text()) + f = d / meta["files"][role]["file"] + f.write_bytes(data) + meta["files"][role].update(size=len(data), sha256=rf.sha256(f)) + (d / "cell.json").write_text(json.dumps(meta)) + + def test_a_filesystem_that_moved_is_caught(self): + # autosize's table for a bigger image: the app grown by 64 KB, the filesystem after it. + self.rewrite("mp-s3-lcd7", "partitions", table_bin([ + ("nvs", 1, 2, 0x9000, 0x6000), ("phy_init", 1, 1, 0xF000, 0x1000), + ("factory", 0, 0, 0x10000, 0x550000), ("coredump", 1, 3, 0x560000, 0x10000)])) + self.fails_with("mp-s3-lcd7: the filesystem is 2624 KB at 0x570000, not the 2688K at 0x560000") + + def test_a_named_filesystem_in_the_wrong_place_is_caught(self): + self.rewrite("mp-p4-devkit", "partitions", table_bin([ + ("nvs", 1, 2, 0x9000, 0x6000), ("factory", 0, 0, 0x10000, 0x500000), + ("vfs", 1, 0x81, 0x510000, 0xAF0000)])) + self.fails_with("mp-p4-devkit: the filesystem is 11200 KB at 0x510000") + + def test_a_flash_image_that_runs_into_the_filesystem_is_caught(self): + d = self.indir / "mp-s3-tembed" + meta = json.loads((d / "cell.json").read_text()) + flash = (d / meta["files"]["flash"]["file"]).read_bytes() + self.rewrite("mp-s3-tembed", "flash", flash + b"\xff" * (0x560000 - len(flash) + 1)) + self.fails_with("mp-s3-tembed: flash.bin writes past 0x560000, into the filesystem") + + def pico2_set(self): + for c in rf.wanted(True): + if c.get("optional"): + fake_cell(self.indir, c) + return self.indir / "mp-pico2" + + @staticmethod + def uf2_block(addr, family=0xE48BFF59): + head = struct.pack("