diff --git a/DEPENDENCIES.lock b/DEPENDENCIES.lock index a36678a..c288e2b 100644 --- a/DEPENDENCIES.lock +++ b/DEPENDENCIES.lock @@ -29,7 +29,7 @@ ulab https://github.com/v923z/micropython-ulab.git 15e9997470 # sibling checkout is linked there instead) and compiles the port for its # interpreter. Move the commit here to move every LVGL build, MicroPython's and # CircuitPython's. -omnipython-lvgl https://github.com/PyDevices/omnipython-lvgl.git a13f5c75e25b8ba5133d7bf6b7a5b09971e3f5dc +omnipython-lvgl https://github.com/PyDevices/omnipython-lvgl.git b44df6b61c99421a9fb8303a3f17bc8bf018fd09 # omnipython-lib is not a module either: it is the repository of our # Python-only packages (pydevices, palettes, pdwidgets, audioinstruments, diff --git a/build_mp.py b/build_mp.py index 665c499..c58755f 100755 --- a/build_mp.py +++ b/build_mp.py @@ -299,7 +299,8 @@ def lvgl_checkout(ws): Fetched shallow, sparse and blobless: the ports, generated/, python/ and the root's files (lv_conf.h), and LVGL's src/ at the commit the repository - records, so it downloads tens of MB rather than LVGL's whole history.""" + records (and its include/, from LVGL 9.6), so it downloads tens of MB + rather than LVGL's whole history.""" url, commit = pin(LVGL_REPO)[1:3] dest = MODULES_DIR / f".{LVGL_REPO}" if dest.is_symlink(): @@ -340,7 +341,8 @@ def sparse(repo, origin, sha, paths): shutil.rmtree(part, ignore_errors=True) die(f"{LVGL_REPO} {commit[:12]} records no lvgl submodule") shutil.rmtree(part / "lvgl") - sparse(part / "lvgl", lvgl_url, lvgl_sha[0], ["src"]) + # LVGL 9.6 keeps its public headers in include/, beside src/. + sparse(part / "lvgl", lvgl_url, lvgl_sha[0], ["include", "src"]) part.rename(dest) return dest.resolve() @@ -2207,6 +2209,9 @@ def main(): ap.add_argument("--coredump", action="store_true", help="esp32: a troubleshooting build that writes a core dump to flash on a crash (standard builds don't)") ap.add_argument("--clean", action="store_true", help="delete this target's build dir first") + ap.add_argument("--python-to-romfs", action="store_true", + help="Arduino GIGA: freeze no Python; compile it into romfs/ beside the firmware instead, " + "for the board's ROMFS partition, which leaves internal flash to the C modules") args, make_extra = ap.parse_known_args() if args.coredump and args.interpreter != "micropython": die("--coredump is for MicroPython on esp32") @@ -2309,6 +2314,8 @@ def main(): module_set = sorted([d.name if d.parent == MODULES_DIR else str(d) for d in module_dirs] + library_packages) want = {"port": port, "board": board, "variant": variant, "flash": flash, "modules": spec, "module_set": module_set} + if args.python_to_romfs: + want["python_to_romfs"] = True # Each module's commit too: a module that moved keeps its name, and make # would reuse frozen content and qstr headers from before it moved. revisions = record(module_dirs, library.source) @@ -2326,6 +2333,13 @@ def main(): # The padding module that moves the code at the M4's boot address # when it must; empty until then. Regenerated before each make. env["PYDEVICES_MODULES"] = ",".join(x for x in (spec, (build / "m4pad-src" / "m4pad").as_posix()) if x) + romfs_list = build / "romfs-python.json" + if args.python_to_romfs: + if port != "stm32" or board not in STM32_M4_SLOT: + die("--python-to-romfs is for the Arduino GIGA, whose manifest moves its Python to ROMFS") + # The board's manifest writes its Python modules here and freezes none. + romfs_list.unlink(missing_ok=True) + env["PYDEVICES_PYTHON_TO_ROMFS"] = romfs_list.as_posix() # The frozen pydevices leaves its host-only files off a board's image. env["PYDEVICES_PLATFORM"] = "desktop" if port in ("unix", "windows", "webassembly") else "board" if version: @@ -2613,6 +2627,8 @@ def relink(p): shown = "\n ".join(stray[:10]) + (f"\n ... and {len(stray) - 10} more" if len(stray) > 10 else "") die(f"some of this build's objects were compiled from outside their module:\n {shown}\n" "A directory of the module's name beside another module was searched before the module's own.") + if args.python_to_romfs: + python_to_romfs(romfs_list, build / "romfs", Path(env["MICROPY_MPYCROSS"]), mpy_cross_arch(mp, port, board)) rec = dict(want, complete=True) if port == "rp2" and rp2_storage: rec["rp2_storage_bytes"] = rp2_storage @@ -2634,6 +2650,45 @@ def relink(p): "micropython", "micropython.exe", "micropython.mjs", "micropython.wasm"): if (build / name).exists(): say(f" {build / name}") + if args.python_to_romfs: + say(f" {build / 'romfs'}/ (the Python modules: add them to the board's ROMFS image, at its root)") + + +# The -march mpy-cross takes for an STM32 series, as ports/stm32/Makefile sets it. +STM32_MPY_ARCH = {"f0": "armv6m", "g0": "armv6m", "l0": "armv6m", "f4": "armv7emsp", "l4": "armv7emsp", + "wb": "armv7emsp", "g4": "armv7emsp", "f7": "armv7emdp", "h7": "armv7emdp"} + + +def mpy_cross_arch(mp, port, board): + """The -march for a board's .mpy files, so native and viper code in them runs.""" + if port != "stm32": + die("--python-to-romfs is for the Arduino GIGA (stm32)") + text = (mp / "ports" / "stm32" / "boards" / board / "mpconfigboard.mk").read_text() + series = re.search(r"^MCU_SERIES\s*=\s*(\w+)", text, re.MULTILINE) + if not series or series.group(1) not in STM32_MPY_ARCH: + die(f"--python-to-romfs: no mpy-cross architecture known for {board}") + return STM32_MPY_ARCH[series.group(1)] + + +def python_to_romfs(listing, out, mpy_cross, arch): + """Compile the Python modules a --python-to-romfs build left out of its + frozen content into out/, laid out as they import, for the board's ROMFS.""" + if not listing.is_file(): + die(f"--python-to-romfs: the build wrote no {listing.name}; this board's manifest doesn't move its Python") + shutil.rmtree(out, ignore_errors=True) + files = json.loads(listing.read_text()) + for source, target, opt in files: + dest = out / target + dest.parent.mkdir(parents=True, exist_ok=True) + if target.endswith(".py"): + cmd = [str(mpy_cross), f"-march={arch}", "-s", target, "-o", str(dest.with_suffix(".mpy")), source] + if opt is not None: + cmd.insert(1, f"-O{opt}") + run(cmd) + else: + shutil.copyfile(source, dest) + size = sum(f.stat().st_size for f in out.rglob("*") if f.is_file()) + say(f"python-to-romfs: {len(files)} modules, {size:,} bytes, in {out}") if __name__ == "__main__": diff --git a/modules/jpegio/src/lvgl_decoder.c b/modules/jpegio/src/lvgl_decoder.c index fe9fba8..73363f9 100644 --- a/modules/jpegio/src/lvgl_decoder.c +++ b/modules/jpegio/src/lvgl_decoder.c @@ -30,8 +30,11 @@ // decoder instead of being claimed. #include "lvgl/lvgl.h" -#include "lvgl/src/draw/lv_image_decoder_private.h" // lv_image_decoder_t.info_cb, lv_image_cache_data_t -#include "lvgl/src/core/lv_global.h" // LV_GLOBAL_DEFAULT()->image_cache_draw_buf_handlers +// lv_image_decoder_t.info_cb, lv_image_cache_data_t and +// LV_GLOBAL_DEFAULT()->image_cache_draw_buf_handlers. lvgl_private.h is where +// every LVGL release keeps them: 9.6 moved the decoder's header from src/draw/ +// to src/image/. +#include "lvgl/src/lvgl_private.h" #include diff --git a/tests/test_build_mp.py b/tests/test_build_mp.py index d39cbd3..8798e37 100644 --- a/tests/test_build_mp.py +++ b/tests/test_build_mp.py @@ -1089,6 +1089,49 @@ def test_a_symbol_is_read_from_an_elf_file(self): self.assertIsNone(build_mp.elf_symbol(self.build / "nothing.elf", "x")) +class PythonToRomfs(unittest.TestCase): + """--python-to-romfs compiles the Python a GIGA build left unfrozen for the board's ROMFS.""" + + def setUp(self): + self.dir = tempfile.TemporaryDirectory() + self.root = Path(self.dir.name) + # A stand-in mpy-cross that records its arguments in the file it writes. + self.cross = self.root / "mpy-cross" + self.cross.write_text("#!/bin/sh\nfor a; do case $prev in -o) out=$a;; esac; prev=$a; done\necho \"$@\" > $out\n") + self.cross.chmod(0o755) + (self.root / "src").mkdir() + (self.root / "src" / "core.py").write_text("x = 1\n") + (self.root / "src" / "data.mpy").write_bytes(b"M\x06") + + def tearDown(self): + self.dir.cleanup() + + def test_compiles_sources_and_copies_mpy_files_where_they_import(self): + listing = self.root / "romfs-python.json" + listing.write_text(json.dumps([[str(self.root / "src" / "core.py"), "asyncio/core.py", None], + [str(self.root / "src" / "core.py"), "fast.py", 3], + [str(self.root / "src" / "data.mpy"), "data.mpy", None]])) + out = self.root / "romfs" + with contextlib.redirect_stdout(io.StringIO()): + build_mp.python_to_romfs(listing, out, self.cross, "armv7emdp") + self.assertEqual(sorted(p.relative_to(out).as_posix() for p in out.rglob("*") if p.is_file()), + ["asyncio/core.mpy", "data.mpy", "fast.mpy"]) + self.assertIn("-march=armv7emdp -s asyncio/core.py", (out / "asyncio" / "core.mpy").read_text()) + self.assertIn("-O3", (out / "fast.mpy").read_text()) + self.assertEqual((out / "data.mpy").read_bytes(), b"M\x06") + + def test_a_build_whose_manifest_wrote_no_list_is_refused(self): + with contextlib.redirect_stderr(io.StringIO()), self.assertRaises(SystemExit): + build_mp.python_to_romfs(self.root / "missing.json", self.root / "romfs", self.cross, "armv7emdp") + + def test_the_giga_compiles_for_its_h7(self): + mp = self.root / "mp" + board = mp / "ports" / "stm32" / "boards" / "ARDUINO_GIGA" + board.mkdir(parents=True) + (board / "mpconfigboard.mk").write_text("USE_MBOOT = 0\nMCU_SERIES = h7\n") + self.assertEqual(build_mp.mpy_cross_arch(mp, "stm32", "ARDUINO_GIGA"), "armv7emdp") + + class Stm32BoardDir(unittest.TestCase): """A variant of ours on stm32 gets a board directory: upstream's, linked through unchanged, a variant file that includes ours, our headers after diff --git a/variants/stm32/ARDUINO_GIGA/DISPLAY_SHIELD/manifest.py b/variants/stm32/ARDUINO_GIGA/DISPLAY_SHIELD/manifest.py index 531047f..3117eea 100644 --- a/variants/stm32/ARDUINO_GIGA/DISPLAY_SHIELD/manifest.py +++ b/variants/stm32/ARDUINO_GIGA/DISPLAY_SHIELD/manifest.py @@ -7,7 +7,24 @@ # variant uses: the networking bundle (webrepl, ntptime, mip, requests) and # logging. It leaves out aioble, which can't run with Bluetooth off, and the # board's senml and msgpackrpc, to keep flash for LVGL. +import os + include("../../../../modules/manifest.py", board_manifest=False) require("bundle-networking") require("logging") + +# build_mp.py --python-to-romfs: every Python module above goes to the board's +# ROMFS partition on its QSPI flash instead of internal flash, which LVGL and +# the C modules need. This writes the list for build_mp.py, which compiles it +# for the board, and freezes nothing. ROMFS is mounted at boot and /rom is on +# sys.path, so the modules import as before. +_python_to_romfs = os.environ.get("PYDEVICES_PYTHON_TO_ROMFS") +_manifest = include.__self__ +# The pass that only collects C modules lists no files: leave the list alone. +if _python_to_romfs and _manifest.files(): + import json + + with open(_python_to_romfs, "w") as _f: + json.dump([[_o.full_path, _o.target_path, _o.opt] for _o in _manifest.files()], _f) + del _manifest.files()[:]