diff --git a/.github/workflows/build-mp.yml b/.github/workflows/build-mp.yml index 00402454..809d096e 100644 --- a/.github/workflows/build-mp.yml +++ b/.github/workflows/build-mp.yml @@ -10,6 +10,8 @@ on: - 'build_mp.py' - 'tests/test_build_mp.py' - 'tests/test_freeze_list.py' + - 'tests/test_effects_selection.py' + - 'modules/audioeffects/**' - 'scripts/freeze_list.py' - 'modules/library.py' - 'modules.lock' @@ -25,6 +27,8 @@ on: - 'build_mp.py' - 'tests/test_build_mp.py' - 'tests/test_freeze_list.py' + - 'tests/test_effects_selection.py' + - 'modules/audioeffects/**' - 'scripts/freeze_list.py' - 'modules/library.py' - 'modules.lock' @@ -47,7 +51,7 @@ jobs: steps: - uses: actions/checkout@v7 - name: Unit tests - run: PYTHONPATH=tests python3 -m unittest test_build_mp test_freeze_list -v + run: PYTHONPATH=tests python3 -m unittest test_build_mp test_freeze_list test_effects_selection -v - name: provenance.json is current run: PYTHONPATH=tests python3 -m unittest test_provenance -v diff --git a/docs/guides/build-with-a-few-instruments.md b/docs/guides/build-with-a-few-instruments.md index f4782bfd..5d86ad7d 100644 --- a/docs/guides/build-with-a-few-instruments.md +++ b/docs/guides/build-with-a-few-instruments.md @@ -1,4 +1,4 @@ -# Build with a few instruments +# Build with a few instruments and effects `audioinstruments` is large: on a 4 MB ESP32, the whole package doesn't fit beside LVGL and the audio modules. Name the instruments you play in @@ -51,3 +51,35 @@ instruments, 82 files and 253 KB of `.mpy`, take about 480 KB of it, where frozen they add 242 KB to the image. Built without `audioinstruments`, the command above makes a 3,157 KB image and leaves a 704 KB filesystem, so they fit there with room for little else. When flash is tight, freeze them. + +## A few effects + +`audioeffects` works the same way. Name the effects you use in +`AUDIOEFFECTS`, by module or class name, and the build freezes only those, +the package's `__init__` and the base they share, and any effect they build +on: + +```bash +AUDIOEFFECTS=chorus,reverb ./build_mp.py --port esp32 --board ESP32_GENERIC --variant AUDIO_KIT --flash 4MB --modules pydevices,displayif,lvgl,btclassic,audioif,audiodsp,audioeffects +``` + +On the board, `audioeffects.ALL` lists `Chorus` and `Reverb`, and +`audioeffects.create("Compressor", ...)` raises `ImportError`. Name a rack, +such as `ShimmerHall`, and the effects it's made of come with it. An unknown +name stops the build and lists the effects the package has. Leave +`AUDIOEFFECTS` unset, or empty, for all of them. + +Only an `audioeffects` that loads each effect the first time it's used can be +frozen in part, which it does from 0.5.2.dev0. With an older one, the build +stops and says so, rather than freezing a package that fails to import. + +Compiled, the whole package is 49 files and 278 KB. Chorus and reverb are 4 +files and 23 KB. On an ESP32 Audio Kit, with the modules above, `soundfont` +from `audioinstruments` and a General MIDI sound module's libraries, the +whole package makes a 4,095 KB image, which no partition table on a 4 MB flash +can hold. With the two effects it's 3,843 KB, 252 KB less, and fits the kit's +3,904 KB app partition beside a 128 KB filesystem. + +`modules/audioeffects/selection.py` writes the same selection as files, with +the same `--out` and `--mpy-cross` options as the instruments' script, for a +board with room on its filesystem. diff --git a/docs/guides/choose-modules.md b/docs/guides/choose-modules.md index 1ee6900c..7188ca9b 100644 --- a/docs/guides/choose-modules.md +++ b/docs/guides/choose-modules.md @@ -42,5 +42,6 @@ The desktop `pydevices` variants freeze a board config that imports `all` is large. On a small flash, name only what your project uses: [Fit firmware on a small flash](fit-firmware-on-a-small-flash.md). -`audioinstruments` can be cut down to the instruments you play: -[Build with a few instruments](build-with-a-few-instruments.md). +`audioinstruments` and `audioeffects` can be cut down to the instruments and +effects you use: +[Build with a few instruments and effects](build-with-a-few-instruments.md). diff --git a/docs/guides/fit-firmware-on-a-small-flash.md b/docs/guides/fit-firmware-on-a-small-flash.md index 1a919891..a1dc4898 100644 --- a/docs/guides/fit-firmware-on-a-small-flash.md +++ b/docs/guides/fit-firmware-on-a-small-flash.md @@ -47,5 +47,5 @@ A display project's set like this one builds to about 540 KB and fits with the Pico's default 1408 KB filesystem untouched. On a 4 MB ESP32, `audioinstruments` is the biggest of the modules, and you can -freeze only the instruments you play: -[Build with a few instruments](build-with-a-few-instruments.md). +freeze only the instruments you play, and of `audioeffects` only the effects you +use: [Build with a few instruments and effects](build-with-a-few-instruments.md). diff --git a/docs/index.md b/docs/index.md index dac55948..4f95086b 100644 --- a/docs/index.md +++ b/docs/index.md @@ -15,7 +15,7 @@ for boards, desktops and the browser. One command, [`build_mp.py`](https://githu - [Choose modules](guides/choose-modules.md) - [Flash a board](guides/flash-a-board.md) - [Fit firmware on a small flash](guides/fit-firmware-on-a-small-flash.md) -- [Build with a few instruments](guides/build-with-a-few-instruments.md) +- [Build with a few instruments and effects](guides/build-with-a-few-instruments.md) - [Build in a codespace](guides/build-in-a-codespace.md) - [Build on Windows](guides/build-on-windows.md) - [Build for CircuitPython](guides/build-for-circuitpython.md) diff --git a/modules/audioeffects/manifest.py b/modules/audioeffects/manifest.py index 3f830b7a..bea16c72 100644 --- a/modules/audioeffects/manifest.py +++ b/modules/audioeffects/manifest.py @@ -6,5 +6,31 @@ # builds from the PyDevices/audioeffects repository at the tag its # lib-lock.json pins. They import ulab and audiodsp, so name those too (or # use all). Frozen at opt=3. +# +# AUDIOEFFECTS in the environment, a comma list of effects (chorus,reverb), +# freezes only those and what they import (selection.py, beside this file, +# works it out), with an `audioeffects.ALL` that lists just them. Unset, the +# whole package. +import os +import sys + include("../library.py") -require("audioeffects", library="omnipython-lib", opt=3) +_picked = [n.strip() for n in os.environ.get("AUDIOEFFECTS", "").split(",") if n.strip()] +if not _picked: + require("audioeffects", library="omnipython-lib", opt=3) +else: + # A manifest runs with its own folder as the working directory. Loaded by + # path under its own name: the other package's selection.py has the same + # file name, and a plain import would find whichever came first. + import importlib.util + + _spec = importlib.util.spec_from_file_location("audioeffects_selection", os.path.join(os.getcwd(), "selection.py")) + selection = importlib.util.module_from_spec(_spec) + _spec.loader.exec_module(selection) + + _library = os.environ["PYDEVICES_LIBRARY"] + _out = os.path.join(os.path.dirname(_library), "audioeffects-selected") + _files = selection.write(os.path.join(_library, "audioeffects", "audioeffects"), _picked, _out) + # On stderr: the build reads stdout for C module paths. + print("audioeffects: freezing %s (%d files)" % (", ".join(_picked), len(_files)), file=sys.stderr) + package("audioeffects", base_path=_out, opt=3) diff --git a/modules/audioeffects/selection.py b/modules/audioeffects/selection.py new file mode 100644 index 00000000..7a817f48 --- /dev/null +++ b/modules/audioeffects/selection.py @@ -0,0 +1,200 @@ +# SPDX-FileCopyrightText: 2026 Brad Barnett +# +# SPDX-License-Identifier: MIT +"""Pick the files of audioeffects that a few named effects need. + +The whole package is about 400 KB of bytecode, more than a 4 MB board has +room for beside Bluetooth or LVGL. This finds what the named effects import +(the shared base, and any effect a rack builds on) by following the import +statements, and writes an ``__init__.py`` whose ``ALL`` and lazy class table +list only them. ``from audioeffects import Reverb`` and ``create("Reverb", +...)`` work as before for the effects you chose; any other effect raises +AttributeError or ImportError. + +The manifest beside this file uses it when ``AUDIOEFFECTS`` names effects. +Run it to put the same selection on a board's filesystem: + + python modules/audioeffects/selection.py chorus,reverb --out build-ae --mpy-cross mpy-cross + +Name an effect by its module (``reverb``) or its class (``Reverb``). +``--source`` is the package folder (the one holding ``__init__.py``), by +default the copy a build assembled under ``deps/``. With ``--mpy-cross``, +the files come out compiled. The package has no native code, so the ``.mpy`` +files suit any board. +""" + +import argparse +import ast +import os +import shutil +import subprocess +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_SOURCE = os.path.join(HERE, "..", "..", "deps", "omnipython-lib", "library", "audioeffects", "audioeffects") +PACKAGE = "audioeffects" + +# Modules every selection carries: the package and the base every effect +# builds on. +ALWAYS = ("__init__", "_component") + + +def _init_tree(root: str) -> tuple[str, ast.Module]: + path = os.path.join(root, "__init__.py") + with open(path, encoding="utf-8") as f: + text = f.read() + return text, ast.parse(text, path) + + +def _assignment(tree: ast.Module, name: str) -> ast.Assign | None: + for node in tree.body: + if isinstance(node, ast.Assign) and any(isinstance(t, ast.Name) and t.id == name for t in node.targets): + return node + return None + + +def classes(root: str) -> dict[str, str]: + """Class name -> module, from the package's lazy class table.""" + _text, tree = _init_tree(root) + node = _assignment(tree, "_MODULES") + if node is None: + raise ValueError( + f"the audioeffects at {root} imports every effect when it loads, so a few can't be frozen alone; " + "it needs a release that loads each effect on first use") + return ast.literal_eval(node.value) + + +def resolve(root: str, names: list[str]) -> list[str]: + """The modules for *names*, each a module or class name, in any case.""" + table = classes(root) + modules = {m.lower(): m for m in table.values()} + by_class = {c.lower(): m for c, m in table.items()} + picked, unknown = [], [] + for name in names: + module = modules.get(name.lower()) or by_class.get(name.lower()) + if module is None: + unknown.append(name) + elif module not in picked: + picked.append(module) + if unknown: + raise ValueError(f"not effects of audioeffects: {', '.join(unknown)} (they are {', '.join(sorted(modules.values()))})") + return picked + + +def _imports(path: str, table: dict[str, str]) -> set[str]: + """Every ``audioeffects`` module *path* imports, anywhere in it, + including an effect it reaches by class name through the package.""" + found = set() + with open(path, encoding="utf-8") as f: + tree = ast.parse(f.read(), path) + for node in ast.walk(tree): + if isinstance(node, ast.ImportFrom): + if node.level == 1: + base = node.module + elif node.level == 0 and node.module and node.module.startswith(PACKAGE + "."): + base = node.module[len(PACKAGE) + 1 :] + elif node.level == 0 and node.module == PACKAGE: + base = None + else: + continue + if base: + found.add(base) + else: + for alias in node.names: + found.add(table.get(alias.name, alias.name)) + elif isinstance(node, ast.Import): + for alias in node.names: + if alias.name.startswith(PACKAGE + "."): + found.add(alias.name[len(PACKAGE) + 1 :]) + elif isinstance(node, ast.Attribute) and node.attr in table: + # audioeffects.Reverb, or a create("Reverb", ...) name below. + found.add(table[node.attr]) + elif isinstance(node, ast.Constant) and isinstance(node.value, str) and node.value in table: + found.add(table[node.value]) + return found + + +def files(root: str, names: list[str]) -> list[str]: + """The files, relative to *root*, that effects *names* need.""" + table = classes(root) + todo = list(ALWAYS[1:]) + resolve(root, names) + seen = {"__init__": os.path.join(root, "__init__.py")} + while todo: + module = todo.pop() + path = os.path.join(root, module + ".py") + if module in seen or not os.path.isfile(path): + continue + seen[module] = path + todo += _imports(path, table) + return sorted(os.path.relpath(p, root) for p in seen.values()) + + +def init_source(root: str, modules: list[str]) -> str: + """The package's ``__init__.py`` with its class table and ``__all__`` + cut down to the effects in *modules*.""" + text, tree = _init_tree(root) + table = classes(root) + kept = {c: m for c, m in table.items() if m in modules} + lines = text.splitlines(keepends=True) + edits = [(_assignment(tree, "_MODULES"), "_MODULES = {\n" + "".join(f" {c!r}: {m!r},\n" for c, m in kept.items()) + "}\n")] + exported = _assignment(tree, "__all__") + if exported is not None: + names = [n for n in ast.literal_eval(exported.value) if n not in table or n in kept] + edits.append((exported, "__all__ = " + repr(names) + "\n")) + # From the bottom up, so earlier line numbers stay put. + for node, new in sorted(edits, key=lambda e: e[0].lineno, reverse=True): + lines[node.lineno - 1 : node.end_lineno] = [new] + return "".join(lines) + + +def write(root: str, names: list[str], out: str) -> list[str]: + """Copy the selection to ``/audioeffects``; returns the files written.""" + modules = resolve(root, names) + dest = os.path.join(out, PACKAGE) + if os.path.isdir(dest): + shutil.rmtree(dest) + os.makedirs(dest) + picked = files(root, names) + # An effect a rack builds on is in the selection, so it goes in the table too. + effects = [m for m in classes(root).values() if m + ".py" in picked] + written = [] + for rel in picked: + target = os.path.join(dest, rel) + if rel == "__init__.py": + with open(target, "w", encoding="utf-8") as f: + f.write(init_source(root, sorted(set(modules) | set(effects)))) + else: + shutil.copyfile(os.path.join(root, rel), target) + written.append(target) + return written + + +def main(argv: list[str]) -> int: + ap = argparse.ArgumentParser(description="Copy the audioeffects files a few effects need.") + ap.add_argument("names", help="comma list of effects, e.g. chorus,reverb") + ap.add_argument("--out", required=True, help="folder to write audioeffects/ into") + ap.add_argument("--source", default=DEFAULT_SOURCE, help="the audioeffects package folder") + ap.add_argument("--mpy-cross", help="compile each file with this mpy-cross and keep only the .mpy") + args = ap.parse_args(argv) + root = os.path.abspath(args.source) + if not os.path.isfile(os.path.join(root, "_component.py")): + ap.error(f"no audioeffects package at {root}: build once with --modules audioeffects, or give --source") + names = [n.strip() for n in args.names.split(",") if n.strip()] + try: + written = write(root, names, args.out) + except ValueError as e: + ap.error(str(e)) + total = 0 + for path in written: + if args.mpy_cross: + mpy = path[: -len(".py")] + ".mpy" + subprocess.run([args.mpy_cross, "-O3", "-o", mpy, path], check=True) + os.remove(path) + path = mpy + total += os.path.getsize(path) + print(f"{len(written)} files, {total} bytes, in {os.path.join(args.out, PACKAGE)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/modules/audioinstruments/manifest.py b/modules/audioinstruments/manifest.py index 4a3b7e87..713ecbaf 100644 --- a/modules/audioinstruments/manifest.py +++ b/modules/audioinstruments/manifest.py @@ -19,9 +19,14 @@ if not _picked: require("audioinstruments", library="omnipython-lib", opt=3) else: - # A manifest runs with its own folder as the working directory. - sys.path.insert(0, os.getcwd()) - import selection + # A manifest runs with its own folder as the working directory. Loaded by + # path under its own name: the other package's selection.py has the same + # file name, and a plain import would find whichever came first. + import importlib.util + + _spec = importlib.util.spec_from_file_location("audioinstruments_selection", os.path.join(os.getcwd(), "selection.py")) + selection = importlib.util.module_from_spec(_spec) + _spec.loader.exec_module(selection) _library = os.environ["PYDEVICES_LIBRARY"] _out = os.path.join(os.path.dirname(_library), "audioinstruments-selected") diff --git a/tests/test_effects_selection.py b/tests/test_effects_selection.py new file mode 100644 index 00000000..9087c727 --- /dev/null +++ b/tests/test_effects_selection.py @@ -0,0 +1,172 @@ +# SPDX-FileCopyrightText: 2026 Brad Barnett +# +# SPDX-License-Identifier: MIT + +"""Unit tests for modules/audioeffects/selection.py: which files a few named +effects need, and the package it writes for them. + + PYTHONPATH=tests python3 -m unittest test_effects_selection -v +""" + +import ast +import os +import sys +import tempfile +import unittest +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +sys.path.insert(0, str(REPO / "modules" / "audioeffects")) +import selection # noqa: E402 + +# A small package shaped like audioeffects: a lazy __init__, the shared base, +# two plain effects, a rack that builds on one of them, and one nobody picks. +INIT = '''"""Effects.""" + +from . import _component + +_MODULES = { + "Chorus": "chorus", + "Reverb": "reverb", + "ShimmerHall": "shimmerhall", + "Compressor": "compressor", +} + + +def __getattr__(name): + module = _MODULES.get(name) + if module is None: + raise AttributeError(name) + cls = getattr(__import__("audioeffects." + module, None, None, [name]), name) + globals()[name] = cls + return cls + + +__all__ = [ + "create", "ALL", + "Chorus", "Reverb", "ShimmerHall", "Compressor", +] + + +def create(name, source, rate, **options): + if name not in _MODULES: + raise ImportError("audioeffects has no %s" % name) + return __getattr__(name).create(source, rate, **options) + + +ALL = tuple(sorted(_MODULES)) +''' + +EAGER_INIT = '''from .chorus import Chorus +from .reverb import Reverb +ALL = ("Chorus", "Reverb") +''' + +FILES = { + "_component.py": "import math\n\nclass Component:\n NAME = None\n", + "chorus.py": "from . import _component\n\nclass Chorus(_component.Component):\n NAME = 'Chorus'\n", + "reverb.py": "import math\nfrom . import _component\n\nclass Reverb(_component.Component):\n NAME = 'Reverb'\n", + "shimmerhall.py": "from . import _component\nfrom . import reverb as _reverb\n\nclass ShimmerHall(_component.Component):\n NAME = 'ShimmerHall'\n", + "compressor.py": "from . import _component\n\nclass Compressor(_component.Component):\n NAME = 'Compressor'\n", +} + + +class Selection(unittest.TestCase): + def setUp(self): + self.dir = tempfile.TemporaryDirectory() + self.root = os.path.join(self.dir.name, "src", "audioeffects") + os.makedirs(self.root) + self.write("__init__.py", INIT) + for name, text in FILES.items(): + self.write(name, text) + self.out = os.path.join(self.dir.name, "out") + + def tearDown(self): + self.dir.cleanup() + + def write(self, name, text): + with open(os.path.join(self.root, name), "w", encoding="utf-8") as f: + f.write(text) + + def written_init(self): + with open(os.path.join(self.out, "audioeffects", "__init__.py"), encoding="utf-8") as f: + return ast.parse(f.read()) + + def assigned(self, tree, name): + node = next(n for n in tree.body if isinstance(n, ast.Assign) and n.targets[0].id == name) + return ast.literal_eval(node.value) + + def test_named_effects_take_the_base_and_nothing_else(self): + self.assertEqual(selection.files(self.root, ["chorus", "reverb"]), + ["__init__.py", "_component.py", "chorus.py", "reverb.py"]) + + def test_a_class_name_in_any_case_picks_its_module(self): + self.assertEqual(selection.resolve(self.root, ["Reverb", "CHORUS", "reverb"]), ["reverb", "chorus"]) + + def test_a_rack_brings_the_effect_it_builds_on(self): + self.assertIn("reverb.py", selection.files(self.root, ["shimmerhall"])) + + def test_an_unknown_name_stops_and_lists_the_effects(self): + with self.assertRaises(ValueError) as caught: + selection.files(self.root, ["reverb", "warp"]) + message = str(caught.exception) + self.assertIn("warp", message) + for module in ("chorus", "compressor", "reverb", "shimmerhall"): + self.assertIn(module, message) + + def test_an_eager_package_is_refused(self): + # An audioeffects that imports every effect when it loads can't be + # frozen in part: the build must stop rather than freeze a package + # whose import fails on the board. + self.write("__init__.py", EAGER_INIT) + with self.assertRaises(ValueError) as caught: + selection.write(self.root, ["reverb"], self.out) + self.assertIn("first use", str(caught.exception)) + + def test_the_written_package_lists_only_the_selection(self): + written = selection.write(self.root, ["chorus", "reverb"], self.out) + self.assertEqual(sorted(os.path.basename(p) for p in written), + ["__init__.py", "_component.py", "chorus.py", "reverb.py"]) + tree = self.written_init() + self.assertEqual(self.assigned(tree, "_MODULES"), {"Chorus": "chorus", "Reverb": "reverb"}) + self.assertEqual(self.assigned(tree, "__all__"), ["create", "ALL", "Chorus", "Reverb"]) + + def test_an_effect_a_rack_needs_is_in_the_written_table(self): + selection.write(self.root, ["shimmerhall"], self.out) + self.assertEqual(self.assigned(self.written_init(), "_MODULES"), + {"Reverb": "reverb", "ShimmerHall": "shimmerhall"}) + + def test_the_written_package_imports_and_refuses_the_rest(self): + selection.write(self.root, ["reverb"], self.out) + sys.path.insert(0, self.out) + try: + import audioeffects + self.assertEqual(audioeffects.ALL, ("Reverb",)) + self.assertEqual(audioeffects.Reverb.NAME, "Reverb") + with self.assertRaises(AttributeError): + _ = audioeffects.Compressor + with self.assertRaises(ImportError): + audioeffects.create("Compressor", None, 48000) + finally: + sys.path.remove(self.out) + for name in [m for m in sys.modules if m == "audioeffects" or m.startswith("audioeffects.")]: + del sys.modules[name] + + def test_writing_again_replaces_the_last_selection(self): + selection.write(self.root, ["chorus", "reverb"], self.out) + selection.write(self.root, ["reverb"], self.out) + self.assertFalse(os.path.exists(os.path.join(self.out, "audioeffects", "chorus.py"))) + + +class Manifest(unittest.TestCase): + """The manifest reads AUDIOEFFECTS the way the instruments' reads AUDIOINSTRUMENTS.""" + + def test_unset_or_empty_freezes_the_whole_package(self): + text = (REPO / "modules" / "audioeffects" / "manifest.py").read_text() + self.assertIn('os.environ.get("AUDIOEFFECTS", "")', text) + self.assertIn('require("audioeffects", library="omnipython-lib", opt=3)', text) + self.assertIn("if n.strip()", text) + + +if __name__ == "__main__": + unittest.main()