From 776548547221ce4b63d3f76bce907ff487db6143 Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:43:18 -0600 Subject: [PATCH 1/2] MIP and PyPI publishing: medialink, sayso and midiports reach users The MIP sync stopped at 'no shared description for own-package medialink', so the index never gained medialink, and it listed no sayso among the applibs. It also looked for top-level packages at the repository root, where midiports and the usb-* packages no longer live (packages/ now). It now describes medialink, adds sayso, and finds packages under packages/ and packages/usb/. pydevices-desktop on PyPI can also carry a top-level package: a section marked pypi-desktop = true in mip-split.toml has the files its package.json lists copied in, pinned upstream URLs fetched (adafruit_midi, for midiports). Ran: the unit tests; the MIP sync for pydevices and applibs into a copy of the index repository, built its index, and mip-installed medialink, sayso and midiports from it on the unix port; built both wheels and installed pydevices-desktop in a clean venv. --- data/stack.json | 2 +- docs/how-it-works/release-chain.md | 5 ++ .../build_pydevices_python_distributions.py | 49 +++++++++++++++ scripts/pydevices_package_metadata.py | 2 + scripts/synchronize_mip_package.py | 24 +++++--- tests/test_synchronize_mip_package.py | 60 +++++++++++++++++++ 6 files changed, 134 insertions(+), 8 deletions(-) diff --git a/data/stack.json b/data/stack.json index a7d752f..4dc92d6 100644 --- a/data/stack.json +++ b/data/stack.json @@ -142,7 +142,7 @@ {"name": "picotts", "group": "sound", "does": "Make a board with PSRAM speak, with no network and no computer.", "repo": "micropython-pydevices", "path": "modules/picotts", "get": ["firmware"], "runs": "MP"}, {"name": "mpvst", "group": "sound", "does": "Run Python instruments and effects in your DAW as a VST3 plug-in.", "repo": "mpvst", "path": "README.md", "get": ["download"], "runs": "Desktop DAW"}, - {"name": "midiports", "group": "midi", "does": "Connect `adafruit_midi` to USB, UART, Web MIDI or a computer's own MIDI ports.", "repo": "pydevices", "path": "packages/midiports", "get": ["mip:midiports"], "runs": "MP, Desktop, Web"}, + {"name": "midiports", "group": "midi", "does": "Connect `adafruit_midi` to USB, UART, Web MIDI or a computer's own MIDI ports.", "repo": "pydevices", "path": "packages/midiports", "get": ["mip:midiports", "pip:pydevices-desktop"], "runs": "MP, Desktop, Web"}, {"name": "soundmodule", "group": "midi", "does": "Play MIDI through a sound engine, headless from a port or through files.", "repo": "apps", "path": "lib/soundmodule", "guide": "docs/guides/play-the-sound-module.md", "get": ["mip:soundmodule", "pip:omnipython-applibs"], "runs": "MP, Desktop"}, {"name": "songmodel", "group": "midi", "does": "Edit a groovebox's song as plain data, and carry it to a front panel.", "repo": "apps", "path": "lib/songmodel", "get": ["mip:songmodel", "pip:omnipython-applibs"], "runs": "MP, Desktop, Web"}, diff --git a/docs/how-it-works/release-chain.md b/docs/how-it-works/release-chain.md index 9432567..834e724 100644 --- a/docs/how-it-works/release-chain.md +++ b/docs/how-it-works/release-chain.md @@ -113,6 +113,11 @@ The `pydevices` build keeps no inclusion list. The distribution is exactly single source for its description, `require()` lines and files. The sync fails if a listed name isn't a directory with a `manifest.py`. This is available from `publishing-v14`. +- **A top-level package can also ride in `pydevices-desktop`.** Adding + `pypi-desktop = true` to its section copies the files its `package.json` + lists into the wheel, fetching any pinned upstream URL (midiports carries + `adafruit_midi` this way), so `pip install pydevices-desktop` gives what + `mip` gives. The wheel build reads the key; the MIP sync ignores it. - Every publishable entry under `utils/`, and everything publishable in `board_configs/desktop/`, is bundled into `pydevices-desktop`, which depends on `pydevices`. All artifacts from one release share a version. diff --git a/scripts/build_pydevices_python_distributions.py b/scripts/build_pydevices_python_distributions.py index 0983582..9c25a32 100755 --- a/scripts/build_pydevices_python_distributions.py +++ b/scripts/build_pydevices_python_distributions.py @@ -8,10 +8,12 @@ from __future__ import annotations import argparse +import json import shutil import subprocess import sys import tomllib +import urllib.request from pathlib import Path from pydevices_package_metadata import PYDEVICES_DESCRIPTIONS @@ -91,6 +93,51 @@ def desktop_requirements(root: Path) -> list[str]: return requirements +def desktop_bundles(root: Path) -> list[Path]: + """Packages that pydevices-desktop carries beyond utils/: the directories + mip-split.toml marks ``pypi-desktop = true`` (midiports, for pip users who + have no mip). Each is found at packages/, or packages/usb/.""" + split_path = root / "mip-split.toml" + if not split_path.exists(): + return [] + with split_path.open("rb") as handle: + declared = tomllib.load(handle) + found = [] + for name, section in declared.items(): + if not section.get("pypi-desktop"): + continue + for candidate in (root / "packages" / name, root / "packages" / "usb" / name): + if (candidate / "package.json").is_file(): + found.append(candidate) + break + else: + raise SystemExit(f"mip-split.toml: [{name}] is pypi-desktop but packages/{name}/package.json is missing") + return found + + +GITHUB_SELF = "github:PyDevices/pydevices/" + + +def copy_bundle(package: Path, root: Path, destination: Path) -> None: + """Lay out one package's files from its package.json, the list mip installs. + + A file of this repository is copied from the checkout. Any other file is + fetched from its pinned URL (midiports pins Adafruit's adafruit_midi at a + tag), so pip users get exactly what mip installs. + """ + urls = json.loads((package / "package.json").read_text(encoding="utf-8"))["urls"] + for relative, source in urls: + target = destination / relative + target.parent.mkdir(parents=True, exist_ok=True) + if source.startswith(GITHUB_SELF): + shutil.copy2(root / source[len(GITHUB_SELF) :], target) + elif source.startswith("https://"): + with urllib.request.urlopen(source, timeout=60) as response: + target.write_bytes(response.read()) + else: + raise SystemExit(f"{package.name}/package.json: cannot bundle {source!r}") + + def project_text( name: str, version: str, @@ -190,6 +237,8 @@ def build(root: Path, output: Path, version: str) -> None: copy_component(source, desktop_source / source.name) for source in components(root / DESKTOP_DIR): copy_component(source, desktop_source / source.name) + for bundle in desktop_bundles(root): + copy_bundle(bundle, root, desktop_source) write_project( desktop, "pydevices-desktop", diff --git a/scripts/pydevices_package_metadata.py b/scripts/pydevices_package_metadata.py index 1374c6c..1883d30 100644 --- a/scripts/pydevices_package_metadata.py +++ b/scripts/pydevices_package_metadata.py @@ -9,4 +9,6 @@ "pydevices-desktop": "Complete PyDevices desktop stack and board configuration", # MIP only: on PyPI bledev is inside the pydevices wheel. "bledev": "Portable async Bluetooth Low Energy for PyDevices, over aioble on boards", + # MIP only, like bledev: on PyPI medialink is inside the pydevices wheel. + "medialink": "Live audio and video between boards and computers: a walkie-talkie, an intercom, a baby monitor", } diff --git a/scripts/synchronize_mip_package.py b/scripts/synchronize_mip_package.py index 7f73b23..7891914 100755 --- a/scripts/synchronize_mip_package.py +++ b/scripts/synchronize_mip_package.py @@ -93,6 +93,7 @@ class Profile: "applibs": { "appkit": "Apps you can switch between, and a launcher to run them", "controlkit": "LVGL controls: encoders, sliders, meters, lamp buttons, pad grids and a dot-matrix display", + "sayso": "Lightweight Spotify Web API client for CPython, MicroPython and CircuitPython", "songmodel": "The song a groovebox edits, and the link that carries it to a front panel", "soundmodule": "A sound module that plays MIDI, headless from a port or through files", "spectrum": "A spectrum analyzer in an app's area, sharing the panel with LVGL", @@ -262,7 +263,7 @@ def render_pydevices_manifest(name: str, version: str, requirements: tuple[str, #: Every key a mip-split.toml section may use. An unknown key fails the sync, #: so a misspelt `own-package` can't quietly ship a package inside pydevices. -MIP_SPLIT_KEYS = frozenset({"host-only", "own-package", "requires", "pypi-extras", "top-level"}) +MIP_SPLIT_KEYS = frozenset({"host-only", "own-package", "requires", "pypi-extras", "pypi-desktop", "top-level"}) def read_split(source_root: Path) -> dict[str, dict]: @@ -371,6 +372,15 @@ def check_no_host_imports(package_dir: Path, host_only: frozenset[str]) -> None: ) +def top_level_source(source_root: Path, package: str) -> Path: + """Where a top-level package lives: packages/, packages/usb/, + or (older releases) a directory at the repository root.""" + for candidate in (source_root / "packages" / package, source_root / "packages" / "usb" / package): + if candidate.is_dir(): + return candidate + return source_root / package + + def read_top_level_packages(source_root: Path) -> list[str]: """Names of the packages that live at the repository root, not in lib/. @@ -385,15 +395,15 @@ def read_top_level_packages(source_root: Path) -> list[str]: for package, section in read_split(source_root).items(): if not section.get("top-level"): continue - extra = sorted(set(section) - {"top-level"}) + extra = sorted(set(section) - {"top-level", "pypi-desktop"}) if extra: raise SystemExit( f"{MIP_SPLIT_FILE} [{package}] is top-level, so it takes no other keys " f"(remove {', '.join(extra)}). Its manifest.py says what it requires and ships." ) - package_dir = source_root / package + package_dir = top_level_source(source_root, package) if not package_dir.is_dir(): - raise SystemExit(f"{MIP_SPLIT_FILE} names top-level package {package!r}, which is not a directory at the repository root") + raise SystemExit(f"{MIP_SPLIT_FILE} names top-level package {package!r}, which is not a directory under packages/ or at the repository root") if not (package_dir / "manifest.py").is_file(): raise SystemExit(f"top-level package {package!r} has no manifest.py") if (source_root / "lib" / package).exists(): @@ -403,9 +413,9 @@ def read_top_level_packages(source_root: Path) -> list[str]: # install, then fail on import. Say so here, in the pull request's own # terms, instead of in the index build. for package in names: - manifest = (source_root / package / "manifest.py").read_text(encoding="utf-8") + manifest = (top_level_source(source_root, package) / "manifest.py").read_text(encoding="utf-8") for required in re.findall(r'^\s*require\(\s*["\']([^"\']+)["\']', manifest, re.MULTILINE): - if required not in names and (source_root / required / "manifest.py").is_file(): + if required not in names and (top_level_source(source_root, required) / "manifest.py").is_file(): raise SystemExit( f"top-level package {package!r} requires {required!r}, which is a directory " f"here but is not listed in {MIP_SPLIT_FILE}; add [{required}] with top-level = true" @@ -430,7 +440,7 @@ def stamp_version(manifest: str, package: str, version: str) -> str: def synchronize_top_level(source_root: Path, destination_root: Path, package: str, version: str) -> None: - source = source_root / package + source = top_level_source(source_root, package) destination = destination_root / package destination.mkdir() for path in sorted(source.iterdir()): diff --git a/tests/test_synchronize_mip_package.py b/tests/test_synchronize_mip_package.py index ef421ce..0ffe5bd 100644 --- a/tests/test_synchronize_mip_package.py +++ b/tests/test_synchronize_mip_package.py @@ -676,5 +676,65 @@ def test_no_file_means_no_requirements(self) -> None: self.assertEqual(build.desktop_requirements(Path(tmp)), []) +class TopLevelSourceTests(unittest.TestCase): + def test_packages_folder_then_usb_then_the_root(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "packages" / "midiports").mkdir(parents=True) + (root / "packages" / "usb" / "usb-core").mkdir(parents=True) + (root / "old").mkdir() + self.assertEqual(sync.top_level_source(root, "midiports"), root / "packages" / "midiports") + self.assertEqual(sync.top_level_source(root, "usb-core"), root / "packages" / "usb" / "usb-core") + self.assertEqual(sync.top_level_source(root, "old"), root / "old") + + +class PublishedNamesTests(unittest.TestCase): + def test_every_own_package_and_applib_the_stack_lists_has_a_description(self) -> None: + self.assertIn("medialink", metadata.PYDEVICES_DESCRIPTIONS) + self.assertIn("sayso", sync.SEVERAL_PACKAGES["applibs"]) + + +class DesktopBundleTests(unittest.TestCase): + def make_root(self, root: Path) -> None: + package = root / "packages" / "midiports" + (package / "midiports").mkdir(parents=True) + (package / "midiports" / "__init__.py").write_text("PORTS = 1\n", encoding="utf-8") + (package / "package.json").write_text( + '{"urls": [["midiports/__init__.py", "github:PyDevices/pydevices/packages/midiports/midiports/__init__.py"]]}', + encoding="utf-8", + ) + (root / "mip-split.toml").write_text( + "[midiports]\ntop-level = true\npypi-desktop = true\n[hmilink]\ntop-level = true\n", encoding="utf-8" + ) + + def test_only_marked_packages_are_bundled(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + self.make_root(root) + self.assertEqual(build.desktop_bundles(root), [root / "packages" / "midiports"]) + out = root / "out" + build.copy_bundle(root / "packages" / "midiports", root, out) + self.assertEqual((out / "midiports" / "__init__.py").read_text(encoding="utf-8"), "PORTS = 1\n") + + def test_the_mip_sync_accepts_the_key(self) -> None: + self.assertIn("pypi-desktop", sync.MIP_SPLIT_KEYS) + + def test_a_marked_package_without_package_json_stops_the_build(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "mip-split.toml").write_text("[midiports]\ntop-level = true\npypi-desktop = true\n", encoding="utf-8") + with self.assertRaises(SystemExit): + build.desktop_bundles(root) + + def test_an_unpinned_source_is_refused(self) -> None: + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + package = root / "p" + package.mkdir() + (package / "package.json").write_text('{"urls": [["a.py", "http://example.com/a.py"]]}', encoding="utf-8") + with self.assertRaises(SystemExit): + build.copy_bundle(package, root, root / "out") + + if __name__ == "__main__": unittest.main() From b0100a5d713ac53b812a405274492522c6e6a208 Mon Sep 17 00:00:00 2001 From: Brad Barnett <127794626+bdbarnett@users.noreply.github.com> Date: Sat, 7 Nov 2026 23:46:46 -0600 Subject: [PATCH 2/2] Stack data: leave the midiports row to land with the pydevices change --- data/stack.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/stack.json b/data/stack.json index 4dc92d6..a7d752f 100644 --- a/data/stack.json +++ b/data/stack.json @@ -142,7 +142,7 @@ {"name": "picotts", "group": "sound", "does": "Make a board with PSRAM speak, with no network and no computer.", "repo": "micropython-pydevices", "path": "modules/picotts", "get": ["firmware"], "runs": "MP"}, {"name": "mpvst", "group": "sound", "does": "Run Python instruments and effects in your DAW as a VST3 plug-in.", "repo": "mpvst", "path": "README.md", "get": ["download"], "runs": "Desktop DAW"}, - {"name": "midiports", "group": "midi", "does": "Connect `adafruit_midi` to USB, UART, Web MIDI or a computer's own MIDI ports.", "repo": "pydevices", "path": "packages/midiports", "get": ["mip:midiports", "pip:pydevices-desktop"], "runs": "MP, Desktop, Web"}, + {"name": "midiports", "group": "midi", "does": "Connect `adafruit_midi` to USB, UART, Web MIDI or a computer's own MIDI ports.", "repo": "pydevices", "path": "packages/midiports", "get": ["mip:midiports"], "runs": "MP, Desktop, Web"}, {"name": "soundmodule", "group": "midi", "does": "Play MIDI through a sound engine, headless from a port or through files.", "repo": "apps", "path": "lib/soundmodule", "guide": "docs/guides/play-the-sound-module.md", "get": ["mip:soundmodule", "pip:omnipython-applibs"], "runs": "MP, Desktop"}, {"name": "songmodel", "group": "midi", "does": "Edit a groovebox's song as plain data, and carry it to a front panel.", "repo": "apps", "path": "lib/songmodel", "get": ["mip:songmodel", "pip:omnipython-applibs"], "runs": "MP, Desktop, Web"},