MIP and PyPI publishing: medialink, sayso and midiports reach users

docs/how-it-works/release-chain.md

@@ -113,6 +113,11 @@ The `pydevices` build keeps no inclusion list. The distribution is exactly
   single source for its description, `require()` lines and files. The sync
   fails if a listed name isn't a directory with a `manifest.py`. This is
   available from `publishing-v14`.
+- **A top-level package can also ride in `pydevices-desktop`.** Adding
+  `pypi-desktop = true` to its section copies the files its `package.json`
+  lists into the wheel, fetching any pinned upstream URL (midiports carries
+  `adafruit_midi` this way), so `pip install pydevices-desktop` gives what
+  `mip` gives. The wheel build reads the key; the MIP sync ignores it.
 - Every publishable entry under `utils/`, and everything publishable in
   `board_configs/desktop/`, is bundled into `pydevices-desktop`, which depends
   on `pydevices`. All artifacts from one release share a version.

scripts/build_pydevices_python_distributions.py

@@ -8,10 +8,12 @@
 from __future__ import annotations
 
 import argparse
+import json
 import shutil
 import subprocess
 import sys
 import tomllib
+import urllib.request
 from pathlib import Path
 
 from pydevices_package_metadata import PYDEVICES_DESCRIPTIONS
@@ -91,6 +93,51 @@ def desktop_requirements(root: Path) -> list[str]:
     return requirements
 
 
+def desktop_bundles(root: Path) -> list[Path]:
+    """Packages that pydevices-desktop carries beyond utils/: the directories
+    mip-split.toml marks ``pypi-desktop = true`` (midiports, for pip users who
+    have no mip). Each is found at packages/<name>, or packages/usb/<name>."""
+    split_path = root / "mip-split.toml"
+    if not split_path.exists():
+        return []
+    with split_path.open("rb") as handle:
+        declared = tomllib.load(handle)
+    found = []
+    for name, section in declared.items():
+        if not section.get("pypi-desktop"):
+            continue
+        for candidate in (root / "packages" / name, root / "packages" / "usb" / name):
+            if (candidate / "package.json").is_file():
+                found.append(candidate)
+                break
+        else:
+            raise SystemExit(f"mip-split.toml: [{name}] is pypi-desktop but packages/{name}/package.json is missing")
+    return found
+
+
+GITHUB_SELF = "github:PyDevices/pydevices/"
+
+
+def copy_bundle(package: Path, root: Path, destination: Path) -> None:
+    """Lay out one package's files from its package.json, the list mip installs.
+
+    A file of this repository is copied from the checkout. Any other file is
+    fetched from its pinned URL (midiports pins Adafruit's adafruit_midi at a
+    tag), so pip users get exactly what mip installs.
+    """
+    urls = json.loads((package / "package.json").read_text(encoding="utf-8"))["urls"]
+    for relative, source in urls:
+        target = destination / relative
+        target.parent.mkdir(parents=True, exist_ok=True)
+        if source.startswith(GITHUB_SELF):
+            shutil.copy2(root / source[len(GITHUB_SELF) :], target)
+        elif source.startswith("https://"):
+            with urllib.request.urlopen(source, timeout=60) as response:
+                target.write_bytes(response.read())
+        else:
+            raise SystemExit(f"{package.name}/package.json: cannot bundle {source!r}")
+
+
 def project_text(
     name: str,
     version: str,
@@ -190,6 +237,8 @@ def build(root: Path, output: Path, version: str) -> None:
         copy_component(source, desktop_source / source.name)
     for source in components(root / DESKTOP_DIR):
         copy_component(source, desktop_source / source.name)
+    for bundle in desktop_bundles(root):
+        copy_bundle(bundle, root, desktop_source)
     write_project(
         desktop,
         "pydevices-desktop",

scripts/pydevices_package_metadata.py

@@ -9,4 +9,6 @@
     "pydevices-desktop": "Complete PyDevices desktop stack and board configuration",
     # MIP only: on PyPI bledev is inside the pydevices wheel.
     "bledev": "Portable async Bluetooth Low Energy for PyDevices, over aioble on boards",
+    # MIP only, like bledev: on PyPI medialink is inside the pydevices wheel.
+    "medialink": "Live audio and video between boards and computers: a walkie-talkie, an intercom, a baby monitor",
 }

scripts/synchronize_mip_package.py

@@ -93,6 +93,7 @@ class Profile:
     "applibs": {
         "appkit": "Apps you can switch between, and a launcher to run them",
         "controlkit": "LVGL controls: encoders, sliders, meters, lamp buttons, pad grids and a dot-matrix display",
+        "sayso": "Lightweight Spotify Web API client for CPython, MicroPython and CircuitPython",
         "songmodel": "The song a groovebox edits, and the link that carries it to a front panel",
         "soundmodule": "A sound module that plays MIDI, headless from a port or through files",
         "spectrum": "A spectrum analyzer in an app's area, sharing the panel with LVGL",
@@ -262,7 +263,7 @@ def render_pydevices_manifest(name: str, version: str, requirements: tuple[str,
 
 #: Every key a mip-split.toml section may use. An unknown key fails the sync,
 #: so a misspelt `own-package` can't quietly ship a package inside pydevices.
-MIP_SPLIT_KEYS = frozenset({"host-only", "own-package", "requires", "pypi-extras", "top-level"})
+MIP_SPLIT_KEYS = frozenset({"host-only", "own-package", "requires", "pypi-extras", "pypi-desktop", "top-level"})
 
 
 def read_split(source_root: Path) -> dict[str, dict]:
@@ -371,6 +372,15 @@ def check_no_host_imports(package_dir: Path, host_only: frozenset[str]) -> None:
                 )
 
 
+def top_level_source(source_root: Path, package: str) -> Path:
+    """Where a top-level package lives: packages/<name>, packages/usb/<name>,
+    or (older releases) a directory at the repository root."""
+    for candidate in (source_root / "packages" / package, source_root / "packages" / "usb" / package):
+        if candidate.is_dir():
+            return candidate
+    return source_root / package
+
+
 def read_top_level_packages(source_root: Path) -> list[str]:
     """Names of the packages that live at the repository root, not in lib/.
 
@@ -385,15 +395,15 @@ def read_top_level_packages(source_root: Path) -> list[str]:
     for package, section in read_split(source_root).items():
         if not section.get("top-level"):
             continue
-        extra = sorted(set(section) - {"top-level"})
+        extra = sorted(set(section) - {"top-level", "pypi-desktop"})
         if extra:
             raise SystemExit(
                 f"{MIP_SPLIT_FILE} [{package}] is top-level, so it takes no other keys "
                 f"(remove {', '.join(extra)}). Its manifest.py says what it requires and ships."
             )
-        package_dir = source_root / package
+        package_dir = top_level_source(source_root, package)
         if not package_dir.is_dir():
-            raise SystemExit(f"{MIP_SPLIT_FILE} names top-level package {package!r}, which is not a directory at the repository root")
+            raise SystemExit(f"{MIP_SPLIT_FILE} names top-level package {package!r}, which is not a directory under packages/ or at the repository root")
         if not (package_dir / "manifest.py").is_file():
             raise SystemExit(f"top-level package {package!r} has no manifest.py")
         if (source_root / "lib" / package).exists():
@@ -403,9 +413,9 @@ def read_top_level_packages(source_root: Path) -> list[str]:
     # install, then fail on import. Say so here, in the pull request's own
     # terms, instead of in the index build.
     for package in names:
-        manifest = (source_root / package / "manifest.py").read_text(encoding="utf-8")
+        manifest = (top_level_source(source_root, package) / "manifest.py").read_text(encoding="utf-8")
         for required in re.findall(r'^\s*require\(\s*["\']([^"\']+)["\']', manifest, re.MULTILINE):
-            if required not in names and (source_root / required / "manifest.py").is_file():
+            if required not in names and (top_level_source(source_root, required) / "manifest.py").is_file():
                 raise SystemExit(
                     f"top-level package {package!r} requires {required!r}, which is a directory "
                     f"here but is not listed in {MIP_SPLIT_FILE}; add [{required}] with top-level = true"
@@ -430,7 +440,7 @@ def stamp_version(manifest: str, package: str, version: str) -> str:
 
 
 def synchronize_top_level(source_root: Path, destination_root: Path, package: str, version: str) -> None:
-    source = source_root / package
+    source = top_level_source(source_root, package)
     destination = destination_root / package
     destination.mkdir()
     for path in sorted(source.iterdir()):

tests/test_synchronize_mip_package.py

@@ -676,5 +676,65 @@ def test_no_file_means_no_requirements(self) -> None:
             self.assertEqual(build.desktop_requirements(Path(tmp)), [])
 
 
+class TopLevelSourceTests(unittest.TestCase):
+    def test_packages_folder_then_usb_then_the_root(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            (root / "packages" / "midiports").mkdir(parents=True)
+            (root / "packages" / "usb" / "usb-core").mkdir(parents=True)
+            (root / "old").mkdir()
+            self.assertEqual(sync.top_level_source(root, "midiports"), root / "packages" / "midiports")
+            self.assertEqual(sync.top_level_source(root, "usb-core"), root / "packages" / "usb" / "usb-core")
+            self.assertEqual(sync.top_level_source(root, "old"), root / "old")
+
+
+class PublishedNamesTests(unittest.TestCase):
+    def test_every_own_package_and_applib_the_stack_lists_has_a_description(self) -> None:
+        self.assertIn("medialink", metadata.PYDEVICES_DESCRIPTIONS)
+        self.assertIn("sayso", sync.SEVERAL_PACKAGES["applibs"])
+
+
+class DesktopBundleTests(unittest.TestCase):
+    def make_root(self, root: Path) -> None:
+        package = root / "packages" / "midiports"
+        (package / "midiports").mkdir(parents=True)
+        (package / "midiports" / "__init__.py").write_text("PORTS = 1\n", encoding="utf-8")
+        (package / "package.json").write_text(
+            '{"urls": [["midiports/__init__.py", "github:PyDevices/pydevices/packages/midiports/midiports/__init__.py"]]}',
+            encoding="utf-8",
+        )
+        (root / "mip-split.toml").write_text(
+            "[midiports]\ntop-level = true\npypi-desktop = true\n[hmilink]\ntop-level = true\n", encoding="utf-8"
+        )
+
+    def test_only_marked_packages_are_bundled(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            self.make_root(root)
+            self.assertEqual(build.desktop_bundles(root), [root / "packages" / "midiports"])
+            out = root / "out"
+            build.copy_bundle(root / "packages" / "midiports", root, out)
+            self.assertEqual((out / "midiports" / "__init__.py").read_text(encoding="utf-8"), "PORTS = 1\n")
+
+    def test_the_mip_sync_accepts_the_key(self) -> None:
+        self.assertIn("pypi-desktop", sync.MIP_SPLIT_KEYS)
+
+    def test_a_marked_package_without_package_json_stops_the_build(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            (root / "mip-split.toml").write_text("[midiports]\ntop-level = true\npypi-desktop = true\n", encoding="utf-8")
+            with self.assertRaises(SystemExit):
+                build.desktop_bundles(root)
+
+    def test_an_unpinned_source_is_refused(self) -> None:
+        with tempfile.TemporaryDirectory() as tmp:
+            root = Path(tmp)
+            package = root / "p"
+            package.mkdir()
+            (package / "package.json").write_text('{"urls": [["a.py", "http://example.com/a.py"]]}', encoding="utf-8")
+            with self.assertRaises(SystemExit):
+                build.copy_bundle(package, root, root / "out")
+
+
 if __name__ == "__main__":
     unittest.main()