feat: lifecycle coverage matrix + parity.json witness convention (#442 PR 6)

.github/workflows/lint.yml

@@ -107,6 +107,18 @@ jobs:
         working-directory: autolens_profiling
         run: python scripts/misc/tooling/build_catalogue.py --check --validate-with ../PyAutoPulse
 
+      - name: parity_witness.py --check (parity.json convention)
+        working-directory: autolens_profiling
+        run: python scripts/misc/tooling/parity_witness.py --check
+
+      - name: coverage_matrix.py --check (lifecycle coverage drift)
+        working-directory: autolens_profiling
+        run: |
+          # dashboard/coverage.{json,md} are rendered from dashboard/catalogue.json and
+          # catalogue/provisional_designations.json; a catalogue change without a re-render,
+          # or a planned class that disagrees with the §3.3 copy, fails here. Pure stdlib.
+          python scripts/misc/tooling/coverage_matrix.py --check
+
       - name: check_wiki.py --check (research wiki drift)
         working-directory: autolens_profiling
         run: |
@@ -150,7 +162,7 @@ jobs:
           # that adds it. Pure stdlib, read-only, deterministic (~3 s).
           python scripts/misc/tooling/list_timing_assertions.py --check
 
-      - name: pytest — scripts/misc/test/
+      - name: pytest — scripts/misc/*/test/
         working-directory: autolens_profiling
         env:
           MPLCONFIGDIR: /tmp/matplotlib
@@ -161,7 +173,8 @@ jobs:
           # warm-compile pins on a PR. They are fast (~1s) and need only the
           # dependency chain this job already installs -- aggregate.py imports
           # matplotlib at module scope, so a bare checkout cannot run them.
-          pytest scripts/misc/test/ -q
+          # Each scripts/misc group keeps its tests in its own test/ dir (#442 PR 5).
+          pytest scripts/misc/*/test -q
 
       - name: Set up Node for browser checks
         uses: actions/setup-node@v4
@@ -176,7 +189,7 @@ jobs:
         run: |
           npm install --prefix "$RUNNER_TEMP/profiling-browser" playwright@1.63.0
           "$RUNNER_TEMP/profiling-browser/node_modules/.bin/playwright" install --with-deps chromium
-          node scripts/misc/test/browser_setup_page.cjs
+          node scripts/misc/tooling/test/browser_setup_page.cjs
 
       - name: lychee — markdown link-rot
         working-directory: autolens_profiling

.gitignore

@@ -116,3 +116,6 @@ dataset/interferometer/*/preload_inputs_*.npz
 results/lens/solver/corpus/delaunay_hst.npz
 results/lens/solver/corpus/rectangular_hst.npz
 results/lens/solver/corpus/slam_mixed_hst.npz
+
+# pytest cache (rootdir is the repo root, see pytest.ini)
+.pytest_cache/

AGENTS.md

@@ -31,8 +31,9 @@ scripts/
                         misc/likelihood_runtime/ (sweep.py + aggregate.py + README dashboard),
                         misc/vram/ (A100 vmap batch-size table), misc/simulators/,
                         misc/latent/, misc/jax_compile/, misc/wall/,
-                        misc/pipeline_resume/, misc/test/, misc/tooling/ (build_readme.py +
-                        build_baseline.py)
+                        misc/pipeline_resume/, misc/tooling/ (build_readme.py +
+                        build_baseline.py). Each group keeps its tests in misc/<group>/test/
+                        (run all: pytest scripts/misc/*/test -q)
 _profile_cli.py         Shared CLI/JSON/auto-simulate helper imported by every per-cell script
 _adapt_image_util.py    Shared adapt-image helper
 instruments/            Instrument definitions (pixel scale, shape) used to frame results

_profile_cli.py

@@ -45,6 +45,9 @@ class ProfileCLI:
     sparse_batch_size: int
     source_pixels: int | None
     over_sampling: str = "flat1"
+    #: ``--output-path``: the exact result JSON a driver (``likelihood_runtime/sweep.py``) has
+    #: already placed; it wins over every other naming rule.
+    output_path: Path | None = None
 
     def parse_cell_args(self, cell_parser: argparse.ArgumentParser, *, argv=None):
         """Finish staged parsing, rejecting flags neither parser understands.
@@ -112,6 +115,14 @@ def _profile_parser() -> argparse.ArgumentParser:
             "package's section under <autolens_profiling>/results/."
         ),
     )
+    parser.add_argument(
+        "--output-path",
+        default=None,
+        help=(
+            "Exact result JSON path (the PNG goes beside it). Set by drivers that place the "
+            "row themselves (likelihood_runtime/sweep.py); overrides --output-dir and naming."
+        ),
+    )
     parser.add_argument(
         "--use-mixed-precision",
         action="store_true",
@@ -351,6 +362,7 @@ def parse_profile_cli(default_config_name: str | None = None, *, argv=None) -> P
         sparse_batch_size=int(args.sparse_batch_size),
         source_pixels=int(args.source_pixels) if args.source_pixels is not None else None,
         over_sampling=args.over_sampling,
+        output_path=Path(args.output_path).resolve() if args.output_path else None,
     )
 
 
@@ -772,6 +784,40 @@ def _in_legacy_results(path: Path) -> bool:
     return rel.parts[:2] in _LEGACY_RESULT_DIRS
 
 
+def _row_instrument(cli: ProfileCLI, default_basename: str) -> str | None:
+    """``--instrument``, else the instrument token of ``default_basename``."""
+    if getattr(cli, "instrument", None):
+        return cli.instrument
+    tokens = f"_{default_basename}_"
+    return next((i for i in _migration_map_module().INSTRUMENTS if f"_{i}_" in tokens), None)
+
+
+def runtime_instrument_dir(legacy_dir: Path, instrument: str | None) -> Path:
+    """``results/runtime/<class>/<model>`` -> ``.../<model>/<instrument>`` (any other path as is).
+
+    The sweep layout names every runtime row under its instrument; a two-level legacy runtime
+    folder would make a config-named row (``delaunay_local_cpu_fp64.json``) of every instrument
+    the same name, so a row whose cell cannot be derived would collide in the landing folder.
+    """
+    try:
+        rel = Path(legacy_dir).resolve().relative_to(_REPO_ROOT).parts
+    except ValueError:
+        return Path(legacy_dir)
+    if instrument and len(rel) == 4 and rel[:2] == ("results", "runtime"):
+        return Path(legacy_dir) / instrument
+    return Path(legacy_dir)
+
+
+def placed_path(legacy_path: Path, doc: dict) -> Path:
+    """:func:`mirrored_output_path` without creating or announcing anything (for readers)."""
+    legacy_path = Path(legacy_path)
+    if not _in_legacy_results(legacy_path):
+        return legacy_path
+    rel = legacy_path.resolve().relative_to(_REPO_ROOT)
+    new, _kind = _migration_map_module().destination(_REPO_ROOT, rel.as_posix(), doc)
+    return _REPO_ROOT / new
+
+
 def place_output_path(
     json_path: Path, cli: ProfileCLI, default_basename: str, hints: dict | None = None
 ) -> Path:
@@ -812,6 +858,157 @@ def mirrored_output_path(legacy_path: Path, doc: dict) -> Path:
     return _REPO_ROOT / new
 
 
+def _migration_map_module():
+    """``scripts/misc/tooling/build_migration_map`` (stdlib-only), imported on first use."""
+    import sys
+
+    tooling = str(_REPO_ROOT / "scripts" / "misc" / "tooling")
+    if tooling not in sys.path:
+        sys.path.insert(0, tooling)
+    import build_migration_map
+
+    return build_migration_map
+
+
+def _legacy_rel(path: Path) -> tuple[str, ...] | None:
+    """``("results", <old group>, ...)`` for a path under a pre-#442 results folder, else None."""
+    try:
+        rel = Path(path).resolve().relative_to(_REPO_ROOT).parts
+    except ValueError:
+        return None
+    mm = _migration_map_module()
+    if len(rel) < 2 or rel[0] != "results" or (rel[1] not in mm.CATEGORY | mm.MISC_GROUPS):
+        return None
+    return rel
+
+
+def legacy_landing_dir(legacy_dir: Path) -> Path:
+    """The folder where a file a writer names under the legacy ``legacy_dir`` keeps its name.
+
+    ``results/runtime/<rest>`` -> ``results/misc/likelihood_runtime/<rest>``,
+    ``results/breakdown/<rest>`` -> ``results/misc/likelihood_breakdown/<rest>`` (where
+    :func:`mirrored_output_path` puts a row with no derivable cell) and
+    ``results/<group>/<rest>`` -> ``results/misc/<MISC_GROUPS[group]>/<rest>`` for the other
+    pre-#442 folders (``build_migration_map.MISC_GROUPS``). Any other path is returned unchanged,
+    so an explicit ``--output-dir`` passes through.
+    """
+    rel = _legacy_rel(legacy_dir)
+    if rel is None:
+        return Path(legacy_dir)
+    mm = _migration_map_module()
+    group = mm.CATEGORY.get(rel[1]) or mm.MISC_GROUPS[rel[1]]
+    return _REPO_ROOT.joinpath("results", "misc", group, *rel[2:])
+
+
+def legacy_read_dirs(legacy_dir: Path) -> list[Path]:
+    """Folders a reader globs for files a writer names under ``legacy_dir``, by that name.
+
+    The landing folder (:func:`legacy_landing_dir`) first, then ``legacy_dir`` itself (the
+    unresolved rows the day-zero move left in place, and runs given that folder verbatim as
+    ``--output-dir``). A path outside the legacy folders is its own only folder.
+    """
+    landing = legacy_landing_dir(legacy_dir)
+    return [landing] if landing == Path(legacy_dir) else [landing, Path(legacy_dir)]
+
+
+def _day_zero_moves() -> dict[str, str]:
+    """{old repo-relative path: new path} of ``catalogue/migration_day_zero.json`` ({} if absent)."""
+    import json
+
+    global _DAY_ZERO_MOVES
+    if _DAY_ZERO_MOVES is None:
+        try:
+            doc = json.loads((_REPO_ROOT / "catalogue" / "migration_day_zero.json").read_text())
+            entries = doc.get("entries", [])
+        except (OSError, ValueError):
+            entries = []
+        _DAY_ZERO_MOVES = {e["old"]: e["new"] for e in entries if e.get("new") != e.get("old")}
+    return _DAY_ZERO_MOVES
+
+
+_DAY_ZERO_MOVES: dict[str, str] | None = None
+
+
+def locate_legacy_result(legacy_path: Path, doc: dict | None = None) -> Path | None:
+    """Where the file a writer names ``legacy_path`` lives now, or None when it is nowhere.
+
+    Tried in order: ``legacy_path`` itself (an unresolved row left in place, or an explicit
+    ``--output-dir`` run), the day-zero map's destination (a committed row), the landing folder
+    (a placed row with no derivable cell keeps its name), and -- for a ``results/runtime`` /
+    ``results/breakdown`` name -- the slot :func:`mirrored_output_path` gives it for ``doc``
+    (the setup facts the reader knows, e.g. ``{"instrument": "hst"}``; a row written after the
+    move with those facts) unless the map put a committed row of another legacy name there.
+    Readers that parse legacy file names use this instead of joining
+    the legacy folder.
+    """
+    legacy_path = Path(legacy_path)
+    candidates = [legacy_path]
+    rel = _legacy_rel(legacy_path)
+    if rel is not None:
+        moved = _day_zero_moves().get("/".join(rel))
+        if moved:
+            candidates.append(_REPO_ROOT / moved)
+        candidates.append(legacy_landing_dir(legacy_path.parent) / legacy_path.name)
+        if rel[1] in ("runtime", "breakdown"):
+            new, _kind = _migration_map_module().destination(_REPO_ROOT, "/".join(rel), doc or {})
+            # A committed row the map moved there belongs to its own legacy name, not this one.
+            claimed = {o for o, n in _day_zero_moves().items() if n == new}
+            if not claimed or claimed == {"/".join(rel)}:
+                candidates.append(_REPO_ROOT / new)
+    return next((p for p in candidates if p.exists()), None)
+
+
+def legacy_result_files(legacy_dir: Path, pattern: str = "*.json") -> list[tuple[Path, str]]:
+    """``(current path, legacy file name)`` of every file once or now named under ``legacy_dir``.
+
+    The legacy-name equivalent of ``sorted(legacy_dir.glob(pattern))`` (``pattern`` matched on
+    the legacy name, sorted by it): committed rows the day-zero map moved (renamed into a
+    mirrored cell, archived, or kept by name), rows in the landing folder, and rows still in
+    ``legacy_dir``. A later source wins a name clash. A row a run placed into a *new* mirrored cell after the move has
+    no legacy name on disk; find it with :func:`locate_legacy_result`.
+    """
+    import fnmatch
+
+    found: dict[str, Path] = {}
+    rel = _legacy_rel(legacy_dir)
+    if rel is not None:
+        prefix = "/".join(rel) + "/"
+        for old, new in _day_zero_moves().items():
+            name = old[len(prefix) :]
+            if old.startswith(prefix) and "/" not in name and fnmatch.fnmatch(name, pattern):
+                if (_REPO_ROOT / new).is_file():
+                    found[name] = _REPO_ROOT / new
+    for folder in legacy_read_dirs(legacy_dir):
+        if folder.is_dir():
+            for path in folder.glob(pattern):
+                if path.is_file():
+                    found[path.name] = path
+    return sorted(((p, n) for n, p in found.items()), key=lambda t: t[1])
+
+
+def legacy_result_dirs(legacy_root: Path) -> list[Path]:
+    """Every legacy folder under ``legacy_root`` that names a file, as legacy paths, sorted.
+
+    The folders of the committed rows the day-zero map moved, plus the folders on disk under
+    the landing folder and under ``legacy_root`` itself (mapped back onto ``legacy_root``).
+    Pair with :func:`legacy_result_files` to walk a legacy tree after the move.
+    """
+    legacy_root = Path(legacy_root)
+    dirs: set[Path] = set()
+    rel = _legacy_rel(legacy_root)
+    if rel is not None:
+        prefix = "/".join(rel) + "/"
+        for old in _day_zero_moves():
+            if old.startswith(prefix) and "/" in old[len(prefix) :]:
+                dirs.add(legacy_root / old[len(prefix) :].rsplit("/", 1)[0])
+    for base in legacy_read_dirs(legacy_root):
+        if base.is_dir():
+            for path in base.rglob("*"):
+                if path.is_file() and path.parent != base:
+                    dirs.add(legacy_root / path.parent.relative_to(base))
+    return sorted(dirs)
+
+
 def resolve_output_paths(
     cli: ProfileCLI,
     default_dir: Path,
@@ -860,7 +1057,12 @@ def resolve_output_paths(
       label (``--config-name hpc_a100_fp64_n3000``) and gets no extra suffix —
       the label already says the size.
     """
+    if getattr(cli, "output_path", None) is not None:
+        cli.output_path.parent.mkdir(parents=True, exist_ok=True)
+        return cli.output_path, cli.output_path.with_suffix(".png")
     results_dir = cli.output_dir if cli.output_dir is not None else default_dir
+    if cli.output_dir is None and _in_legacy_results(results_dir):
+        results_dir = runtime_instrument_dir(results_dir, _row_instrument(cli, default_basename))
     if cli.config_name is None:
         basename = default_basename
     else:

catalogue/README.md

@@ -31,6 +31,13 @@ The rules live in [`catalogue_lifecycle.py`](../scripts/misc/tooling/catalogue_l
 - **Class**: the provisional §3.3 designations. A provisional Optimal row is `class: secondary`
   with `provisional_optimal: true` in `limitations` until a Pulse designation decision exists
   (R3); mixed precision is `experimental`. `planned_cells` carry the provisional class.
+  [provisional_designations.json](provisional_designations.json) transcribes the §3.3 table
+  row by row with its machine reading; a test keeps it and `catalogue_lifecycle.py` in step.
+- **Parity**: a cell's `parity.json` witness becomes a `measurement: parity` record; see
+  [results/README.md](../results/README.md#parity-witness).
+- **Coverage**: `coverage_matrix.py` renders [`dashboard/coverage.md`](../dashboard/coverage.md)
+  (current / stale / superseded / missing per likelihood x class x measurement; a cell
+  covered only by a `size_fallback` row is missing). Re-run it after `build_catalogue.py`.
 - **measured_at**: a row with no recorded timestamp is dated by the UTC day its file was first
   committed to git (an upper bound on the measurement date), from
   [measured_at_first_commit.json](measured_at_first_commit.json), and says so in
@@ -45,6 +52,7 @@ From the repository root (Python standard library only):
 python scripts/misc/tooling/build_catalogue.py
 python scripts/misc/tooling/build_catalogue.py --check
 python scripts/misc/tooling/build_catalogue.py --check --validate-with ../PyAutoPulse
+python scripts/misc/tooling/coverage_matrix.py        # then --check
 ```
 
 `build_dashboard.py` also builds/checks the catalogue and the page bound to its hash.