From daa131d9a81e65e47d544476fa98d801a49304d4 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Wed, 21 Oct 2026 18:50:23 +0100 Subject: [PATCH 1/5] pulse: profiling-summary@3 lifecycle contract (class, status, one current per key) Records declare likelihood, setup tuple + canonical setup_id, host_class, model_variant, measurement, class and status. The reader refuses a second current record per key, class optimal without a decision_id, current rows on local_laptop/local_gpu and archived rows inside records[]. planned_cells carry class. v2 validates unchanged; hardware()/group_label() report host_class as provenance beside the identity. Co-Authored-By: Claude Fable 5.1 --- pulse/catalogue.py | 305 ++++++++++- pulse/summary.py | 38 +- tests/fixtures/setup_summary_v3.json | 511 ++++++++++++++++++ tests/fixtures/v3_archived_in_records.json | 510 +++++++++++++++++ tests/fixtures/v3_duplicate_current.json | 509 +++++++++++++++++ tests/fixtures/v3_laptop_current.json | 511 ++++++++++++++++++ .../fixtures/v3_optimal_without_decision.json | 511 ++++++++++++++++++ tests/test_catalogue.py | 149 ++++- 8 files changed, 3024 insertions(+), 20 deletions(-) create mode 100644 tests/fixtures/setup_summary_v3.json create mode 100644 tests/fixtures/v3_archived_in_records.json create mode 100644 tests/fixtures/v3_duplicate_current.json create mode 100644 tests/fixtures/v3_laptop_current.json create mode 100644 tests/fixtures/v3_optimal_without_decision.json diff --git a/pulse/catalogue.py b/pulse/catalogue.py index 82ddada..fa6a53f 100644 --- a/pulse/catalogue.py +++ b/pulse/catalogue.py @@ -1,15 +1,20 @@ -"""Validate the setup-oriented v2 exchange contract without judging evidence. - -Stable setup IDs decouple scientific navigation from script paths. Selections -name exact records and their measurement context; missing cells remain explicit. -This module is stdlib-only and does not read files, choose results or compute -comparisons. See REFERENCE.md for the producer-facing grammar. +"""Validate the setup-oriented v2 and lifecycle v3 contracts without judging evidence. + +Stable setup IDs decouple scientific navigation from script paths. In v2, +selections name exact records and their measurement context; missing cells +remain explicit. In v3 every record declares its own likelihood, setup tuple, +host class, ``class`` (optimal/secondary/experimental) and ``status`` +(current/superseded); the producer declares exactly one ``current`` record per +key and Pulse only checks that the declaration is consistent. This module is +stdlib-only and does not read files, choose results, infer an optimal setup or +compute comparisons. See REFERENCE.md for the producer-facing grammar. """ from __future__ import annotations import json import math +import re from pulse import summary @@ -17,6 +22,58 @@ VALIDATION_STATES = {"accepted", "unreviewed", "rejected"} MEMORY_METRICS = {"host_peak_rss", "device_peak_allocated", "device_peak_reserved", "device_total"} +# ---- profiling-summary@3 vocabulary (REFERENCE.md, "profiling-summary v3") ---- +DATASET_TYPES = { + "imaging", + "interferometer", + "datacube", + "point_source_image", + "point_source_source", + "multi_galaxy", + "group", + "cluster", + "multi_dataset", + "weak", +} +HOST_CLASSES = set(summary.HOST_CLASSES) +LOCAL_HOST_CLASSES = set(summary.LOCAL_HOST_CLASSES) +_SECONDS = {"s"} +_MEMORY = summary.AXIS_UNITS["memory"] +MEASUREMENT_UNITS = { + "runtime": _SECONDS, + "breakdown": _SECONDS, + "gradient": _SECONDS, + "compile": _SECONDS, + "memory_host": _MEMORY, + "memory_device": _MEMORY, + "parity": {"dimensionless"}, + "walltime_validation": _SECONDS, +} +CLASSES = {"optimal", "secondary", "experimental"} +STATUSES = {"current", "superseded", "archived"} +SETUP_CHOICES = { + "backend": {"numba", "jax"}, + "device": {"cpu", "gpu"}, + "operator": {"dense", "sparse", "na"}, + "precision": {"fp64", "mixed"}, + "transformer": {"dft", "nufft", "wtilde", "streaming", "na"}, +} +KEY_FIELDS = ("likelihood", "instrument", "setup_id", "host_class", "model_variant", "measurement") +_SHA256 = re.compile(r"^[0-9a-f]{64}$") + + +def setup_id(setup: dict) -> str: + """The canonical ``___[_]``.""" + parts = [str(setup.get(k)) for k in ("backend", "device", "operator", "precision")] + if setup.get("transformer") != "na": + parts.append(str(setup.get("transformer"))) + return "_".join(parts) + + +def lifecycle_key(item: dict) -> str: + """The one-``current``-per-key identity of a v3 record or planned cell.""" + return json.dumps({k: item.get(k) for k in KEY_FIELDS}, sort_keys=True) + class _Check: def __init__(self): @@ -120,6 +177,10 @@ def _finite(value): return False +def _list(value): + return value if isinstance(value, list) else [] + + def _integer(value): return isinstance(value, int) and not isinstance(value, bool) and value >= 0 @@ -202,6 +263,11 @@ def _record(check, item, setups): ) if axis == "memory": check.choice(metric, MEMORY_METRICS, f"{where}.metric") + _record_body(check, item, where) + + +def _record_body(check, item, where): + """Identity, method, provenance, validation and evidence: shared by v2 and v3.""" _identity(check, item.get("identity"), f"{where}.identity") method = item.get("method") if not isinstance(method, dict): @@ -238,6 +304,216 @@ def _record(check, item, setups): check.evidence(item.get("evidence"), f"{where}.evidence") +def _likelihood(check, value, where): + if not isinstance(value, dict): + check.errors.append(f"{where}: must be a likelihood object {{dataset_type, variant}}") + return + check.choice(value.get("dataset_type"), DATASET_TYPES, f"{where}.dataset_type") + check.text(value.get("variant"), f"{where}.variant") + + +def _setup_tuple(check, item, where): + """The compute tuple and its canonical ``setup_id`` (R1 grammar).""" + setup = item.get("setup") + if not isinstance(setup, dict): + check.errors.append(f"{where}.setup: must be an object") + check.text(item.get("setup_id"), f"{where}.setup_id") + return + for key, choices in SETUP_CHOICES.items(): + check.choice(setup.get(key), choices, f"{where}.setup.{key}") + threads = setup.get("threads") + check.require( + threads is None or (_integer(threads) and threads > 0), + f"{where}.setup.threads", + "must be a positive integer or null", + ) + if all(isinstance(setup.get(k), str) and setup.get(k) in c for k, c in SETUP_CHOICES.items()): + expected = setup_id(setup) + check.require( + item.get("setup_id") == expected, + f"{where}.setup_id", + f"must be {expected!r} (___[_])", + ) + + +def _cell_key_fields(check, item, where): + """The key fields shared by v3 records and planned cells.""" + _likelihood(check, item.get("likelihood"), f"{where}.likelihood") + check.text(item.get("instrument"), f"{where}.instrument") + check.text(item.get("setup_id"), f"{where}.setup_id") + check.choice(item.get("host_class"), HOST_CLASSES, f"{where}.host_class") + check.text(item.get("model_variant"), f"{where}.model_variant") + check.choice(item.get("measurement"), MEASUREMENT_UNITS, f"{where}.measurement") + check.choice(item.get("class"), CLASSES, f"{where}.class") + + +def _nullable_text(check, item, key, where): + check.require(key in item, where, f"missing {key} (null when not applicable)") + value = item.get(key) + check.require(value is None or summary._text(value), f"{where}.{key}", "string or null") + + +def _record_v3(check, item, ids): + where = f"record {item['id']!r}" + _cell_key_fields(check, item, where) + _setup_tuple(check, item, where) + check.text(item.get("regime"), f"{where}.regime") + check.text(item.get("run_id"), f"{where}.run_id") + metric = item.get("metric") + check.text(metric, f"{where}.metric") + measurement = item.get("measurement") + if isinstance(measurement, str) and measurement in MEASUREMENT_UNITS: + check.choice(item.get("unit"), MEASUREMENT_UNITS[measurement], f"{where}.unit") + values = item.get("values") + check.require( + isinstance(values, dict) and isinstance(metric, str) and metric in values, + where, + "values must be a non-empty object containing its headline metric", + ) + if isinstance(values, dict): + for value in values.values(): + check.require( + _finite(value) and value >= 0, + where, + "values must be finite and non-negative; missing cells use planned_cells", + ) + status = item.get("status") + check.choice(status, STATUSES, f"{where}.status") + check.require( + status != "archived", + where, + "archived records are not rendered; drop them from records[] and count them in " + "archive_record_count", + ) + if status == "current": + check.require( + item.get("host_class") not in summary.LOCAL_HOST_CLASSES, + where, + f"host_class {item.get('host_class')!r} is never current (laptop and local-GPU rows " + "are provenance only)", + ) + check.require("superseded_by" in item, where, "missing superseded_by (null unless superseded)") + target = item.get("superseded_by") + if status == "superseded": + check.require( + isinstance(target, str) and target in ids and target != item["id"], + f"{where}.superseded_by", + f"must name another record in this document, got {target!r}", + ) + else: + check.require(target is None, f"{where}.superseded_by", "only superseded records set it") + _nullable_text(check, item, "decision_id", where) + if item.get("class") == "optimal": + check.require( + summary._text(item.get("decision_id")), + where, + "class optimal requires a decision_id (a Pulse designation decision)", + ) + _nullable_text(check, item, "day_zero", where) + _nullable_text(check, item, "release_pin", where) + check.require( + summary.parse_utc(item.get("measured_at")) is not None, + f"{where}.measured_at", + "must be an ISO-8601 UTC timestamp", + ) + for i, fig in enumerate(check.objects(item.get("figures"), f"{where}.figures")): + loc = f"{where}.figures[{i}]" + check.text(fig.get("kind"), f"{loc}.kind") + check.require(summary.safe_relative_path(fig.get("path")), loc, "unsafe figure path") + sha = fig.get("sha256") + check.require( + isinstance(sha, str) and bool(_SHA256.match(sha)), loc, "sha256 is not 64-hex" + ) + _record_body(check, item, where) + + +def _validate_v3(check, doc): + """The lifecycle rules: one current per key, optimal ⇒ decision, no archived rows.""" + records = check.index(doc.get("records"), "records") + ids = set(records) + for item in records.values(): + _record_v3(check, item, ids) + current = {} + for item in records.values(): + if item.get("status") == "current": + key = lifecycle_key(item) + prior = current.setdefault(key, item["id"]) + check.require( + prior == item["id"], + "records", + f"second current record {item['id']!r} on the key of {prior!r}; the producer " + "declares exactly one current per (likelihood, instrument, setup_id, host_class, " + "model_variant, measurement)", + ) + planned = check.objects(doc.get("planned_cells", []), "planned_cells") + seen = set() + for i, cell in enumerate(planned): + where = f"planned_cells[{i}]" + _cell_key_fields(check, cell, where) + key = lifecycle_key(cell) + check.require(key not in seen, where, "duplicate planned cell") + seen.add(key) + count = doc.get("archive_record_count") + check.require( + _integer(count), "archive_record_count", "must be a non-negative integer (0 when none)" + ) + # Advice keeps the v2 shape; its setup_ids name the canonical v3 setup_ids. + setups = { + r["setup_id"]: {"id": r["setup_id"]} + for r in records.values() + if summary._text(r.get("setup_id")) + } + for field, kind in (("hazards", "hazard"), ("recommendations", "recommendation")): + for item in check.index(doc.get(field, []), field).values(): + _advice(check, item, setups, records, kind) + cov = doc.get("coverage") + if isinstance(cov, dict): + expected, observed = cov.get("expected"), cov.get("observed") + check.require( + isinstance(expected, dict) + and type(expected.get("cells")) is int + and expected.get("cells") == len(planned), + "coverage.expected.cells", + "must equal the planned_cells count", + ) + if isinstance(observed, dict): + counts = lifecycle_counts(doc) + check.require( + type(observed.get("records")) is int and observed.get("records") == len(records), + "coverage.observed.records", + "must match catalogue contents", + ) + for key in ("current", "superseded"): + if key in observed: + check.require( + observed[key] == counts[key], + f"coverage.observed.{key}", + "must match catalogue contents", + ) + + +def lifecycle_counts(doc: dict) -> dict: + """``{current, superseded, archived}`` as the producer declared them (v3 only).""" + records = [r for r in _list(doc.get("records")) if isinstance(r, dict)] + count = doc.get("archive_record_count") + return { + "current": sum(r.get("status") == "current" for r in records), + "superseded": sum(r.get("status") == "superseded" for r in records), + "archived": count if _integer(count) else 0, + } + + +def decision_ids(doc: dict) -> list[str]: + """Every ``decision_id`` a v3 document names, for the repo-aware resolution step.""" + return sorted( + { + r["decision_id"] + for r in _list(doc.get("records")) + if isinstance(r, dict) and summary._text(r.get("decision_id")) + } + ) + + def _selection(check, item, setups, records): where = f"selection {item['id']!r}" check.reference(item.get("setup_id"), setups, f"{where}.setup_id") @@ -342,20 +618,27 @@ def _advice(check, item, setups, records, kind): ) -def validate(doc): - """Return all detected v2 contract violations; never choose a baseline.""" +def validate(doc, version=2): + """Return all detected v2/v3 contract violations; never choose a baseline. + + v3 resolves ``decision_id`` presence only; resolving each id against the + organ's ``decisions.yaml`` is the repo-aware step in ``pulse.ingest``. + """ check = _Check() _json_values(check, doc, "summary") # The common envelope retains v1 transport/receipt/board fields. Validate - # it through the established reader with empty records/comparisons; v2 - # records are checked below and v2 deliberately carries no trend pairs. + # it through the established reader with empty records/comparisons; v2/v3 + # records are checked below and neither carries trend pairs. envelope = dict(doc, records=[], comparisons=[]) check.errors.extend(summary._validate_v1(envelope)) check.require( doc.get("comparisons") == [], "comparisons", - "v2 setup feed has no temporal comparisons; must be []", + f"v{version} setup feed has no temporal comparisons; must be []", ) + if version == 3: + _validate_v3(check, doc) + return check.errors setups = check.index(doc.get("setups"), "setups") records = check.index(doc.get("records"), "records") selections = check.index(doc.get("selections"), "selections") diff --git a/pulse/summary.py b/pulse/summary.py index 4ca41db..b9f8780 100644 --- a/pulse/summary.py +++ b/pulse/summary.py @@ -1,9 +1,11 @@ -"""The ``profiling-summary`` v1/v2 exchange contracts, as the organ reads them. +"""The ``profiling-summary`` v1/v2/v3 and ``walltime-validation`` v1 exchange +contracts, as the organ reads them. The contract is owned by the producer and documented beside it (``autolens_profiling/dashboard/README.md``, "summary.json — the profiling-summary v1 read contract": Envelope / Records / Comparisons). This -module dispatches setup-oriented v2 validation to ``pulse.catalogue`` and +module dispatches setup-oriented v2 and lifecycle v3 validation to +``pulse.catalogue``, full-fit wall-time evidence to ``pulse.walltime``, and validates the **exchange** contract only — known schema/version, required fields, unique ids, finite numerics, UTC dates, safe evidence paths, axis/unit consistency — and pairs each producer comparison with its two @@ -20,7 +22,12 @@ from dataclasses import dataclass, field from datetime import datetime -SUPPORTED = {("profiling-summary", 1), ("profiling-summary", 2)} +SUPPORTED = { + ("profiling-summary", 1), + ("profiling-summary", 2), + ("profiling-summary", 3), + ("walltime-validation", 1), +} ENVELOPE = ( "schema", @@ -47,6 +54,10 @@ } STATUSES = {"drifted", "improved", "flat", "insufficient"} HARDWARE = ("device", "backend", "precision", "host") +# Where a v3 record ran (provenance, never identity). Laptop/local-GPU rows are +# never ``current``; ``hardware()`` reports the class beside the identity. +HOST_CLASSES = ("hpc_ral_cpu", "hpc_a100", "local_laptop", "local_gpu") +LOCAL_HOST_CLASSES = ("local_laptop", "local_gpu") _SHA = re.compile(r"^[0-9a-f]{40}$") @@ -121,10 +132,14 @@ def validate(doc) -> list[str]: why = supported(doc) if why: return [why] - if doc["version"] == 2: + if doc["schema"] == "walltime-validation": + from pulse import walltime + + return walltime.validate(doc) + if doc["version"] in (2, 3): from pulse import catalogue - return catalogue.validate(doc) + return catalogue.validate(doc, version=doc["version"]) return _validate_v1(doc) @@ -324,21 +339,30 @@ def classify(doc, wanted: tuple[str, int] | None = None) -> tuple[str, list[str] def hardware(record: dict) -> dict: - """The hardware identity a record was measured on (None = not recorded).""" + """The hardware identity a record was measured on (None = not recorded). + + A v3 record also names its ``host_class``; it is reported beside the + identity (provenance) and never compared as part of it. + """ ident = record.get("identity") or {} prov = record.get("provenance") or {} - return { + hw = { "device": ident.get("device"), "backend": ident.get("backend"), "precision": ident.get("precision"), "host": prov.get("host"), } + if "host_class" in record: + hw["host_class"] = record.get("host_class") + return hw def group_label(hw: dict | None) -> str: if hw is None: return "unresolved" parts = [str(hw[k]) if hw[k] is not None else f"{k} unknown" for k in HARDWARE[:3]] + if hw.get("host_class") is not None: + parts.append(str(hw["host_class"])) return " · ".join(parts) diff --git a/tests/fixtures/setup_summary_v3.json b/tests/fixtures/setup_summary_v3.json new file mode 100644 index 0000000..fca79da --- /dev/null +++ b/tests/fixtures/setup_summary_v3.json @@ -0,0 +1,511 @@ +{ + "schema": "profiling-summary", + "version": 3, + "project": "fixture_profiling", + "scope": "likelihood-lifecycle", + "generated_at": "2026-10-21T12:00:00Z", + "evidence_updated_at": "2026-10-18T12:00:00Z", + "valid_until": null, + "producer_revision": "0123456789abcdef0123456789abcdef01234567", + "comparison_policy": { + "id": "no-temporal-comparisons" + }, + "coverage": { + "expected": { + "cells": 5 + }, + "observed": { + "records": 6, + "current": 5, + "superseded": 1 + }, + "excluded": [] + }, + "records": [ + { + "id": "imaging-delaunay-hst-a100-runtime", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-runtime", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "fp64", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.012 + }, + "class": "optimal", + "status": "current", + "superseded_by": null, + "day_zero": "2026.10.16.2", + "figures": [ + { + "kind": "fit", + "path": "figures/imaging_delaunay_hst.png", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000" + } + ], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-runtime.json", + "fragment": "single_call" + } + }, + { + "id": "imaging-delaunay-hst-a100-runtime-old", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-runtime-old", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "fp64", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.019 + }, + "class": "optimal", + "status": "superseded", + "superseded_by": "imaging-delaunay-hst-a100-runtime", + "day_zero": null, + "figures": [], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-runtime-old.json", + "fragment": "single_call" + } + }, + { + "id": "imaging-delaunay-hst-a100-breakdown", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-breakdown", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "fp64", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "breakdown", + "metric": "component_total", + "unit": "s", + "values": { + "component_total": 0.011, + "mapping": 0.004, + "inversion": 0.007 + }, + "class": "optimal", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "breakdown-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-breakdown.json", + "fragment": "component_total" + } + }, + { + "id": "imaging-delaunay-hst-ral-runtime", + "run_id": "synthetic-run-imaging-delaunay-hst-ral-runtime", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "numba", + "device": "cpu", + "operator": "sparse", + "precision": "fp64", + "transformer": "na", + "threads": 16 + }, + "setup_id": "numba_cpu_sparse_fp64", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.21 + }, + "class": "secondary", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": null, + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "cpu", + "backend": "numba", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_ral_cpu", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-ral-runtime.json", + "fragment": "single_call" + } + }, + { + "id": "imaging-delaunay-hst-a100-mixed", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-mixed", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "mixed", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_mixed", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.009 + }, + "class": "experimental", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": null, + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "mixed", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-mixed.json", + "fragment": "single_call" + } + }, + { + "id": "interferometer-mge-sma-ral-runtime", + "run_id": "synthetic-run-interferometer-mge-sma-ral-runtime", + "likelihood": { + "dataset_type": "interferometer", + "variant": "mge" + }, + "instrument": "sma", + "regime": "n_vis_lt_1e5", + "setup": { + "backend": "jax", + "device": "cpu", + "operator": "dense", + "precision": "fp64", + "transformer": "nufft", + "threads": 8 + }, + "setup_id": "jax_cpu_dense_fp64_nufft", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.4 + }, + "class": "optimal", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "cpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_ral_cpu", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/interferometer/mge/interferometer-mge-sma-ral-runtime.json", + "fragment": "single_call" + } + } + ], + "comparisons": [], + "limitations": [ + "fixture: hand-written lifecycle catalogue; not a scientific result" + ], + "planned_cells": [ + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "class": "optimal" + }, + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "breakdown", + "class": "optimal" + }, + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "euclid", + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "class": "optimal" + }, + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "setup_id": "numba_cpu_sparse_fp64", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "class": "secondary" + }, + { + "likelihood": { + "dataset_type": "interferometer", + "variant": "mge" + }, + "instrument": "sma", + "setup_id": "jax_cpu_dense_fp64_nufft", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "class": "optimal" + } + ], + "archive_record_count": 3, + "hazards": [], + "recommendations": [] +} diff --git a/tests/fixtures/v3_archived_in_records.json b/tests/fixtures/v3_archived_in_records.json new file mode 100644 index 0000000..a9e6349 --- /dev/null +++ b/tests/fixtures/v3_archived_in_records.json @@ -0,0 +1,510 @@ +{ + "schema": "profiling-summary", + "version": 3, + "project": "fixture_profiling", + "scope": "likelihood-lifecycle", + "generated_at": "2026-10-21T12:00:00Z", + "evidence_updated_at": "2026-10-18T12:00:00Z", + "valid_until": null, + "producer_revision": "0123456789abcdef0123456789abcdef01234567", + "comparison_policy": { + "id": "no-temporal-comparisons" + }, + "coverage": { + "expected": { + "cells": 5 + }, + "observed": { + "records": 6, + "current": 5 + }, + "excluded": [] + }, + "records": [ + { + "id": "imaging-delaunay-hst-a100-runtime", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-runtime", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "fp64", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.012 + }, + "class": "optimal", + "status": "current", + "superseded_by": null, + "day_zero": "2026.10.16.2", + "figures": [ + { + "kind": "fit", + "path": "figures/imaging_delaunay_hst.png", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000" + } + ], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-runtime.json", + "fragment": "single_call" + } + }, + { + "id": "imaging-delaunay-hst-a100-runtime-old", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-runtime-old", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "fp64", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.019 + }, + "class": "optimal", + "status": "archived", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-runtime-old.json", + "fragment": "single_call" + } + }, + { + "id": "imaging-delaunay-hst-a100-breakdown", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-breakdown", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "fp64", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "breakdown", + "metric": "component_total", + "unit": "s", + "values": { + "component_total": 0.011, + "mapping": 0.004, + "inversion": 0.007 + }, + "class": "optimal", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "breakdown-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-breakdown.json", + "fragment": "component_total" + } + }, + { + "id": "imaging-delaunay-hst-ral-runtime", + "run_id": "synthetic-run-imaging-delaunay-hst-ral-runtime", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "numba", + "device": "cpu", + "operator": "sparse", + "precision": "fp64", + "transformer": "na", + "threads": 16 + }, + "setup_id": "numba_cpu_sparse_fp64", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.21 + }, + "class": "secondary", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": null, + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "cpu", + "backend": "numba", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_ral_cpu", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-ral-runtime.json", + "fragment": "single_call" + } + }, + { + "id": "imaging-delaunay-hst-a100-mixed", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-mixed", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "mixed", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_mixed", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.009 + }, + "class": "experimental", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": null, + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "mixed", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-mixed.json", + "fragment": "single_call" + } + }, + { + "id": "interferometer-mge-sma-ral-runtime", + "run_id": "synthetic-run-interferometer-mge-sma-ral-runtime", + "likelihood": { + "dataset_type": "interferometer", + "variant": "mge" + }, + "instrument": "sma", + "regime": "n_vis_lt_1e5", + "setup": { + "backend": "jax", + "device": "cpu", + "operator": "dense", + "precision": "fp64", + "transformer": "nufft", + "threads": 8 + }, + "setup_id": "jax_cpu_dense_fp64_nufft", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.4 + }, + "class": "optimal", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "cpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_ral_cpu", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/interferometer/mge/interferometer-mge-sma-ral-runtime.json", + "fragment": "single_call" + } + } + ], + "comparisons": [], + "limitations": [ + "fixture: hand-written lifecycle catalogue; not a scientific result" + ], + "planned_cells": [ + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "class": "optimal" + }, + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "breakdown", + "class": "optimal" + }, + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "euclid", + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "class": "optimal" + }, + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "setup_id": "numba_cpu_sparse_fp64", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "class": "secondary" + }, + { + "likelihood": { + "dataset_type": "interferometer", + "variant": "mge" + }, + "instrument": "sma", + "setup_id": "jax_cpu_dense_fp64_nufft", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "class": "optimal" + } + ], + "archive_record_count": 3, + "hazards": [], + "recommendations": [] +} diff --git a/tests/fixtures/v3_duplicate_current.json b/tests/fixtures/v3_duplicate_current.json new file mode 100644 index 0000000..168f811 --- /dev/null +++ b/tests/fixtures/v3_duplicate_current.json @@ -0,0 +1,509 @@ +{ + "schema": "profiling-summary", + "version": 3, + "project": "fixture_profiling", + "scope": "likelihood-lifecycle", + "generated_at": "2026-10-21T12:00:00Z", + "evidence_updated_at": "2026-10-18T12:00:00Z", + "valid_until": null, + "producer_revision": "0123456789abcdef0123456789abcdef01234567", + "comparison_policy": { + "id": "no-temporal-comparisons" + }, + "coverage": { + "expected": { + "cells": 5 + }, + "observed": { + "records": 6 + }, + "excluded": [] + }, + "records": [ + { + "id": "imaging-delaunay-hst-a100-runtime", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-runtime", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "fp64", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.012 + }, + "class": "optimal", + "status": "current", + "superseded_by": null, + "day_zero": "2026.10.16.2", + "figures": [ + { + "kind": "fit", + "path": "figures/imaging_delaunay_hst.png", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000" + } + ], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-runtime.json", + "fragment": "single_call" + } + }, + { + "id": "imaging-delaunay-hst-a100-runtime-old", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-runtime-old", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "fp64", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.019 + }, + "class": "optimal", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-runtime-old.json", + "fragment": "single_call" + } + }, + { + "id": "imaging-delaunay-hst-a100-breakdown", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-breakdown", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "fp64", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "breakdown", + "metric": "component_total", + "unit": "s", + "values": { + "component_total": 0.011, + "mapping": 0.004, + "inversion": 0.007 + }, + "class": "optimal", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "breakdown-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-breakdown.json", + "fragment": "component_total" + } + }, + { + "id": "imaging-delaunay-hst-ral-runtime", + "run_id": "synthetic-run-imaging-delaunay-hst-ral-runtime", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "numba", + "device": "cpu", + "operator": "sparse", + "precision": "fp64", + "transformer": "na", + "threads": 16 + }, + "setup_id": "numba_cpu_sparse_fp64", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.21 + }, + "class": "secondary", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": null, + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "cpu", + "backend": "numba", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_ral_cpu", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-ral-runtime.json", + "fragment": "single_call" + } + }, + { + "id": "imaging-delaunay-hst-a100-mixed", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-mixed", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "mixed", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_mixed", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.009 + }, + "class": "experimental", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": null, + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "mixed", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-mixed.json", + "fragment": "single_call" + } + }, + { + "id": "interferometer-mge-sma-ral-runtime", + "run_id": "synthetic-run-interferometer-mge-sma-ral-runtime", + "likelihood": { + "dataset_type": "interferometer", + "variant": "mge" + }, + "instrument": "sma", + "regime": "n_vis_lt_1e5", + "setup": { + "backend": "jax", + "device": "cpu", + "operator": "dense", + "precision": "fp64", + "transformer": "nufft", + "threads": 8 + }, + "setup_id": "jax_cpu_dense_fp64_nufft", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.4 + }, + "class": "optimal", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "cpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_ral_cpu", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/interferometer/mge/interferometer-mge-sma-ral-runtime.json", + "fragment": "single_call" + } + } + ], + "comparisons": [], + "limitations": [ + "fixture: hand-written lifecycle catalogue; not a scientific result" + ], + "planned_cells": [ + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "class": "optimal" + }, + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "breakdown", + "class": "optimal" + }, + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "euclid", + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "class": "optimal" + }, + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "setup_id": "numba_cpu_sparse_fp64", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "class": "secondary" + }, + { + "likelihood": { + "dataset_type": "interferometer", + "variant": "mge" + }, + "instrument": "sma", + "setup_id": "jax_cpu_dense_fp64_nufft", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "class": "optimal" + } + ], + "archive_record_count": 3, + "hazards": [], + "recommendations": [] +} diff --git a/tests/fixtures/v3_laptop_current.json b/tests/fixtures/v3_laptop_current.json new file mode 100644 index 0000000..718d1ce --- /dev/null +++ b/tests/fixtures/v3_laptop_current.json @@ -0,0 +1,511 @@ +{ + "schema": "profiling-summary", + "version": 3, + "project": "fixture_profiling", + "scope": "likelihood-lifecycle", + "generated_at": "2026-10-21T12:00:00Z", + "evidence_updated_at": "2026-10-18T12:00:00Z", + "valid_until": null, + "producer_revision": "0123456789abcdef0123456789abcdef01234567", + "comparison_policy": { + "id": "no-temporal-comparisons" + }, + "coverage": { + "expected": { + "cells": 5 + }, + "observed": { + "records": 6, + "current": 5, + "superseded": 1 + }, + "excluded": [] + }, + "records": [ + { + "id": "imaging-delaunay-hst-a100-runtime", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-runtime", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "fp64", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.012 + }, + "class": "optimal", + "status": "current", + "superseded_by": null, + "day_zero": "2026.10.16.2", + "figures": [ + { + "kind": "fit", + "path": "figures/imaging_delaunay_hst.png", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000" + } + ], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-runtime.json", + "fragment": "single_call" + } + }, + { + "id": "imaging-delaunay-hst-a100-runtime-old", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-runtime-old", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "fp64", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.019 + }, + "class": "optimal", + "status": "superseded", + "superseded_by": "imaging-delaunay-hst-a100-runtime", + "day_zero": null, + "figures": [], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-runtime-old.json", + "fragment": "single_call" + } + }, + { + "id": "imaging-delaunay-hst-a100-breakdown", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-breakdown", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "fp64", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "breakdown", + "metric": "component_total", + "unit": "s", + "values": { + "component_total": 0.011, + "mapping": 0.004, + "inversion": 0.007 + }, + "class": "optimal", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "breakdown-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-breakdown.json", + "fragment": "component_total" + } + }, + { + "id": "imaging-delaunay-hst-ral-runtime", + "run_id": "synthetic-run-imaging-delaunay-hst-ral-runtime", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "numba", + "device": "cpu", + "operator": "sparse", + "precision": "fp64", + "transformer": "na", + "threads": 16 + }, + "setup_id": "numba_cpu_sparse_fp64", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.21 + }, + "class": "secondary", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": null, + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "cpu", + "backend": "numba", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_ral_cpu", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-ral-runtime.json", + "fragment": "single_call" + } + }, + { + "id": "imaging-delaunay-hst-a100-mixed", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-mixed", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "mixed", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_mixed", + "host_class": "local_laptop", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.009 + }, + "class": "experimental", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": null, + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "mixed", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-mixed.json", + "fragment": "single_call" + } + }, + { + "id": "interferometer-mge-sma-ral-runtime", + "run_id": "synthetic-run-interferometer-mge-sma-ral-runtime", + "likelihood": { + "dataset_type": "interferometer", + "variant": "mge" + }, + "instrument": "sma", + "regime": "n_vis_lt_1e5", + "setup": { + "backend": "jax", + "device": "cpu", + "operator": "dense", + "precision": "fp64", + "transformer": "nufft", + "threads": 8 + }, + "setup_id": "jax_cpu_dense_fp64_nufft", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.4 + }, + "class": "optimal", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "cpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_ral_cpu", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/interferometer/mge/interferometer-mge-sma-ral-runtime.json", + "fragment": "single_call" + } + } + ], + "comparisons": [], + "limitations": [ + "fixture: hand-written lifecycle catalogue; not a scientific result" + ], + "planned_cells": [ + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "class": "optimal" + }, + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "breakdown", + "class": "optimal" + }, + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "euclid", + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "class": "optimal" + }, + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "setup_id": "numba_cpu_sparse_fp64", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "class": "secondary" + }, + { + "likelihood": { + "dataset_type": "interferometer", + "variant": "mge" + }, + "instrument": "sma", + "setup_id": "jax_cpu_dense_fp64_nufft", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "class": "optimal" + } + ], + "archive_record_count": 3, + "hazards": [], + "recommendations": [] +} diff --git a/tests/fixtures/v3_optimal_without_decision.json b/tests/fixtures/v3_optimal_without_decision.json new file mode 100644 index 0000000..faebeaf --- /dev/null +++ b/tests/fixtures/v3_optimal_without_decision.json @@ -0,0 +1,511 @@ +{ + "schema": "profiling-summary", + "version": 3, + "project": "fixture_profiling", + "scope": "likelihood-lifecycle", + "generated_at": "2026-10-21T12:00:00Z", + "evidence_updated_at": "2026-10-18T12:00:00Z", + "valid_until": null, + "producer_revision": "0123456789abcdef0123456789abcdef01234567", + "comparison_policy": { + "id": "no-temporal-comparisons" + }, + "coverage": { + "expected": { + "cells": 5 + }, + "observed": { + "records": 6, + "current": 5, + "superseded": 1 + }, + "excluded": [] + }, + "records": [ + { + "id": "imaging-delaunay-hst-a100-runtime", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-runtime", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "fp64", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.012 + }, + "class": "optimal", + "status": "current", + "superseded_by": null, + "day_zero": "2026.10.16.2", + "figures": [ + { + "kind": "fit", + "path": "figures/imaging_delaunay_hst.png", + "sha256": "0000000000000000000000000000000000000000000000000000000000000000" + } + ], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-runtime.json", + "fragment": "single_call" + } + }, + { + "id": "imaging-delaunay-hst-a100-runtime-old", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-runtime-old", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "fp64", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.019 + }, + "class": "optimal", + "status": "superseded", + "superseded_by": "imaging-delaunay-hst-a100-runtime", + "day_zero": null, + "figures": [], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-runtime-old.json", + "fragment": "single_call" + } + }, + { + "id": "imaging-delaunay-hst-a100-breakdown", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-breakdown", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "fp64", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "breakdown", + "metric": "component_total", + "unit": "s", + "values": { + "component_total": 0.011, + "mapping": 0.004, + "inversion": 0.007 + }, + "class": "optimal", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": "fixture-designation", + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "breakdown-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-breakdown.json", + "fragment": "component_total" + } + }, + { + "id": "imaging-delaunay-hst-ral-runtime", + "run_id": "synthetic-run-imaging-delaunay-hst-ral-runtime", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "numba", + "device": "cpu", + "operator": "sparse", + "precision": "fp64", + "transformer": "na", + "threads": 16 + }, + "setup_id": "numba_cpu_sparse_fp64", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.21 + }, + "class": "secondary", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": null, + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "cpu", + "backend": "numba", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_ral_cpu", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-ral-runtime.json", + "fragment": "single_call" + } + }, + { + "id": "imaging-delaunay-hst-a100-mixed", + "run_id": "synthetic-run-imaging-delaunay-hst-a100-mixed", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "regime": "standard", + "setup": { + "backend": "jax", + "device": "gpu", + "operator": "dense", + "precision": "mixed", + "transformer": "na", + "threads": null + }, + "setup_id": "jax_gpu_dense_mixed", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.009 + }, + "class": "experimental", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": null, + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "gpu", + "backend": "jax", + "precision": "mixed", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_a100", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/imaging/delaunay/imaging-delaunay-hst-a100-mixed.json", + "fragment": "single_call" + } + }, + { + "id": "interferometer-mge-sma-ral-runtime", + "run_id": "synthetic-run-interferometer-mge-sma-ral-runtime", + "likelihood": { + "dataset_type": "interferometer", + "variant": "mge" + }, + "instrument": "sma", + "regime": "n_vis_lt_1e5", + "setup": { + "backend": "jax", + "device": "cpu", + "operator": "dense", + "precision": "fp64", + "transformer": "nufft", + "threads": 8 + }, + "setup_id": "jax_cpu_dense_fp64_nufft", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "metric": "single_call", + "unit": "s", + "values": { + "single_call": 0.4 + }, + "class": "optimal", + "status": "current", + "superseded_by": null, + "day_zero": null, + "figures": [], + "decision_id": null, + "release_pin": "2026.10.16.2", + "measured_at": "2026-10-18T12:00:00Z", + "identity": { + "device": "cpu", + "backend": "jax", + "precision": "float64", + "library_version": "2026.10.16.2", + "software": { + "PyAutoLens": "2026.10.16.2", + "PyAutoArray": "2026.10.16.2", + "jax": "fixture" + } + }, + "method": { + "id": "runtime-fixture-method", + "statistic": "median", + "repetitions": 5, + "warmup": "one call", + "synchronization": "block_until_ready", + "cache_state": "warm" + }, + "provenance": { + "host": "fixture-hpc_ral_cpu", + "measured_at": "2026-10-18T12:00:00Z", + "has_provenance": true, + "qualified": true + }, + "validation": { + "status": "accepted", + "reason": "synthetic fixture only; not a scientific result" + }, + "evidence": { + "path": "results/interferometer/mge/interferometer-mge-sma-ral-runtime.json", + "fragment": "single_call" + } + } + ], + "comparisons": [], + "limitations": [ + "fixture: hand-written lifecycle catalogue; not a scientific result" + ], + "planned_cells": [ + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "class": "optimal" + }, + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "breakdown", + "class": "optimal" + }, + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "euclid", + "setup_id": "jax_gpu_dense_fp64", + "host_class": "hpc_a100", + "model_variant": "default", + "measurement": "runtime", + "class": "optimal" + }, + { + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "setup_id": "numba_cpu_sparse_fp64", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "class": "secondary" + }, + { + "likelihood": { + "dataset_type": "interferometer", + "variant": "mge" + }, + "instrument": "sma", + "setup_id": "jax_cpu_dense_fp64_nufft", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "measurement": "runtime", + "class": "optimal" + } + ], + "archive_record_count": 3, + "hazards": [], + "recommendations": [] +} diff --git a/tests/test_catalogue.py b/tests/test_catalogue.py index 05f3932..c63f390 100644 --- a/tests/test_catalogue.py +++ b/tests/test_catalogue.py @@ -7,7 +7,7 @@ import yaml from conftest import SHA_A, SHA_B, fixture_doc -from pulse import board, ingest, registry, summary +from pulse import board, catalogue, ingest, registry, summary V2 = ("profiling-summary", 2) @@ -157,7 +157,13 @@ def test_accepted_recommendation_cannot_rest_on_unreviewed_evidence(doc): @pytest.mark.parametrize( "schema,version", - [("profiling-summary", 3), ("profiling-summary", True), ([], 2), ("profiling-summary", {})], + [ + ("profiling-summary", 4), + ("walltime-validation", 2), + ("profiling-summary", True), + ([], 2), + ("profiling-summary", {}), + ], ) def test_bad_versions_fail_closed_without_crashing(doc, schema, version): doc.update(schema=schema, version=version) @@ -236,3 +242,142 @@ def test_the_autofit_profiling_producer_feed_validates_through_the_shared_reader assert memory and {(r["metric"], r["unit"]) for r in memory} == {("host_peak_rss", "KiB")} slots = [s for s in doc["selections"] if s["status"] == "inapplicable"] assert slots and all(s["record_id"] is None for s in slots) + + +# ---------------------------------------------------- profiling-summary@3 --- + +V3 = ("profiling-summary", 3) + + +@pytest.fixture +def v3(): + return fixture_doc("setup_summary_v3.json") + + +def test_lifecycle_catalogue_validates_and_v2_is_unchanged(v3): + assert summary.classify(v3, V3) == ("ok", []) + assert summary.classify(v3, V2)[0] == "unsupported" + assert summary.classify(fixture_doc("setup_summary_v2.json"), V2) == ("ok", []) + assert summary.classify(fixture_doc("autofit_profiling_v2.json"), V2) == ("ok", []) + assert catalogue.lifecycle_counts(v3) == {"current": 5, "superseded": 1, "archived": 3} + assert catalogue.decision_ids(v3) == ["fixture-designation"] + assert summary.pair(v3) == [] + + +@pytest.mark.parametrize( + "fixture,needle", + [ + ("v3_duplicate_current.json", "second current record"), + ("v3_optimal_without_decision.json", "class optimal requires a decision_id"), + ("v3_laptop_current.json", "is never current"), + ("v3_archived_in_records.json", "archive_record_count"), + ], +) +def test_lifecycle_refusal_fixtures(fixture, needle): + outcome, errors = summary.classify(fixture_doc(fixture), V3) + assert outcome == "invalid" + assert any(needle in e for e in errors), errors + + +@pytest.mark.parametrize( + "path,value,needle", + [ + (("records", 0, "setup_id"), "jax_gpu_dense", "must be 'jax_gpu_dense_fp64'"), + (("records", 5, "setup_id"), "jax_cpu_dense_fp64", "jax_cpu_dense_fp64_nufft"), + (("records", 0, "setup", "backend"), "torch", "setup.backend"), + (("records", 0, "setup", "precision"), "fp32", "setup.precision"), + (("records", 0, "setup", "threads"), 0, "threads"), + (("records", 0, "likelihood", "dataset_type"), "pixelized", "dataset_type"), + (("records", 0, "host_class"), "cloud", "host_class"), + (("records", 0, "measurement"), "memory", "measurement"), + (("records", 0, "unit"), "MiB", "unit"), + (("records", 0, "class"), "fastest", "class"), + (("records", 0, "status"), "latest", "status"), + (("records", 0, "values"), {"other": 1.0}, "headline metric"), + (("records", 0, "values", "single_call"), -1, "non-negative"), + (("records", 0, "measured_at"), "2026-10-18", "measured_at"), + (("records", 0, "figures", 0, "sha256"), "abc", "sha256"), + (("records", 0, "figures", 0, "path"), "/abs.png", "unsafe figure path"), + ( + ("records", 0, "superseded_by"), + "imaging-delaunay-hst-a100-runtime-old", + "only superseded", + ), + (("records", 1, "superseded_by"), "missing", "must name another record"), + (("records", 1, "superseded_by"), None, "must name another record"), + (("records", 3, "decision_id"), "", "string or null"), + (("planned_cells", 0, "class"), None, "class"), + (("planned_cells", 1), "cell", "must be an object"), + (("archive_record_count",), -1, "archive_record_count"), + (("archive_record_count",), True, "archive_record_count"), + (("coverage", "expected", "cells"), 4, "planned_cells count"), + (("coverage", "observed", "current"), 9, "observed.current"), + ], +) +def test_lifecycle_contract_breaks_are_named(v3, path, value, needle): + target = v3 + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + outcome, errors = summary.classify(v3, V3) + assert outcome == "invalid" + assert any(needle in e for e in errors), errors + + +def test_planned_cells_carry_class_and_are_unique(v3): + v3["planned_cells"][0].pop("class") + assert any("planned_cells[0].class" in e for e in summary.validate(v3)) + v3 = fixture_doc("setup_summary_v3.json") + v3["planned_cells"].append(copy.deepcopy(v3["planned_cells"][0])) + v3["coverage"]["expected"]["cells"] += 1 + assert any("duplicate planned cell" in e for e in summary.validate(v3)) + + +def test_one_current_per_key_counts_host_class_and_variant_as_identity(v3): + other = copy.deepcopy(v3["records"][3]) + other.update(id="light-fixed", model_variant="light_fixed") + v3["records"].append(other) + v3["coverage"]["observed"].update(records=7, current=6) + assert summary.classify(v3, V3) == ("ok", []) + clash = copy.deepcopy(v3["records"][3]) + clash["id"] = "clash" + v3["records"].append(clash) + v3["coverage"]["observed"].update(records=8, current=7) + assert any("second current record 'clash'" in e for e in summary.validate(v3)) + + +def test_local_gpu_rows_may_be_kept_but_never_current(v3): + row = copy.deepcopy(v3["records"][4]) + row.update(id="laptop-gpu", host_class="local_gpu", status="superseded") + row["superseded_by"] = v3["records"][4]["id"] + v3["records"].append(row) + v3["coverage"]["observed"].update(records=7, superseded=2) + assert summary.classify(v3, V3) == ("ok", []) + row["status"], row["superseded_by"] = "current", None + row["model_variant"] = "laptop" + v3["coverage"]["observed"].update(current=6, superseded=1) + assert any("'local_gpu' is never current" in e for e in summary.validate(v3)) + + +@pytest.mark.parametrize("value", [None, [], {}, True, 0, "", float("nan")]) +def test_malformed_v3_fields_return_errors_not_exceptions(v3, value): + def paths(obj, prefix=()): + items = ( + obj.items() + if isinstance(obj, dict) + else enumerate(obj) + if isinstance(obj, list) + else [] + ) + for key, child in items: + yield (*prefix, key) + yield from paths(child, (*prefix, key)) + + for path in list(paths(v3))[::3]: + changed = copy.deepcopy(v3) + target = changed + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + outcome, errors = summary.classify(changed) + assert outcome in ("ok", "invalid", "unsupported"), (path, outcome, errors) From 9361125a1bf3fbc95cbb846aea432ca81cd74651 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Wed, 21 Oct 2026 18:50:23 +0100 Subject: [PATCH 2/5] pulse: walltime-validation@1 full-fit evidence axis Records keyed by (likelihood, instrument, setup_id, host_class, model_variant, sampler, seed); every derived number is producer-declared and finite or null; duplicate keys refused. Pulse validates and renders only. Co-Authored-By: Claude Fable 5.1 --- pulse/walltime.py | 109 +++++++++++++++++++++ tests/fixtures/walltime_validation_v1.json | 62 ++++++++++++ tests/test_summary.py | 71 ++++++++++++++ 3 files changed, 242 insertions(+) create mode 100644 pulse/walltime.py create mode 100644 tests/fixtures/walltime_validation_v1.json diff --git a/pulse/walltime.py b/pulse/walltime.py new file mode 100644 index 0000000..d2ab711 --- /dev/null +++ b/pulse/walltime.py @@ -0,0 +1,109 @@ +"""Validate the ``walltime-validation`` v1 evidence axis without judging it. + +A wall-time validation record is a **full-fit** measurement: one sampler run of +one likelihood, setup and seed, with its wall time, likelihood evaluations and +the producer's own comparison against the per-likelihood ``runtime`` record it +names. Every derived number (``implied_ms_per_eval``, ``predicted_ms_per_eval``, +``ratio_measured_over_predicted``, ``parallel_efficiency``) is computed and +declared by the producer; Pulse validates the shape and renders the values. It +computes no ratio, infers no optimal setup and never reads a full-fit time as a +per-likelihood measurement (or the reverse). Stdlib only. See REFERENCE.md. +""" + +from __future__ import annotations + +import json +import re + +from pulse import catalogue, summary + +ENVELOPE = ("schema", "version", "project", "generated_at", "producer_revision", "records") +PARITY_STATUSES = {"match", "diverged", "not_assessed"} +NUMERIC = ( + "wall_s", + "implied_ms_per_eval", + "predicted_ms_per_eval", + "ratio_measured_over_predicted", + "parallel_efficiency", + "max_log_likelihood", +) +KEY_FIELDS = ( + "likelihood", + "instrument", + "setup_id", + "host_class", + "model_variant", + "sampler", + "seed", +) +_SHA = re.compile(r"^[0-9a-f]{40}$") + + +def _count(value) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value >= 0 + + +def key(record: dict) -> str: + """The identity of one full-fit run: one record per key.""" + return json.dumps({k: record.get(k) for k in KEY_FIELDS}, sort_keys=True) + + +def _record(check, item, where): + catalogue._likelihood(check, item.get("likelihood"), f"{where}.likelihood") + check.text(item.get("instrument"), f"{where}.instrument") + check.text(item.get("setup_id"), f"{where}.setup_id") + check.choice(item.get("host_class"), catalogue.HOST_CLASSES, f"{where}.host_class") + check.text(item.get("model_variant"), f"{where}.model_variant") + check.text(item.get("sampler"), f"{where}.sampler") + check.require(_count(item.get("seed")), f"{where}.seed", "must be a non-negative integer") + for field in NUMERIC: + check.require(field in item, where, f"missing {field} (null when not measured)") + check.require( + summary._finite_or_null(item.get(field)), f"{where}.{field}", "finite number or null" + ) + for field in ("likelihood_evals", "cores"): + value = item.get(field) + check.require(field in item, where, f"missing {field} (null when not recorded)") + check.require( + value is None or _count(value), f"{where}.{field}", "non-negative integer or null" + ) + for field in ("reference_runtime_record_id", "release_pin", "slurm_job_id"): + check.require(field in item, where, f"missing {field} (null when not applicable)") + value = item.get(field) + check.require(value is None or summary._text(value), f"{where}.{field}", "string or null") + check.choice(item.get("parity_status"), PARITY_STATUSES, f"{where}.parity_status") + + +def validate(doc) -> list[str]: + """Every contract break in a ``walltime-validation@1`` document (empty = valid).""" + if not isinstance(doc, dict): + return ["summary is not a JSON object"] + check = catalogue._Check() + catalogue._json_values(check, doc, "summary") + check.errors += [f"missing required field {k!r}" for k in ENVELOPE if k not in doc] + check.require(summary._text(doc.get("project")), "project", "must be a non-empty string") + check.require( + summary.parse_utc(doc.get("generated_at")) is not None, + "generated_at", + "must be an ISO-8601 UTC timestamp", + ) + rev = doc.get("producer_revision") + check.require( + rev is None or (isinstance(rev, str) and bool(_SHA.match(rev))), + "producer_revision", + f"is not a 40-hex commit or null: {rev!r}", + ) + lim = doc.get("limitations", []) + check.require( + isinstance(lim, list) and all(summary._text(x) for x in lim), + "limitations", + "must be a list of non-empty strings", + ) + records = check.index(doc.get("records"), "records") + seen: dict[str, str] = {} + for rid, item in records.items(): + where = f"record {rid!r}" + _record(check, item, where) + prior = seen.setdefault(key(item), rid) + check.require(prior == rid, where, f"duplicate key of record {prior!r}") + return check.errors diff --git a/tests/fixtures/walltime_validation_v1.json b/tests/fixtures/walltime_validation_v1.json new file mode 100644 index 0000000..ce24b34 --- /dev/null +++ b/tests/fixtures/walltime_validation_v1.json @@ -0,0 +1,62 @@ +{ + "schema": "walltime-validation", + "version": 1, + "project": "fixture_inference", + "generated_at": "2026-10-21T12:00:00Z", + "producer_revision": "0123456789abcdef0123456789abcdef01234567", + "limitations": [ + "fixture: full-fit measurements; synthetic, not a scientific result" + ], + "records": [ + { + "id": "nautilus-1core-s0", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "setup_id": "numba_cpu_sparse_fp64", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "sampler": "nautilus", + "seed": 0, + "wall_s": 4200.0, + "likelihood_evals": 20000, + "implied_ms_per_eval": 210.0, + "reference_runtime_record_id": "imaging-delaunay-hst-ral-runtime", + "predicted_ms_per_eval": 210.0, + "ratio_measured_over_predicted": 1.0, + "cores": 1, + "parallel_efficiency": 1.0, + "release_pin": "2026.10.16.2", + "slurm_job_id": "412000", + "max_log_likelihood": 12345.6, + "parity_status": "match" + }, + { + "id": "nautilus-16core-s0", + "likelihood": { + "dataset_type": "imaging", + "variant": "delaunay" + }, + "instrument": "hst", + "setup_id": "numba_cpu_sparse_fp64", + "host_class": "hpc_ral_cpu", + "model_variant": "default", + "sampler": "nautilus", + "seed": 1, + "wall_s": 330.0, + "likelihood_evals": 20000, + "implied_ms_per_eval": 16.5, + "reference_runtime_record_id": "imaging-delaunay-hst-ral-runtime", + "predicted_ms_per_eval": 13.1, + "ratio_measured_over_predicted": 1.26, + "cores": 16, + "parallel_efficiency": 0.795, + "release_pin": "2026.10.16.2", + "slurm_job_id": "412000", + "max_log_likelihood": 12345.6, + "parity_status": "not_assessed" + } + ] +} diff --git a/tests/test_summary.py b/tests/test_summary.py index 305230c..654ac63 100644 --- a/tests/test_summary.py +++ b/tests/test_summary.py @@ -136,3 +136,74 @@ def test_second_producer_takes_the_same_code_path(): assert summary.classify(doc, V1) == ("ok", []) (p,) = summary.pair(doc) assert p.group == "a100 · gpu · mixed" and not p.refused + + +# --------------------------------------------- v3 dispatch and walltime@1 --- + +WALL = ("walltime-validation", 1) + + +def test_supported_set_names_every_reader_version(): + assert summary.SUPPORTED == { + ("profiling-summary", 1), + ("profiling-summary", 2), + ("profiling-summary", 3), + ("walltime-validation", 1), + } + assert summary.classify(fixture_doc("setup_summary_v3.json"), ("profiling-summary", 3)) == ( + "ok", + [], + ) + + +def test_hardware_reports_host_class_beside_identity_never_inside_it(): + record = fixture_doc("setup_summary_v3.json")["records"][0] + hw = summary.hardware(record) + assert hw["host_class"] == "hpc_a100" + assert summary.group_label(hw) == "gpu · jax · float64 · hpc_a100" + v1 = fixture_doc("lens_summary_v1.json")["records"][0] + assert "host_class" not in summary.hardware(v1) + + +def test_walltime_validation_is_accepted_and_pulse_computes_nothing(): + doc = fixture_doc("walltime_validation_v1.json") + assert summary.classify(doc, WALL) == ("ok", []) + assert summary.classify(doc, ("profiling-summary", 3))[0] == "unsupported" + before = copy.deepcopy(doc) + # A producer ratio that disagrees with its own inputs is still the producer's. + doc["records"][1]["ratio_measured_over_predicted"] = 99.0 + assert summary.classify(doc, WALL) == ("ok", []) + assert before["records"][0] == doc["records"][0] + + +@pytest.mark.parametrize( + "mutation,needle", + [ + (lambda d: d["records"][0].update(wall_s=math.inf), "finite number or null"), + (lambda d: d["records"][0].update(implied_ms_per_eval="fast"), "finite number or null"), + (lambda d: d["records"][0].pop("max_log_likelihood"), "missing max_log_likelihood"), + (lambda d: d["records"][0].update(parity_status="ok"), "parity_status"), + (lambda d: d["records"][0].update(seed=-1), "seed"), + (lambda d: d["records"][0].update(cores=1.5), "cores"), + (lambda d: d["records"][0].update(host_class="cloud"), "host_class"), + (lambda d: d["records"][0].update(slurm_job_id=412000), "slurm_job_id"), + (lambda d: d["records"][1].update(seed=0), "duplicate key"), + (lambda d: d["records"].append(copy.deepcopy(d["records"][0])), "duplicate id"), + (lambda d: d.pop("producer_revision"), "producer_revision"), + (lambda d: d.update(generated_at="2026-10-21"), "generated_at"), + ], +) +def test_walltime_contract_breaks_are_named(mutation, needle): + doc = fixture_doc("walltime_validation_v1.json") + mutation(doc) + outcome, errors = summary.classify(doc, WALL) + assert outcome == "invalid" + assert any(needle in e for e in errors), errors + + +def test_null_derived_numbers_are_allowed_when_not_measured(): + doc = fixture_doc("walltime_validation_v1.json") + doc["records"][0].update( + predicted_ms_per_eval=None, ratio_measured_over_predicted=None, parallel_efficiency=None + ) + assert summary.classify(doc, WALL) == ("ok", []) From 6c9e885244a6509a022ca6670d2f964c2844350c Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Wed, 21 Oct 2026 18:52:43 +0100 Subject: [PATCH 3/5] pulse: designation decisions, decision_id resolution and walltime registry rows decisions.yaml entries may carry kind: designation (likelihood, regime, hardware_class, optimal, secondary, evidence, rule, supersedes), validated when the kind is set; decisions/README.md gains the template. Ingest and check resolve every v3 decision_id against the organ's decisions.yaml and receipts record current/superseded/archived counts for v3. The registry refuses a supported_schema this reader does not implement and accepts walltime-validation@1 rows. No designation record is written. Co-Authored-By: Claude Fable 5.1 --- decisions/README.md | 44 ++++++++++++++++++++ pulse/cli.py | 13 ++++-- pulse/decisions.py | 90 ++++++++++++++++++++++++++++++++++++++--- pulse/ingest.py | 62 +++++++++++++++++++++++++--- pulse/registry.py | 9 ++++- tests/test_decisions.py | 72 +++++++++++++++++++++++++++++++++ tests/test_ingest.py | 79 ++++++++++++++++++++++++++++++++++++ tests/test_registry.py | 26 ++++++++++++ 8 files changed, 381 insertions(+), 14 deletions(-) diff --git a/decisions/README.md b/decisions/README.md index 6263c67..18b4291 100644 --- a/decisions/README.md +++ b/decisions/README.md @@ -66,3 +66,47 @@ work. Distinguish decided direction from implementation already shipped.> ## History ``` + +## Designation decisions (`kind: designation`) + +A designation is the human's choice of the optimal and secondary setups for one +likelihood, regime and hardware class. It is a decision like any other (one +canonical record, one index row), with extra fields on its `decisions.yaml` +entry so `profiling-summary@3` records can cite it: + +| Field | Meaning | +|---|---| +| `kind` | `designation`. Without it, the designation fields below are refused. | +| `likelihood` | `/`, e.g. `imaging/delaunay`; `dataset_type` is one of the ten in REFERENCE.md. | +| `regime` | The named size bin the rule branches on, e.g. `standard` or `n_vis_1e5_1e7`. | +| `hardware_class` | `cpu_many_core`, `gpu_a100` or `laptop`. | +| `optimal` | One canonical `setup_id`, `___[_]`. | +| `secondary` | A list of canonical `setup_id`s (may be empty), distinct from `optimal`. | +| `evidence` | A non-empty list of the record ids the decision rests on. | +| `rule` | One sentence, in terms of dataset properties, that an agent can apply from data alone. | +| `supersedes` | The id of the designation this one replaces, or `null`. | + +A record with `class: optimal` must carry a `decision_id`, and `pyauto-pulse +check` resolves it against this index. Pulse never writes a designation from +timings: the producer declares the class, the human makes the decision, and +the record cites it. Index entry template (not a published decision): + +```yaml +- id: -- + title: + date: 'YYYY-MM-DD' + url: https://github.com/PyAutoLabs/PyAutoPulse/blob/main/decisions/.md + kind: designation + likelihood: / + regime: + hardware_class: + optimal: + secondary: [, ...] + evidence: [, ...] + rule: + supersedes: null +``` + +The canonical `decisions/.md` record uses the template above; its +Decision section states the optimal and secondary setups and the rule, and its +Evidence section links the cited records at pinned commits. diff --git a/pulse/cli.py b/pulse/cli.py index eec3679..65599b7 100644 --- a/pulse/cli.py +++ b/pulse/cli.py @@ -198,16 +198,23 @@ def cmd_check(args) -> int: print(f"FAIL {key}: receipt does not record the snapshot commit") continue # The committed snapshot must still validate under this reader. - outcome, errors = summary_mod.classify( - s.doc, (s.instance.schema_name, s.instance.schema_version) + outcome, errors = ingest.classify( + s.doc, (s.instance.schema_name, s.instance.schema_version), args.out ) if outcome != "ok": problems.append(f"{key}: snapshot {outcome}: {'; '.join(errors)}") print(f"FAIL {key}: snapshot {outcome}: {'; '.join(errors[:5])}") continue + counts = ingest.lifecycle_counts(s.doc) + lifecycle = ( + f" ({counts['current']} current, {counts['superseded']} superseded, " + f"{counts['archived']} archived)" + if counts + else "" + ) print( f"ok {key}: {s.doc['schema']}@{s.doc['version']} valid, " - f"{len(board.records(s))} records, {len(board.comparisons(s))} comparisons at " + f"{len(board.records(s))} records{lifecycle}, {len(board.comparisons(s))} comparisons at " f"{(s.commit or 'unknown')[:12]} ({mode if s.source == 'remote' else 'local read'})" ) out = Path(args.out) diff --git a/pulse/decisions.py b/pulse/decisions.py index 0bf9dc0..56f0f1e 100644 --- a/pulse/decisions.py +++ b/pulse/decisions.py @@ -1,4 +1,11 @@ -"""Flat links to human-authored campaign decisions; never infer conclusions.""" +"""Flat links to human-authored campaign decisions; never infer conclusions. + +An entry may carry ``kind: designation``: the human's choice of the optimal and +secondary setups for one likelihood, regime and hardware class. A +``profiling-summary@3`` record with ``class: optimal`` names such an entry in +its ``decision_id``; Pulse checks that the id resolves and never derives a +designation from timings. +""" import hashlib import html @@ -10,9 +17,65 @@ import yaml -from pulse import ORGAN_ROOT +from pulse import ORGAN_ROOT, catalogue REPO = "PyAutoPulse" +BASE_FIELDS = {"id", "title", "date", "url"} +DESIGNATION_FIELDS = { + "likelihood", + "regime", + "hardware_class", + "optimal", + "secondary", + "evidence", + "rule", + "supersedes", +} +HARDWARE_CLASSES = {"cpu_many_core", "gpu_a100", "laptop"} +_SETUP_ID = re.compile( + "^({backend})_({device})_({operator})_({precision})(_({transformer}))?$".format( + **{ + k: "|".join(sorted(v - {"na"} if k == "transformer" else v)) + for k, v in catalogue.SETUP_CHOICES.items() + } + ) +) +_LIKELIHOOD = re.compile( + "^(" + "|".join(sorted(catalogue.DATASET_TYPES)) + r")/[a-z0-9]+(?:_[a-z0-9]+)*$" +) + + +def _line(value) -> bool: + return isinstance(value, str) and bool(value.strip()) and not any(ord(c) < 32 for c in value) + + +def _designation(row: dict) -> None: + """Validate the designation fields of one ``kind: designation`` entry.""" + key = row["id"] + missing = sorted(DESIGNATION_FIELDS - set(row)) + if missing: + raise ValueError(f"Designation {key} lacks {', '.join(missing)}") + if not (isinstance(row["likelihood"], str) and _LIKELIHOOD.match(row["likelihood"])): + raise ValueError(f"Designation {key}: likelihood must be /") + if not _line(row["regime"]) or not _line(row["rule"]): + raise ValueError(f"Designation {key}: regime and rule must be single-line strings") + if row["hardware_class"] not in HARDWARE_CLASSES: + raise ValueError( + f"Designation {key}: hardware_class must be one of {sorted(HARDWARE_CLASSES)}" + ) + secondary = row["secondary"] + if not isinstance(secondary, list): + raise ValueError(f"Designation {key}: secondary must be a list of setup_ids") + for setup in [row["optimal"], *secondary]: + if not (isinstance(setup, str) and _SETUP_ID.match(setup)): + raise ValueError(f"Designation {key}: {setup!r} is not a canonical setup_id") + if row["optimal"] in secondary or len(set(secondary)) != len(secondary): + raise ValueError(f"Designation {key}: optimal and secondary setups must be distinct") + evidence = row["evidence"] + if not (isinstance(evidence, list) and evidence and all(_line(e) for e in evidence)): + raise ValueError(f"Designation {key}: evidence must be a non-empty list of record ids") + if row["supersedes"] is not None and not _line(row["supersedes"]): + raise ValueError(f"Designation {key}: supersedes must be a decision id or null") def load(root: Path = ORGAN_ROOT) -> list[dict]: @@ -30,11 +93,18 @@ def load(root: Path = ORGAN_ROOT) -> list[dict]: seen = set() rows = [] for row in document["decisions"]: - if not isinstance(row, dict) or set(row) != {"id", "title", "date", "url"}: + if not isinstance(row, dict): raise ValueError("Decision requires id, title, date and url") + kind = row.get("kind") + allowed = BASE_FIELDS | ({"kind"} | DESIGNATION_FIELDS if kind == "designation" else set()) + if not BASE_FIELDS <= set(row) or not set(row) <= allowed: + raise ValueError( + "Decision requires id, title, date and url" + + ("" if kind in (None, "designation") else "; kind must be designation") + ) if any( - not isinstance(v, str) or not v.strip() or any(ord(c) < 32 for c in v) - for v in row.values() + not isinstance(row[k], str) or not row[k].strip() or any(ord(c) < 32 for c in row[k]) + for k in BASE_FIELDS ): raise ValueError("Decision fields must be nonempty single-line strings; quote dates") key = row["id"] @@ -70,10 +140,20 @@ def load(root: Path = ORGAN_ROOT) -> list[dict]: or f"Date: {row['date']}\n" not in content ): raise ValueError(f"Decision title/date differ from index: {key}") + if kind == "designation": + _designation(row) rows.append(dict(row)) + for row in rows: + if row.get("supersedes") is not None and row["supersedes"] not in seen: + raise ValueError(f"Designation {row['id']} supersedes unknown {row['supersedes']}") return sorted(rows, key=lambda row: (row["date"], row["id"]), reverse=True) +def ids(root: Path = ORGAN_ROOT) -> set[str]: + """Every decision id in the validated index (``ValueError`` when it is invalid).""" + return {row["id"] for row in load(root)} + + def marker(rows): digest = hashlib.sha256(json.dumps(rows, sort_keys=True).encode()).hexdigest() return f"" diff --git a/pulse/ingest.py b/pulse/ingest.py index c7c2f34..9ffc281 100644 --- a/pulse/ingest.py +++ b/pulse/ingest.py @@ -14,6 +14,11 @@ record ``refreshed_at`` for each successful observation, so real refreshes can commit metadata even when evidence is unchanged. Offline replay retains the recorded observation time; a failed attempt exposes no successful refresh. +A ``profiling-summary@3`` document passes one repo-aware step after the +document-only validator: every ``decision_id`` it names must resolve in this +organ's ``decisions.yaml`` (``classify``); its receipt also records the +producer's current/superseded/archived counts. + A local ``--from`` read reports the checkout's revision and dirty state, is labelled "local checkout", and writes no receipt or snapshot: it cannot masquerade as @@ -32,7 +37,7 @@ from datetime import UTC, datetime from pathlib import Path -from pulse import ORGAN_ROOT +from pulse import ORGAN_ROOT, catalogue, decisions from pulse import summary as summary_mod from pulse.registry import Instance @@ -87,6 +92,49 @@ def fetch_at(github: str, sha: str, path: str) -> bytes: return _get(RAW_FILE.format(github=github, sha=sha, path=path)) +# -------------------------------------------------------------- classify --- + + +def _lifecycle(doc) -> bool: + return ( + isinstance(doc, dict) + and doc.get("schema") == "profiling-summary" + and doc.get("version") == 3 + ) + + +def unresolved_decisions(doc: dict, root: Path = ORGAN_ROOT) -> list[str]: + """Errors for v3 ``decision_id`` values absent from ``root``'s decisions.yaml.""" + wanted = catalogue.decision_ids(doc) + if not wanted: + return [] + try: + known = decisions.ids(Path(root)) + except ValueError as exc: + return [f"decision index unreadable, cannot resolve decision_id: {exc}"] + return [ + f"decision_id {d!r} does not resolve in decisions.yaml (a designation decision is " + "required before a record is declared optimal)" + for d in wanted + if d not in known + ] + + +def classify(doc, wanted, root: Path = ORGAN_ROOT) -> tuple[str, list[str]]: + """``summary.classify`` plus the repo-aware decision resolution for v3.""" + outcome, errors = summary_mod.classify(doc, wanted) + if outcome == "ok" and _lifecycle(doc): + errors = unresolved_decisions(doc, root) + if errors: + outcome = "invalid" + return outcome, errors + + +def lifecycle_counts(doc) -> dict | None: + """``{current, superseded, archived}`` for a v3 document, else None.""" + return catalogue.lifecycle_counts(doc) if _lifecycle(doc) else None + + # -------------------------------------------------------------- snapshot --- @@ -211,7 +259,7 @@ def ingest( out_dir = Path(out_dir) wanted = (inst.schema_name, inst.schema_version) if local_path is not None: - return _ingest_local(inst, Path(local_path).expanduser(), wanted) + return _ingest_local(inst, Path(local_path).expanduser(), wanted, out_dir) if offline: return _from_files(inst, out_dir) @@ -225,7 +273,7 @@ def ingest( except ValueError as exc: outcome, errors = "invalid", [f"{inst.summary_path} is not JSON ({exc})"] else: - outcome, errors = summary_mod.classify(doc, wanted) + outcome, errors = classify(doc, wanted, out_dir) except FetchError as exc: outcome, errors = "unavailable", [str(exc)] @@ -262,6 +310,10 @@ def ingest( "errors": errors, "cached_from": last_good.get("fetched_at") if last_good and outcome != "ok" else None, } + # Lifecycle counts exist only for v3, so earlier receipts keep their shape. + counts = lifecycle_counts(doc) + if counts is not None: + receipt["lifecycle"] = counts # Preserve capture history while recording a real successful observation. prior = read_receipt(out_dir, inst.instance) strip = lambda d: {k: v for k, v in d.items() if k not in ("fetched_at", "refreshed_at")} # noqa: E731 @@ -282,7 +334,7 @@ def _git(checkout: Path, *args: str) -> str | None: return proc.stdout if proc.returncode == 0 else None -def _ingest_local(inst: Instance, path: Path, wanted) -> Snapshot: +def _ingest_local(inst: Instance, path: Path, wanted, root: Path = ORGAN_ROOT) -> Snapshot: file = path / inst.summary_path if path.is_dir() else path view = Snapshot(inst, "ok", "local", checkout=str(file.parent), fetched_at=utc_now()) view.attempt_at = view.fetched_at @@ -304,7 +356,7 @@ def _ingest_local(inst: Instance, path: Path, wanted) -> Snapshot: view.outcome, view.errors = "invalid", [f"{file} is not JSON ({exc})"] view.doc = None return view - view.outcome, view.errors = summary_mod.classify(view.doc, wanted) + view.outcome, view.errors = classify(view.doc, wanted, root) if view.failed: view.doc = None else: diff --git a/pulse/registry.py b/pulse/registry.py index d3e8e69..ef041e1 100644 --- a/pulse/registry.py +++ b/pulse/registry.py @@ -15,7 +15,7 @@ import yaml -from pulse import ORGAN_ROOT +from pulse import ORGAN_ROOT, summary REGISTRY_PATH = ORGAN_ROOT / "registry.yaml" REGISTRY_SCHEMA = 1 @@ -198,6 +198,13 @@ def validate(data, body_map: dict[str, dict] | None = None) -> list[str]: schema = row.get("supported_schema") if isinstance(schema, str) and not _SCHEMA.match(schema): problems.append(f"{where}: supported_schema must be @") + elif isinstance(schema, str): + # A row pins one exact contract this reader implements, e.g. + # profiling-summary@3 or the full-fit axis walltime-validation@1. + got = (_SCHEMA.match(schema)[1], int(_SCHEMA.match(schema)[2])) + if got not in summary.SUPPORTED: + known = ", ".join(f"{n}@{v}" for n, v in sorted(summary.SUPPORTED)) + problems.append(f"{where}: supported_schema {schema} is not read here ({known})") url = row.get("dashboard_url") if isinstance(url, str) and not url.startswith("https://"): problems.append(f"{where}: dashboard_url must be an https:// URL") diff --git a/tests/test_decisions.py b/tests/test_decisions.py index 5654bb3..a76c2d2 100644 --- a/tests/test_decisions.py +++ b/tests/test_decisions.py @@ -94,3 +94,75 @@ def invalid(): monkeypatch.setattr(decisions, "load", invalid) assert cli.main(["check", "--offline"]) == 1 assert "FAIL decisions: broken decision index" in capsys.readouterr().out + + +# ----------------------------------------------------------- designation --- + + +def designation(key="imaging-delaunay-gpu", **extra): + entry = row(key) + entry.update( + kind="designation", + likelihood="imaging/delaunay", + regime="standard", + hardware_class="gpu_a100", + optimal="jax_gpu_dense_fp64", + secondary=["jax_gpu_sparse_fp64"], + evidence=["imaging-delaunay-hst-a100-runtime"], + rule="Any imaging Delaunay fit on an A100 uses the dense JAX route.", + supersedes=None, + ) + entry.update(extra) + return entry + + +def test_designation_entries_validate_and_resolve(tmp_path): + first = designation("first", date="2026-10-01") + second = designation("second", supersedes="first", secondary=[]) + write(tmp_path, [first, second, row("plain")]) + loaded = decisions.load(tmp_path) + assert {r["id"] for r in loaded} == {"first", "second", "plain"} + assert decisions.ids(tmp_path) == {"first", "second", "plain"} + assert next(r for r in loaded if r["id"] == "second")["supersedes"] == "first" + assert "NNLS <solver>" in decisions.render_html(loaded) + + +@pytest.mark.parametrize( + "mutation,needle", + [ + ({"kind": "ranking"}, "kind must be designation"), + ({"hardware_class": "a100"}, "hardware_class"), + ({"likelihood": "imaging"}, "likelihood"), + ({"likelihood": "pixelized/delaunay"}, "likelihood"), + ({"optimal": "jax_gpu_dense"}, "canonical setup_id"), + ({"optimal": "jax_gpu_dense_fp64_na"}, "canonical setup_id"), + ({"secondary": ["numba_cpu_sparse_fp32"]}, "canonical setup_id"), + ({"secondary": "jax_cpu_dense_fp64"}, "secondary must be a list"), + ({"secondary": ["jax_gpu_dense_fp64"]}, "distinct"), + ({"evidence": []}, "evidence"), + ({"rule": "two\nlines"}, "single-line"), + ({"supersedes": "missing"}, "supersedes unknown"), + ], +) +def test_invalid_designations_rejected(tmp_path, mutation, needle): + entry = designation(**mutation) + write(tmp_path, [entry]) + with pytest.raises(ValueError, match=needle): + decisions.load(tmp_path) + + +def test_designation_fields_require_the_kind(tmp_path): + entry = designation() + entry.pop("kind") + write(tmp_path, [entry]) + with pytest.raises(ValueError, match="requires id, title, date and url"): + decisions.load(tmp_path) + entry = designation() + entry.pop("rule") + write(tmp_path, [entry]) + with pytest.raises(ValueError, match="lacks rule"): + decisions.load(tmp_path) + + +def test_the_live_index_holds_no_designation_yet(): + assert all("kind" not in r for r in decisions.load()) diff --git a/tests/test_ingest.py b/tests/test_ingest.py index 25b1aa7..4c784cf 100644 --- a/tests/test_ingest.py +++ b/tests/test_ingest.py @@ -118,3 +118,82 @@ def test_a_local_read_reports_revision_and_dirty_state_and_writes_nothing(lens, (checkout / "scratch.txt").write_text("edit") assert ingest.ingest(lens, out, local_path=checkout).dirty is True assert not (out / "receipts").exists() and not (out / "snapshots").exists() + + +# ------------------------------------------- v3 decision resolution, counts --- + + +def _v3_lens(registry_file, fake_mind, schema="profiling-summary@3"): + import yaml + + config = yaml.safe_load(registry_file.read_text()) + config["instances"][0]["supported_schema"] = schema + registry_file.write_text(yaml.safe_dump(config)) + return registry.get(registry.load(registry_file, mind=fake_mind), "lens") + + +def _decisions(root, ids): + import yaml + + rows = [ + { + "id": key, + "title": f"Designate {key}", + "date": "2026-10-21", + "url": f"https://github.com/PyAutoLabs/PyAutoPulse/blob/main/decisions/{key}.md", + } + for key in ids + ] + root.mkdir(parents=True, exist_ok=True) + (root / "decisions.yaml").write_text(yaml.safe_dump({"decisions": rows})) + (root / "decisions").mkdir(exist_ok=True) + for row in rows: + (root / "decisions" / f"{row['id']}.md").write_text( + f"# {row['title']}\n\nDate: {row['date']}\n" + ) + + +def test_v3_decision_ids_resolve_and_the_receipt_counts_lifecycle( + registry_file, fake_mind, web, tmp_path +): + lens = _v3_lens(registry_file, fake_mind) + _decisions(tmp_path, ["fixture-designation"]) + web.publish(GH, SHA_A, PATH, fixture_doc("setup_summary_v3.json")) + s = ingest.ingest(lens, tmp_path, now="2026-10-21T00:00:00Z") + assert s.outcome == "ok", s.errors + receipt = ingest.read_receipt(tmp_path, "lens") + assert (receipt["schema"], receipt["version"]) == ("profiling-summary", 3) + assert receipt["lifecycle"] == {"current": 5, "superseded": 1, "archived": 3} + + +def test_v3_unresolved_decision_is_invalid_and_keeps_the_last_good( + registry_file, fake_mind, web, tmp_path +): + lens = _v3_lens(registry_file, fake_mind) + _decisions(tmp_path, ["fixture-designation"]) + web.publish(GH, SHA_A, PATH, fixture_doc("setup_summary_v3.json")) + ingest.ingest(lens, tmp_path, now="2026-10-21T00:00:00Z") + _decisions(tmp_path, ["some-other-decision"]) + (tmp_path / "decisions/fixture-designation.md").unlink() + web.publish(GH, SHA_B, PATH, fixture_doc("setup_summary_v3.json")) + s = ingest.ingest(lens, tmp_path, now="2026-10-22T00:00:00Z") + assert s.outcome == "invalid" and s.cached and s.commit == SHA_A + assert any("'fixture-designation' does not resolve" in e for e in s.errors) + + +def test_v3_without_a_decision_index_cannot_resolve(registry_file, fake_mind, web, tmp_path): + lens = _v3_lens(registry_file, fake_mind) + web.publish(GH, SHA_A, PATH, fixture_doc("setup_summary_v3.json")) + s = ingest.ingest(lens, tmp_path, now="2026-10-21T00:00:00Z") + assert s.outcome == "invalid" + assert "decision index unreadable" in s.errors[0] + + +def test_v2_and_walltime_receipts_keep_their_shape(registry_file, fake_mind, web, tmp_path): + lens = _v3_lens(registry_file, fake_mind, "walltime-validation@1") + web.publish(GH, SHA_A, PATH, fixture_doc("walltime_validation_v1.json")) + s = ingest.ingest(lens, tmp_path, now="2026-10-21T00:00:00Z") + assert s.outcome == "ok", s.errors + receipt = ingest.read_receipt(tmp_path, "lens") + assert receipt["schema"] == "walltime-validation" and "lifecycle" not in receipt + assert receipt["record_count"] == 2 diff --git a/tests/test_registry.py b/tests/test_registry.py index bb84d42..25548ad 100644 --- a/tests/test_registry.py +++ b/tests/test_registry.py @@ -95,3 +95,29 @@ def test_get_names_the_known_instances(registry_file, fake_mind): assert registry.get(instances, "lens").repo == "autolens_profiling" with pytest.raises(registry.RegistryError, match="known: lens"): registry.get(instances, "nope") + + +WALLTIME_ROW = { + "instance": "lens-walltime", + "repo": "autolens_profiling", + "summary_path": "dashboard/walltime.json", + "supported_schema": "walltime-validation@1", + "dashboard_url": "https://pyautolabs.github.io/autolens_profiling/", + "library_refs": ["PyAutoLens"], + "cortex_project": None, +} + + +def test_a_walltime_validation_row_registers_as_its_own_instance(body_map, tmp_path, fake_mind): + v3 = dict(LENS_ROW, supported_schema="profiling-summary@3") + assert registry.validate(_doc(v3, WALLTIME_ROW), body_map) == [] + path = tmp_path / "walltime.yaml" + path.write_text(yaml.safe_dump(_doc(WALLTIME_ROW))) + (inst,) = registry.load(path, mind=fake_mind) + assert (inst.schema_name, inst.schema_version) == ("walltime-validation", 1) + + +@pytest.mark.parametrize("schema", ["walltime-validation@2", "profiling-summary@4", "nope@1"]) +def test_a_schema_this_reader_does_not_implement_is_refused(body_map, schema): + problems = registry.validate(_doc(dict(LENS_ROW, supported_schema=schema)), body_map) + assert any("is not read here" in p for p in problems), problems From a018ca1f9dc364ee15b9ab8d3998988d72b18aee Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Wed, 21 Oct 2026 18:56:41 +0100 Subject: [PATCH 4/5] pulse: Class-first board for profiling-summary@3 and the wall-time table A v3 instance renders Class (Optimal default) -> Likelihood (dataset type, variant) -> Instrument -> Hardware class -> Measurement over server-rendered rows: optimal + current by default, superseded only with History, archived counted and never rendered, and a Coverage panel of optimal planned cells with no current record. The v2 browser path is untouched. Status lines and the census item carry current/superseded/archived counts; a walltime-validation instance renders its full-fit records as its own table under Profiling Results. Co-Authored-By: Claude Fable 5.1 --- pulse/board.py | 116 +++++++++++++++++++-- pulse/setup_browser.css | 25 +++++ pulse/setup_browser.js | 38 ++++++- pulse/setup_browser.py | 197 +++++++++++++++++++++++++++++++++++- tests/browser_fixture.py | 23 +++++ tests/browser_setup.cjs | 58 +++++++++++ tests/test_board.py | 65 ++++++++++++ tests/test_setup_browser.py | 55 ++++++++++ 8 files changed, 568 insertions(+), 9 deletions(-) diff --git a/pulse/board.py b/pulse/board.py index a4826b0..00d5793 100644 --- a/pulse/board.py +++ b/pulse/board.py @@ -15,6 +15,11 @@ manual handoff. Pairs the reader refuses (axis mismatch, changed hardware, missing endpoint) are listed apart with the reason and no ratio. +A ``profiling-summary@3`` instance adds its producer-declared lifecycle counts +(current / superseded / archived) to its status lines and renders the +Class-first setup browser; a ``walltime-validation@1`` instance renders its +full-fit records as their own table under Profiling Results. + The board never recomputes a ratio, never combines records whose axis or hardware differs, never ranks cells across projects and never emits a release verdict. Its output is deterministic: the pages carry no wall-clock @@ -31,7 +36,7 @@ from datetime import UTC, datetime from pathlib import Path -from pulse import ORGAN_ROOT, campaigns, decisions, setup_browser +from pulse import ORGAN_ROOT, campaigns, catalogue, decisions, setup_browser from pulse import summary as summary_mod from pulse.ingest import Snapshot @@ -60,6 +65,28 @@ def _days(later: str | None, earlier: str | None) -> int | None: return max(0, (a - b).days) +def is_walltime(s: Snapshot) -> bool: + return (s.doc or {}).get("schema") == "walltime-validation" + + +def is_catalogue(doc: dict | None) -> bool: + """A setup catalogue (v2/v3) or the wall-time axis: the browser layout.""" + doc = doc or {} + return doc.get("schema") == "walltime-validation" or doc.get("version") in (2, 3) + + +def lifecycle(s: Snapshot) -> str | None: + """``N current · N superseded · N archived`` for a v3 capture, else None.""" + doc = s.doc or {} + if doc.get("schema") != "profiling-summary" or doc.get("version") != 3: + return None + c = catalogue.lifecycle_counts(doc) + return ( + f"{c['current']} current · {c['superseded']} superseded · {c['archived']} archived " + "(producer-declared)" + ) + + def records(s: Snapshot) -> list[dict]: return list((s.doc or {}).get("records") or []) @@ -119,6 +146,8 @@ def qualification(s: Snapshot) -> str: recs, comps = records(s), comparisons(s) if not recs: return "no measurements" + if is_walltime(s): + return f"{len(recs)} full-fit records (walltime-validation carries no qualification flag)" q = summary_mod.qualified_records(s.doc) cq = sum(1 for c in comps if c.get("qualified") is True) return f"{q}/{len(recs)} records · {cq}/{len(comps)} comparisons qualified" @@ -280,6 +309,8 @@ def _status_lines(s: Snapshot, now: str | None = None) -> list[str]: f"- **Freshness:** {freshness(s, now)} (evidence {evidence(s)})", f"- **Qualification:** {qualification(s)}", ] + if lifecycle(s): + out.append(f"- **Lifecycle:** {lifecycle(s)}") return out @@ -337,9 +368,75 @@ def _md_detail(s: Snapshot, now: str | None = None) -> list[str]: return out +WALLTIME_COLUMNS = ( + ( + "Likelihood", + lambda r: ( + f"{(r.get('likelihood') or {}).get('dataset_type')}/" + f"{(r.get('likelihood') or {}).get('variant')}" + ), + ), + ("Instrument", lambda r: r.get("instrument")), + ("Setup", lambda r: r.get("setup_id")), + ("Hardware class", lambda r: r.get("host_class")), + ("Model variant", lambda r: r.get("model_variant")), + ("Sampler", lambda r: r.get("sampler")), + ("Seed", lambda r: r.get("seed")), + ("Cores", lambda r: r.get("cores")), + ("Wall s", lambda r: r.get("wall_s")), + ("Likelihood evals", lambda r: r.get("likelihood_evals")), + ("Implied ms/eval", lambda r: r.get("implied_ms_per_eval")), + ("Predicted ms/eval", lambda r: r.get("predicted_ms_per_eval")), + ("Measured/predicted", lambda r: r.get("ratio_measured_over_predicted")), + ("Parallel efficiency", lambda r: r.get("parallel_efficiency")), + ("Parity", lambda r: r.get("parity_status")), + ("Reference runtime record", lambda r: r.get("reference_runtime_record_id")), + ("Release", lambda r: r.get("release_pin")), +) +WALLTIME_NOTE = ( + "Full-fit measurements: one sampler run per row. Every derived number is the " + "producer's; Pulse computes no ratio and never reads a full-fit time as a " + "per-likelihood measurement." +) + + +def _cell(value) -> str: + return "—" if value is None else str(value) + + +def walltime_markdown(s: Snapshot) -> list[str]: + name = s.instance.repo + out = [ + f"
{html.escape(name)} — wall-time validation", + "", + WALLTIME_NOTE, + "", + "| " + " | ".join(t for t, _ in WALLTIME_COLUMNS) + " |", + "|" + "---|" * len(WALLTIME_COLUMNS), + ] + for r in records(s): + out.append("| " + " | ".join(_md_escape(_cell(f(r))) for _, f in WALLTIME_COLUMNS) + " |") + return out + ["", "
", ""] + + +def walltime_html(s: Snapshot) -> str: + head = "".join(f"{_e(t)}" for t, _ in WALLTIME_COLUMNS) + body = "".join( + "" + "".join(f"{_e(_cell(f(r)))}" for _, f in WALLTIME_COLUMNS) + "" + for r in records(s) + ) + return ( + f'
' + f"

{_e(s.instance.repo)}: wall-time validation

{_e(WALLTIME_NOTE)}

" + f"
{head}{body}" + "
" + ) + + def render_markdown(views, now: str | None = None, campaign_data: dict | None = None) -> str: now = now or _utc_now() - if any((s.doc or {}).get("version") == 2 for s in views): + if any(is_catalogue(s.doc) for s in views): out = [ "# PyAutoPulse — profiling dashboard", "", @@ -350,7 +447,9 @@ def render_markdown(views, now: str | None = None, campaign_data: dict | None = for snapshot in views: if snapshot.failed or snapshot.cached: out += [f"**{snapshot.instance.instance}: {integrity(snapshot)}**", ""] - if (snapshot.doc or {}).get("version") == 2: + if is_walltime(snapshot): + out += walltime_markdown(snapshot) + elif (snapshot.doc or {}).get("version") in (2, 3): out += setup_browser.markdown(snapshot) out += [ "
Capture, qualification and legacy diagnostics", @@ -444,7 +543,8 @@ def _html_notions(s: Snapshot, now: str | None = None) -> str: f"
Integrity{_e(integrity(s))}
" f"
Freshness{_e(freshness(s, now))} (evidence {_e(evidence(s))}){live}
" f"
Qualification{_e(qualification(s))}
" - "" + + (f"
Lifecycle{_e(lifecycle(s))}
" if lifecycle(s) else "") + + "" ) @@ -557,8 +657,11 @@ def render_html(views, now: str | None = None, campaign_data: dict | None = None content.append( f'

{_e(name)}: {_e(integrity(snapshot))}. Displayed evidence retains its captured source.

' ) + if is_walltime(snapshot): + content.append(walltime_html(snapshot)) + continue content.append(setup_browser.render(snapshot)) - if (snapshot.doc or {}).get("version") != 2: + if (snapshot.doc or {}).get("version") not in (2, 3): content.append( f"

{_e(name)}: setup catalogue unavailable; use the project dashboard for original evidence.

" ) @@ -713,7 +816,7 @@ def _instance_items(s: Snapshot, now: str) -> tuple[list, list, list]: "actions": [open_action], } ) - else: + elif not is_walltime(s): unq = len(recs) - summary_mod.qualified_records(doc) if unq: yellow.append( @@ -792,6 +895,7 @@ def _instance_items(s: Snapshot, now: str) -> tuple[list, list, list]: "text": ( f"{key}: {len(recs)} records · {len(comparisons(s))} comparisons · " f"{len(lims)} limitations · {summary_mod.qualified_records(doc)} qualified" + + (f" · {lifecycle(s)}" if lifecycle(s) else "") ), "url": inst.dashboard_url, "prompt": None, diff --git a/pulse/setup_browser.css b/pulse/setup_browser.css index 740ae65..255b742 100644 --- a/pulse/setup_browser.css +++ b/pulse/setup_browser.css @@ -318,3 +318,28 @@ button.retry { .timing-diagnostics { margin-top: 0.8rem; } +/* profiling-summary@3: Class-first lifecycle browser. */ +.lifecycle-selectors { + grid-template-columns: repeat(auto-fit, minmax(9rem, 1fr)); +} +.lifecycle-selectors .history-toggle { + align-self: end; + font-weight: 500; +} +.lifecycle-table tr[hidden], +.lifecycle-browser [hidden] { + display: none; +} +.lifecycle-table td, +.lifecycle-table th { + font-size: 0.8rem; + padding: 0.35rem 0.45rem; + vertical-align: top; +} +.lifecycle-coverage { + margin: 1rem 0; +} +.lifecycle-coverage summary { + cursor: pointer; + font-weight: 600; +} diff --git a/pulse/setup_browser.js b/pulse/setup_browser.js index 15583ff..550960f 100644 --- a/pulse/setup_browser.js +++ b/pulse/setup_browser.js @@ -1,8 +1,44 @@ -/* Pulse v2 reader UI; conventions shared with the project setup browser. +/* Pulse v2/v3 reader UI; conventions shared with the project setup browser. * Consumes captured producer data only. No cross-setup joins or scientific judgement. */ (() => { "use strict"; + // profiling-summary@3: the producer declares class and status. Filtering only + // shows or hides server-rendered rows; nothing is ranked, joined or chosen. + function lifecycle(root) { + const filters = [...root.querySelectorAll("select[data-filter]")]; + const history = root.querySelector('input[data-filter="history"]'); + const rows = [...root.querySelectorAll("tr[data-record]")]; + const empty = root.querySelector('[data-id="no-rows"]'); + function apply() { + let shown = 0; + for (const row of rows) { + const status = row.getAttribute("data-status"); + const visible = + (status === "current" || + (history.checked && status === "superseded")) && + filters.every((select) => { + const key = select.dataset.filter; + const value = select.value; + if (key === "class" && value === "all") return true; + return ( + !value || + row.getAttribute("data-" + key.replaceAll("_", "-")) === value + ); + }); + row.hidden = !visible; + if (visible) shown += 1; + } + empty.hidden = shown > 0; + } + for (const control of [...filters, history]) + control.addEventListener("change", apply); + apply(); + } for (const root of document.querySelectorAll("[data-setup-browser]")) { + if (root.hasAttribute("data-lifecycle")) { + lifecycle(root); + continue; + } const capture = JSON.parse( root.querySelector(".setup-payload").textContent, ); diff --git a/pulse/setup_browser.py b/pulse/setup_browser.py index d6a037b..e4b1daa 100644 --- a/pulse/setup_browser.py +++ b/pulse/setup_browser.py @@ -1,4 +1,11 @@ -"""Render setup navigation from captured v2 evidence; never select scientific truth.""" +"""Render setup navigation from captured v2/v3 evidence; never select scientific truth. + +A v3 (lifecycle) catalogue renders a Class-first browser: Class (Optimal by +default) → Likelihood (dataset type → variant) → Instrument → Hardware class → +Measurement, showing ``status: current`` rows unless History is ticked. The +producer declares class and status; the board ranks nothing, picks no fastest +row and shows archived records only as a count. +""" from __future__ import annotations @@ -9,7 +16,7 @@ from pathlib import Path from urllib.parse import quote -from pulse import ORGAN_ROOT +from pulse import ORGAN_ROOT, catalogue def theme(): @@ -34,8 +41,157 @@ def assets(): ) +CLASS_OPTIONS = ( + ("optimal", "Optimal"), + ("secondary", "Secondary"), + ("experimental", "Experimental"), + ("all", "All"), +) +# The selector order after Class; each is a data-* attribute on every row. +FILTERS = ( + ("dataset_type", "Likelihood"), + ("variant", "Variant"), + ("instrument", "Instrument"), + ("host_class", "Hardware class"), + ("measurement", "Measurement"), +) + + +def lifecycle(doc) -> bool: + return isinstance(doc, dict) and doc.get("version") == 3 + + +def _row_fields(record): + likelihood = record.get("likelihood") or {} + return { + "class": record.get("class"), + "status": record.get("status"), + "dataset_type": likelihood.get("dataset_type"), + "variant": likelihood.get("variant"), + "instrument": record.get("instrument"), + "host_class": record.get("host_class"), + "measurement": record.get("measurement"), + } + + +def default_visible(record) -> bool: + """The default view: Optimal class, current status, no other filter.""" + return record.get("class") == "optimal" and record.get("status") == "current" + + +def not_measured(doc) -> list[dict]: + """Optimal planned cells with no current record on their key (producer-declared).""" + current = { + catalogue.lifecycle_key(r) for r in doc.get("records") or [] if r.get("status") == "current" + } + return [ + cell + for cell in doc.get("planned_cells") or [] + if cell.get("class") == "optimal" and catalogue.lifecycle_key(cell) not in current + ] + + +def _cell_label(cell): + likelihood = cell.get("likelihood") or {} + return ( + f"{likelihood.get('dataset_type')}/{likelihood.get('variant')} · {cell.get('instrument')} · " + f"{cell.get('setup_id')} · {cell.get('host_class')} · {cell.get('model_variant')} · " + f"{cell.get('measurement')}" + ) + + +def _value(record): + values = record.get("values") or {} + metric = record.get("metric") + return f"{values.get(metric)} {record.get('unit')} ({metric})" + + +def _render_lifecycle(snapshot): + doc = snapshot.doc + instance = snapshot.instance + e = lambda v: html.escape(str(v), quote=True) # noqa: E731 + records = [r for r in doc.get("records") or [] if isinstance(r, dict)] + counts = catalogue.lifecycle_counts(doc) + fields = [ + f'" + ] + for key, title in FILTERS: + values = sorted({str(_row_fields(r)[key]) for r in records}) + fields.append( + f'" + ) + fields.append( + '" + ) + rows = [] + for r in records: + attrs = "".join(f' data-{k.replace("_", "-")}="{e(v)}"' for k, v in _row_fields(r).items()) + hidden = "" if default_visible(r) else " hidden" + ev = r.get("evidence") or {} + links = [] + if ev.get("path"): + url = instance.blob_url(snapshot.commit, quote(ev["path"], safe="/")) + links.append(f'evidence') + for fig in r.get("figures") or []: + url = instance.blob_url(snapshot.commit, quote(fig["path"], safe="/")) + links.append(f'{e(fig["kind"])} figure') + likelihood = r.get("likelihood") or {} + rows.append( + f'' + f"{e(likelihood.get('dataset_type'))}/{e(likelihood.get('variant'))}" + f"{e(r.get('instrument'))}{e(r.get('regime'))}" + f"{e(r.get('setup_id'))}{e(r.get('host_class'))}" + f"{e(r.get('model_variant'))}{e(r.get('measurement'))}" + f"{e(_value(r))}{e(r.get('class'))}" + f"{e(r.get('status'))}{e(r.get('decision_id') or '—')}" + f"{' · '.join(links) or '—'}" + ) + visible = sum(default_visible(r) for r in records) + missing = not_measured(doc) + coverage = ( + f'
Coverage: ' + f"{len(missing)} optimal cell{'' if len(missing) == 1 else 's'} not measured" + + ( + "
    " + "".join(f"
  • {e(_cell_label(c))}
    • " for c in missing) + "
" + if missing + else "

Every planned optimal cell has a current record.

" + ) + + "
" + ) + label = instance.repo.removesuffix("_profiling").replace("autolens", "PyAutoLens") + return ( + f'
' + f"

{e(label)}: setups by class

" + f'
{"".join(fields)}
' + f'

{counts["current"]} current · {counts["superseded"]} ' + f"superseded (shown with History) · {counts['archived']} archived (counted, not shown). " + "Class and status are the producer's declarations; this board ranks nothing.

" + '
' + "" + "" + "" + f"{''.join(rows)}
LikelihoodInstrumentRegimeSetupHardware classModel variantMeasurementValue (producer)ClassStatusDecisionEvidence
" + f'

' + "No record matches this selection.

" + coverage + "
" + ) + + def render(snapshot): doc = snapshot.doc or {} + if lifecycle(doc): + return _render_lifecycle(snapshot) if doc.get("version") != 2: return "" instance = snapshot.instance @@ -74,8 +230,45 @@ def render(snapshot): ) +def _markdown_lifecycle(snapshot): + doc = snapshot.doc + counts = catalogue.lifecycle_counts(doc) + name = snapshot.instance.repo.removesuffix("_profiling") + lines = [ + f"
{html.escape(name)} — optimal, current setups", + "", + f"[Interactive setup browser]({snapshot.instance.dashboard_url})", + "", + f"{counts['current']} current · {counts['superseded']} superseded · " + f"{counts['archived']} archived (counted, not shown). Class and status are the " + "producer's declarations.", + "", + "| Likelihood | Instrument | Setup | Hardware class | Model variant | Measurement " + "| Value (producer) | Decision |", + "|---|---|---|---|---|---|---:|---|", + ] + esc = lambda v: html.escape(str(v)).replace("|", "\\|") # noqa: E731 + for r in doc.get("records") or []: + if not default_visible(r): + continue + likelihood = r.get("likelihood") or {} + lines.append( + f"| {esc(likelihood.get('dataset_type'))}/{esc(likelihood.get('variant'))} " + f"| {esc(r.get('instrument'))} | `{esc(r.get('setup_id'))}` | {esc(r.get('host_class'))} " + f"| {esc(r.get('model_variant'))} | {esc(r.get('measurement'))} | {esc(_value(r))} " + f"| {esc(r.get('decision_id'))} |" + ) + missing = not_measured(doc) + lines += ["", f"Coverage: {len(missing)} optimal cell(s) not measured.", ""] + lines += [f"- {esc(_cell_label(c))}" for c in missing] + lines += ["", "
", ""] + return lines + + def markdown(snapshot): doc = snapshot.doc or {} + if lifecycle(doc): + return _markdown_lifecycle(snapshot) lines = [ f"
{snapshot.instance.repo.removesuffix('_profiling')} — choose a setup", "", diff --git a/tests/browser_fixture.py b/tests/browser_fixture.py index 9dd938a..d3ba299 100644 --- a/tests/browser_fixture.py +++ b/tests/browser_fixture.py @@ -276,6 +276,29 @@ def write(destination): legacy.doc["records"] = [] (destination / "unknown-axes.html").write_text(board.render_html([legacy])) + # profiling-summary@3: the Class-first lifecycle browser beside a wall-time table. + lifecycle = copy.deepcopy(snapshots[0]) + lifecycle.doc = json.loads( + (Path(__file__).parent / "fixtures/setup_summary_v3.json").read_text() + ) + walltime_instance = Instance( + "walltime", + "autolens_inference", + "dashboard/walltime.json", + "walltime-validation@1", + "https://example.invalid", + github="PyAutoLabs/autolens_inference", + ) + walltime = Snapshot( + walltime_instance, + "ok", + "remote", + json.loads((Path(__file__).parent / "fixtures/walltime_validation_v1.json").read_text()), + "b" * 40, + "2026-10-05T00:00:00Z", + ) + (destination / "lifecycle.html").write_text(board.render_html([lifecycle, walltime])) + if __name__ == "__main__": write(Path(sys.argv[1])) diff --git a/tests/browser_setup.cjs b/tests/browser_setup.cjs index e8dfe81..ae9a9ac 100644 --- a/tests/browser_setup.cjs +++ b/tests/browser_setup.cjs @@ -623,6 +623,64 @@ const { execFileSync } = require("node:child_process"); ), ); } + // profiling-summary@3: Class first, Optimal + current by default, + // superseded rows only with History, archived never rendered. + await page.setViewportSize({ width: 1280, height: 1000 }); + await page.goto(base + "/lifecycle.html"); + const life = page.locator("[data-lifecycle]"); + await life.evaluate((el) => { + for (let p = el.parentElement; p; p = p.parentElement) + if (p.tagName === "DETAILS") p.open = true; + }); + const firstSelect = life.locator("select").first(); + assert.equal(await firstSelect.getAttribute("data-filter"), "class"); + assert.equal(await firstSelect.inputValue(), "optimal"); + assert.deepEqual( + await life + .locator(".lifecycle-selectors > label") + .evaluateAll((els) => els.map((e) => e.firstChild.textContent.trim())), + [ + "Class", + "Likelihood", + "Variant", + "Instrument", + "Hardware class", + "Measurement", + "", + ], + ); + const row = (id) => life.locator(`tr[data-record="${id}"]`); + const old = row("imaging-delaunay-hst-a100-runtime-old"); + assert(await row("imaging-delaunay-hst-a100-runtime").isVisible()); + assert(!(await old.isVisible())); + assert(!(await row("imaging-delaunay-hst-ral-runtime").isVisible())); + await life.locator('input[data-filter="history"]').check(); + assert(await old.isVisible()); + await life.locator('input[data-filter="history"]').uncheck(); + assert(!(await old.isVisible())); + await firstSelect.selectOption("secondary"); + assert(await row("imaging-delaunay-hst-ral-runtime").isVisible()); + assert(!(await row("imaging-delaunay-hst-a100-runtime").isVisible())); + await firstSelect.selectOption("all"); + await life.locator('select[data-filter="instrument"]').selectOption("sma"); + assert.equal(await life.locator("tr[data-record]:visible").count(), 1); + await life.locator('select[data-filter="measurement"]').selectOption("breakdown"); + assert(await life.locator('[data-id="no-rows"]').isVisible()); + assert.equal(await life.locator('tr[data-status="archived"]').count(), 0); + assert( + (await life.locator('[data-id="coverage"] summary').textContent()).includes( + "1 optimal cell not measured", + ), + ); + assert.equal(await page.locator('[data-walltime="walltime"]').count(), 1); + for (const width of [390, 1280]) { + await page.setViewportSize({ width, height: 1000 }); + assert( + await page.evaluate( + () => document.documentElement.scrollWidth <= window.innerWidth, + ), + ); + } assert.deepEqual(errors, []); console.log( "Browser checks passed: new tabs, filtering, scales, transport, navigation and responsive layout", diff --git a/tests/test_board.py b/tests/test_board.py index 2e14ade..fa21ae7 100644 --- a/tests/test_board.py +++ b/tests/test_board.py @@ -220,3 +220,68 @@ def test_capture_freshness_is_conservative(lens, web, tmp_path): s.outcome = "ok" s.refreshed_at = None assert "Last updated unavailable" in board.render_html([s]) + + +# ------------------------------------------- v3 lifecycle and walltime axis --- + + +def _snapshot(name, schema, fixture): + from pulse.ingest import Snapshot + from pulse.registry import Instance + + instance = Instance( + name, + "autolens_profiling" if name == "lens" else "autolens_inference", + "dashboard/catalogue.json", + schema, + "https://example.invalid", + github="PyAutoLabs/autolens_profiling", + ) + return Snapshot(instance, "ok", "remote", fixture_doc(fixture), SHA_A, NOW, refreshed_at=NOW) + + +def test_v3_status_lines_carry_lifecycle_counts(): + s = _snapshot("lens", "profiling-summary@3", "setup_summary_v3.json") + expected = "5 current · 1 superseded · 3 archived (producer-declared)" + assert board.lifecycle(s) == expected + md = board.render_markdown([s], NOW, campaign_data={"campaigns": [], "tasks": []}) + assert f"- **Lifecycle:** {expected}" in md + assert "optimal, current setups" in md + assert "imaging/delaunay | hst | `jax_gpu_dense_fp64`" in md + assert "imaging-delaunay-hst-a100-runtime-old" not in md + assert "Coverage: 1 optimal cell(s) not measured." in md + state = board.render_state([s], updated=NOW, now=NOW) + census = next(i for i in state["items"] if i["id"] == "pulse:lens:census") + assert expected in census["text"] + + +def test_v2_status_lines_are_unchanged(): + s = _snapshot("lens", "profiling-summary@2", "setup_summary_v2.json") + assert board.lifecycle(s) is None + md = board.render_markdown([s], NOW, campaign_data={"campaigns": [], "tasks": []}) + assert "**Lifecycle:**" not in md + + +def test_walltime_instance_renders_its_own_table_under_profiling_results(): + lens = _snapshot("lens", "profiling-summary@3", "setup_summary_v3.json") + wall = _snapshot("lens-walltime", "walltime-validation@1", "walltime_validation_v1.json") + data = {"campaigns": [], "tasks": []} + page = board.render_html([lens, wall], NOW, campaign_data=data) + results = page.index('id="evidence"') + table = page.index('data-walltime="lens-walltime"') + assert results < table + assert "Full-fit measurements" in page and "nautilus" in page + assert "setup catalogue unavailable" not in page + md = board.render_markdown([lens, wall], NOW, campaign_data=data) + assert md.index("## Profiling Results") < md.index("wall-time validation") + assert "| Measured/predicted |" in md and "| 1.26 |" in md + state = board.render_state([wall], updated=NOW, now=NOW) + assert not any(i["id"].endswith(":unqualified") for i in state["items"]) + assert "full-fit records" in board.qualification(wall) + + +def test_v3_and_walltime_state_meets_the_brain_contract(): + lens = _snapshot("lens", "profiling-summary@3", "setup_summary_v3.json") + wall = _snapshot("lens-walltime", "walltime-validation@1", "walltime_validation_v1.json") + state = board.render_state([lens, wall], updated=NOW, now=NOW) + assert _validator().validate_state(state) == [] diff --git a/tests/test_setup_browser.py b/tests/test_setup_browser.py index 0ff0aa1..12f91e1 100644 --- a/tests/test_setup_browser.py +++ b/tests/test_setup_browser.py @@ -95,3 +95,58 @@ def test_real_browser_fixture_preserves_capture_hashes_and_setup_identity(): assert shard["setups"] == [ setup for setup in fixture["catalogue"]["setups"] if setup["id"] == manifest["setup_id"] ] + + +# ------------------------------------------------ v3: Class-first browser --- + + +def lifecycle_capture(): + snapshot = capture() + return replace(snapshot, doc=fixture_doc("setup_summary_v3.json")) + + +def test_v3_first_select_is_class_with_optimal_selected(): + page = setup_browser.render(lifecycle_capture()) + selects = re.findall(r"]*data-filter=\"([a-z_]+)\"[^>]*>(.*?)", page, re.S) + assert [key for key, _ in selects] == [ + "class", + "dataset_type", + "variant", + "instrument", + "host_class", + "measurement", + ] + options = re.findall(r'
', page, re.S)[1] + assert "1 optimal cell not measured" in coverage + assert "euclid" in coverage and "numba_cpu_sparse_fp64" not in coverage + assert setup_browser.not_measured(lifecycle_capture().doc)[0]["instrument"] == "euclid" + + +def test_v3_bypasses_the_v2_payload_and_heuristics(): + page = setup_browser.render(lifecycle_capture()) + assert "setup-payload" not in page and "data-lifecycle" in page + v2 = setup_browser.render(capture()) + assert "data-lifecycle" not in v2 and "setup-payload" in v2 From d08db2f387ba6da54d0160965babd1c504d51597 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Wed, 21 Oct 2026 18:58:02 +0100 Subject: [PATCH 5/5] docs: profiling-summary@3, walltime-validation@1 and designation decisions REFERENCE.md no longer calls v1 live: v2 is the registry pin and v3 is supported. New sections give the v3 vocabulary, record fields and lifecycle rules, the full-fit walltime-validation@1 contract, the v3 receipt counts and the designation decision kind. README mirrors the contracts; CHECKIN.md says research lookups read class optimal + status current unless asked otherwise. dashboard.html carries the new browser assets (offline re-render; v2 content unchanged). Co-Authored-By: Claude Fable 5.1 --- CHECKIN.md | 3 +- README.md | 23 +++++-- REFERENCE.md | 163 +++++++++++++++++++++++++++++++++++++++++++++---- dashboard.html | 63 ++++++++++++++++++- 4 files changed, 232 insertions(+), 20 deletions(-) diff --git a/CHECKIN.md b/CHECKIN.md index 8dd34bb..5c049c6 100644 --- a/CHECKIN.md +++ b/CHECKIN.md @@ -13,7 +13,8 @@ campaigns and tasks. This static page does not run jobs or monitor them itself. 2. Read each campaign's current project wiki/ledger, results and issue/PR state. Evidence URLs pin the last reviewed source; inspect the current default branch for subsequent changes too. Consult Brain's profiling conductor for - scientific interpretation. With HPC access, use project pull/status commands; + scientific interpretation. Research lookups in Pulse catalogues read + `class: optimal` + `status: current` records unless asked otherwise. With HPC access, use project pull/status commands; otherwise report which jobs/results remain unverified. Never infer completion from an old job id or absent data. 3. Compare with the last check-in and report one table: changes, current state, diff --git a/README.md b/README.md index 0427199..ab17531 100644 --- a/README.md +++ b/README.md @@ -18,17 +18,25 @@ It owns: - the **instance registry** (`registry.yaml`) — which projects publish a profiling summary, where, under which contract version; each project is named by its PyAutoMind body-map identity, never a second repository catalogue; -- the versioned **`profiling-summary` read contract** — v2 is live at - `autolens_profiling/dashboard/catalogue.json`, documented in that repo's - [dashboard/README.md](https://github.com/PyAutoLabs/autolens_profiling/blob/main/dashboard/README.md); +- the versioned **`profiling-summary` read contract** — v2 is the live pin for + `autolens_profiling` and `autofit_profiling` (`dashboard/catalogue.json`, + documented in autolens_profiling's + [dashboard/README.md](https://github.com/PyAutoLabs/autolens_profiling/blob/main/dashboard/README.md)); + v3, the lifecycle catalogue in which the producer declares each record's + `class` (optimal/secondary/experimental) and `status` (current/superseded), + is supported for the next migration; v1 snapshots still validate; +- the **`walltime-validation` v1 contract** — full-fit sampler runs, registered + as their own instance row; +- the **decision index** (`decisions.yaml`), including `designation` entries + that `class: optimal` records must cite; - the **ingest receipts** (`receipts/`) — the one commit each project was read at, per render — and the last-good **snapshots** (`snapshots/`), shown as *cached* when a later fetch fails; - the cross-project **Pages board** (`dashboard.md`, `dashboard.html`, plus `state.json` for the Brain cockpit and `badge.json`). -It validates the exchange contract only. It never moves pins, combines unmatched -timings, applies the compile threshold to runtime, computes an ecosystem-wide +It validates the exchange contract only. It never moves pins, infers an optimal +setup, selects the fastest row, combines unmatched timings, applies the compile threshold to runtime, computes an ecosystem-wide speed score or issues a release-health verdict, and it never judges a timing — the Brain's profiling conductor does (`/profiling triage `). The project repos keep their producers, results, drift policy and their own Pages page. @@ -70,7 +78,10 @@ Pulse embeds the validated captured v2 index and loads a chosen evidence shard from that **same captured commit**, verifying its SHA-256 and setup identity. A failed or cached fetch remains visible; a missing deep link never substitutes a different run. Local reads cannot load remote shards as if they were published. -Inline v2 records and v1 diagnostics remain supported. No rendering action accepts +Inline v2 records and v1 diagnostics remain supported. A v3 catalogue instead +opens on Class (Optimal by default) → Likelihood → Instrument → Hardware class → +Measurement, shows current records unless History is ticked, counts archived +records without showing them and lists optimal cells not yet measured. No rendering action accepts a baseline, changes campaign state, runs profiling, or selects the fastest run. Temporal diagnostics remain collapsed while the front page focuses on setup choices. Existing archive evidence still requires scientific review. diff --git a/REFERENCE.md b/REFERENCE.md index 100572d..06abeab 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -6,8 +6,9 @@ active campaigns and Pulse-owned tasks above the detailed profiling evidence. prompts and their source commit. Board refreshes never change task status or stamp a check-in. Run `bin/pyauto-pulse board --offline` after ledger edits. -The registry schema, the `profiling-summary` v1 contract as this organ reads -it, and the formats of the receipts, snapshots, board markers and cockpit feed. +The registry schema, the `profiling-summary` contracts as this organ reads +them (v1 historical, v2 the live registry pin, v3 the lifecycle catalogue now +supported), the `walltime-validation` v1 evidence axis, and the formats of the receipts, snapshots, board markers and cockpit feed. Prose boundaries live in [AGENTS.md](AGENTS.md); the design is [profiling_inference_organs.md](https://github.com/PyAutoLabs/PyAutoBrain/blob/main/docs/research/profiling_inference_organs.md). @@ -20,28 +21,33 @@ One row per instance under `instances:`. Unknown fields are refused. | `instance` | yes | Stable key within the organ, `^[a-z][a-z0-9_-]*$`, unique. | | `repo` | yes | A PyAutoMind body-map identity (`repos.yaml` key). `path` and `github` are read from the body map at load time and **may not** be written in the row. | | `summary_path` | yes | Repository-relative path of the producer's summary (no `/`-prefix, `..`, URL). | -| `supported_schema` | yes | `@` the reader expects, e.g. `profiling-summary@1`. | +| `supported_schema` | yes | `@` the reader expects and implements: `profiling-summary@1`, `@2`, `@3` or `walltime-validation@1`. Any other pin is refused. | | `dashboard_url` | yes | The project's own human-facing page (`https://`). | | `library_refs` | no | Body-map identities the project exercises (each must be in the body map). | | `cortex_project` | no | An existing Cortex key, or `null`. | Further rules: the same `(repo, summary_path)` may be registered once (disjoint -scopes; no inflated coverage). Every ingest reads branch `main`. +scopes; no inflated coverage). Every ingest reads branch `main`. A project that +publishes wall-time validation registers it as a second instance row with its +own `summary_path` and `supported_schema: walltime-validation@1`. Body map resolution order: `--mind PATH` (must hold `repos.yaml`, never a silent fall-through), `$PYAUTO_MIND`, `$PYAUTO_ROOT/organs/PyAutoMind`, `$PYAUTO_ROOT/PyAutoMind`, then beside this organ (grouped `organs/` or flat task-bundle layout). No body map → an error, never a guess. -## `profiling-summary` v1, as read here +## `profiling-summary` v1, as read here (historical) The producer documents the contract ([autolens_profiling `dashboard/README.md`](https://github.com/PyAutoLabs/autolens_profiling/blob/main/dashboard/README.md)); -this is what `pulse/summary.py` enforces. Additive fields are ignored. +this is what `pulse/summary.py` enforces. Additive fields are ignored. No live +registry row pins v1 today; saved v1 snapshots and fixtures still validate. -**Support.** `(schema, version)` must be `("profiling-summary", 1)` *and* match the +**Support.** `(schema, version)` must be one this reader implements +(`profiling-summary` 1, 2 or 3, `walltime-validation` 1) *and* match the registry row's `supported_schema`; otherwise the outcome is **unsupported** and -nothing else is validated. +nothing else is validated. The envelope, record and comparison rules below are +the v1 shape; v2 and v3 reuse the envelope. **Envelope** — required: `schema, version, project, scope, generated_at, evidence_updated_at, producer_revision, comparison_policy, coverage, records, @@ -96,9 +102,9 @@ precision`, in producer order (never sorted by ratio). ## `profiling-summary` v2: setup catalogue -The v2 reader lands before the project producer migrates. Registry entries -still pin an exact version; supporting v2 does **not** silently upgrade the live -`profiling-summary@1` registry or reinterpret a saved v1 snapshot. The full +V2 is the live registry pin for both instances (`lens`, `fit`). Registry entries +pin an exact version; supporting a newer version never silently upgrades a row +or reinterprets a saved snapshot of an older one. The full [synthetic fixture](tests/fixtures/setup_summary_v2.json) is an executable producer example. Validation is stdlib-only in `pulse/catalogue.py`, through `pulse.summary.classify`; it never opens evidence files or judges science. @@ -199,13 +205,126 @@ a recommended solver is correct. Consumers must inspect the cited records' hardware/method and applicability, and must not extrapolate to a different setup or interpret a per-likelihood measurement as a total-fit prediction. +## `profiling-summary` v3: lifecycle catalogue + +V3 is v2's envelope with every record declaring where it sits in the +likelihood lifecycle. The [synthetic fixture](tests/fixtures/setup_summary_v3.json) +is an executable producer example; validation is stdlib-only in +`pulse/catalogue.py`. The producer declares `class` and `status`; Pulse checks +that the declarations are consistent and renders them. It infers no optimal +setup, selects no fastest row and computes no ratio. + +**Envelope.** The common v1 envelope (as v2), `comparisons: []`, plus: + +- `records`: list (empty is valid; an empty catalogue is no evidence). +- `planned_cells` (optional list): the cells the producer intends to measure, + each with the key fields below and `class`. Duplicate cells are refused. +- `archive_record_count`: non-negative integer. Archived records are moved out + of `records[]` by the producer and only counted here. +- `hazards`, `recommendations` (optional): the v2 shape; `applies_to.setup_ids` + name canonical v3 `setup_id`s present in `records`. +- `coverage.expected.cells` equals the `planned_cells` count; + `coverage.observed.records` equals the record count, and `current` / + `superseded`, when present, match the declared statuses. + +**Vocabulary** (the validator's enumerations): + +| Field | Values | +|---|---| +| `likelihood.dataset_type` | `imaging`, `interferometer`, `datacube`, `point_source_image`, `point_source_source`, `multi_galaxy`, `group`, `cluster`, `multi_dataset`, `weak` | +| `likelihood.variant` | non-empty string, e.g. `delaunay`, `mge`, `source_plane` | +| `setup.backend` | `numba`, `jax` | +| `setup.device` | `cpu`, `gpu` | +| `setup.operator` | `dense`, `sparse`, `na` | +| `setup.precision` | `fp64`, `mixed` (mixed is expected to be `class: experimental`) | +| `setup.transformer` | `dft`, `nufft`, `wtilde`, `streaming`, `na` | +| `setup.threads` | positive integer or `null` | +| `setup_id` | `___`, plus `_` when the transformer is not `na`, e.g. `jax_gpu_sparse_fp64_wtilde` | +| `host_class` | `hpc_ral_cpu`, `hpc_a100`, `local_laptop`, `local_gpu` (provenance, never identity) | +| `measurement` | `runtime`, `breakdown`, `gradient`, `compile`, `memory_host`, `memory_device`, `parity`, `walltime_validation` | +| `class` | `optimal`, `secondary`, `experimental` | +| `status` | `current`, `superseded`, `archived` | + +**Records.** Each record has `id` (unique), `run_id`, `likelihood +{dataset_type, variant}`, `instrument`, `regime` (the named size bin, e.g. +`standard`), `setup {backend, device, operator, precision, transformer, +threads}` and the matching `setup_id`, `host_class`, `model_variant` (e.g. +`default`, `light_fixed`), `measurement`, `metric`, `unit`, `values`, `class`, +`status`, `superseded_by`, `day_zero`, `figures`, `decision_id`, +`release_pin`, `measured_at`, and v2's `identity`, `method`, `provenance`, +`validation` and `evidence` with the same rules. + +- `unit` belongs to the measurement: seconds for the time measurements, a + memory unit for `memory_host`/`memory_device`, `dimensionless` for `parity`. +- `values` is a non-empty object of finite, non-negative numbers that contains + the headline `metric` (a breakdown carries its components beside its total). + Missing cells are `planned_cells`, never zero-valued records. +- `superseded_by` is `null` unless `status: superseded`, when it names another + record in the same document. +- `day_zero`, `release_pin`, `decision_id`: string or `null`. `measured_at`: + ISO-8601 UTC. +- `figures`: list of `{kind, path, sha256}` with safe repo-relative paths and + 64-hex digests (may be empty). + +**Rules.** + +- Exactly one `current` record per key (`likelihood`, `instrument`, + `setup_id`, `host_class`, `model_variant`, `measurement`). The producer + chooses which one under the day-zero rule; a second `current` is refused. +- `class: optimal` requires a `decision_id`. `pyauto-pulse check` and ingest + resolve every `decision_id` against this organ's `decisions.yaml` + (the `designation` decisions below); an unresolved id makes the capture + **invalid**. +- `host_class` `local_laptop` or `local_gpu` is never `current`; such rows may + be kept as `superseded` provenance. +- An `archived` record inside `records[]` is refused. + +**Board.** The setup browser's first selector is Class (Optimal by default, +then Secondary, Experimental, All), then Likelihood (dataset type, variant), +Instrument, Hardware class and Measurement. Only `current` rows show until +History is ticked, which adds `superseded` rows. Archived records appear only +as a count. A Coverage panel lists optimal `planned_cells` with no current +record. Research lookups read `class: optimal` + `status: current` unless asked +otherwise. The v2 eligibility heuristics never apply to a v3 catalogue. + +## `walltime-validation` v1: full-fit evidence axis + +A second evidence axis, produced by autolens_inference and registered as its +own instance row. These are **full-fit** measurements: one sampler run per +record. A per-likelihood `runtime` record is never read as a total-fit +prediction, and a full-fit wall time is never read as a per-likelihood cost. +The [fixture](tests/fixtures/walltime_validation_v1.json) is an executable +example; validation is stdlib-only in `pulse/walltime.py`. + +**Envelope** — required: `schema` (`walltime-validation`), `version` (1), +`project` (non-empty), `generated_at` (ISO-8601 UTC), `producer_revision` +(40-hex or `null`), `records`; optional `limitations` (non-empty strings). + +**Records** — each with a unique `id` and one record per key (`likelihood +{dataset_type, variant}`, `instrument`, `setup_id`, `host_class`, +`model_variant`, `sampler`, `seed`); a duplicate key is refused. Fields: + +- `wall_s`, `implied_ms_per_eval`, `predicted_ms_per_eval`, + `ratio_measured_over_predicted`, `parallel_efficiency` (relative to the + 1-core row of the same key), `max_log_likelihood`: finite number or `null`. +- `likelihood_evals`, `cores`: non-negative integer or `null`; `seed`: + non-negative integer. +- `reference_runtime_record_id` (the day-zero `runtime` record compared with), + `release_pin`, `slurm_job_id`: string or `null`. +- `parity_status` ∈ `match | diverged | not_assessed`. + +Every derived number is computed and declared by the producer. Pulse renders +the records as their own table under Profiling Results and computes nothing; +the instance has no qualification flag, so it raises no "unqualified" item. + ## Ingest 1. `GET https://api.github.com/repos///commits/main` → one 40-hex SHA (stdlib `urllib`, 20 s timeout, `User-Agent`, `Authorization: Bearer $GH_TOKEN` on API calls only, when set). 2. `GET https://raw.githubusercontent.com////`. -3. Validate; write the receipt; on `ok` also write the snapshot. +3. Validate; for a `profiling-summary@3` document resolve every `decision_id` + against `decisions.yaml`; write the receipt; on `ok` also write the snapshot. `--offline` reads the committed receipt + snapshot and writes nothing. `--from [INSTANCE=]PATH` reads a local checkout (or the file), reports its `HEAD` and @@ -224,6 +343,10 @@ writes **no** receipt or snapshot. } ``` +A `profiling-summary@3` receipt also carries `"lifecycle": {"current": N, +"superseded": N, "archived": N}` (the producer's declared counts); receipts of +other schemas keep the shape above. + ### Snapshot — `snapshots/.json` `{instance, repo, github, summary_path, commit, fetched_at, summary}` — the last @@ -247,6 +370,9 @@ nothing; a new project commit moves both. `/profiling triage `. Freshness without a policy is shown as "freshness policy unspecified · age N d at fetch " (the HTML adds a live "N d today"); a passed `valid_until` reads "stale". +- A v3 instance's status lines add **Lifecycle** (`N current · N superseded · + N archived`), and its census item carries the same counts. A + `walltime-validation` instance renders as its own table. - Instance marker (what `check` compares to the receipt): ``. - `badge.json`: shields endpoint, label `pulse`, message = the state headline. @@ -336,3 +462,16 @@ Results in HTML and Markdown. Both boards can link to one canonical record. The reader validates IDs, dates, GitHub destinations and owned record metadata; it never makes a scientific decision. Remote record existence is checked when publishing the cross-dashboard link, not by offline rendering. + +### Designation decisions + +An entry with `kind: designation` records the human's choice of setups for one +likelihood, regime and hardware class. Its extra fields are validated only when +the kind is set: `likelihood` (`/`), `regime`, +`hardware_class` ∈ `cpu_many_core | gpu_a100 | laptop`, `optimal` (one +canonical `setup_id`), `secondary` (list of distinct canonical `setup_id`s), +`evidence` (non-empty list of record ids), `rule` (one sentence in +dataset-property terms) and `supersedes` (a decision id in the index, or +`null`). `profiling-summary@3` records with `class: optimal` cite these entries +in `decision_id`. The template is in [decisions/README.md](decisions/README.md); +no designation is recorded yet. diff --git a/dashboard.html b/dashboard.html index 0dce4f9..cbeefb2 100644 --- a/dashboard.html +++ b/dashboard.html @@ -626,6 +626,31 @@ .timing-diagnostics { margin-top: 0.8rem; } +/* profiling-summary@3: Class-first lifecycle browser. */ +.lifecycle-selectors { + grid-template-columns: repeat(auto-fit, minmax(9rem, 1fr)); +} +.lifecycle-selectors .history-toggle { + align-self: end; + font-weight: 500; +} +.lifecycle-table tr[hidden], +.lifecycle-browser [hidden] { + display: none; +} +.lifecycle-table td, +.lifecycle-table th { + font-size: 0.8rem; + padding: 0.35rem 0.45rem; + vertical-align: top; +} +.lifecycle-coverage { + margin: 1rem 0; +} +.lifecycle-coverage summary { + cursor: pointer; + font-weight: 600; +}

PyAutoPulseProfiling

Measure. Trace. Compare.

Check your Pulse

Read the prompt