From b23be3e44a6ec3192df36e1c9c35560144e0e59a Mon Sep 17 00:00:00 2001 From: Daniel Ari Friedman Date: Thu, 12 Nov 2026 05:23:19 -0800 Subject: [PATCH] metrics: record baseline receipts and the render arm in the gate page --- docs/_generated/README.md | 6 +- docs/_generated/gate-baseline.json | 77 +++++++++++++++++++++++++ docs/_generated/gate-metrics.md | 49 ++++++++++------ docs/_generated/gate-timings.json | 39 +++++++------ docs/development.md | 4 +- docs/gate.md | 6 +- scripts/gate_metrics.py | 90 ++++++++++++++++++++++-------- 7 files changed, 210 insertions(+), 61 deletions(-) create mode 100644 docs/_generated/gate-baseline.json diff --git a/docs/_generated/README.md b/docs/_generated/README.md index 85e1c13..afc23b5 100644 --- a/docs/_generated/README.md +++ b/docs/_generated/README.md @@ -5,8 +5,10 @@ checked by a gate arm; do not edit these files by hand. | File | Producer | Check | | --- | --- | --- | -| [`gate-metrics.md`](gate-metrics.md) | `scripts/gate_metrics.py` from [`gate-timings.json`](gate-timings.json) | `scripts/release_check.py --arm docs` | +| [`gate-metrics.md`](gate-metrics.md) | `scripts/gate_metrics.py` from [`gate-timings.json`](gate-timings.json) and [`gate-baseline.json`](gate-baseline.json) | `scripts/release_check.py --arm docs` | | [`manuscript-variables.json`](manuscript-variables.json) | `scripts/manuscript_variables.py` | `scripts/release_check.py --arm manuscript` | `gate-timings.json` is a recorded receipt from one `--timings` run of the gate; -it is evidence for the measured numbers, not a claim about another host. +`gate-baseline.json` records the pre-change arm commands and their measured wall +times. Both are evidence for the measured numbers, not claims about another +host. diff --git a/docs/_generated/gate-baseline.json b/docs/_generated/gate-baseline.json new file mode 100644 index 0000000..aa4d6aa --- /dev/null +++ b/docs/_generated/gate-baseline.json @@ -0,0 +1,77 @@ +{ + "schema": "activeblockference-gate-baseline/v1", + "measured": "2026-11-12", + "host": "macOS-27.0.1-arm64-arm-64bit", + "python": "3.12.16", + "conditions": "One manual run per command before the composable gate landed, on the same host. Other estate jobs drove the load average to about 100 and all caches were warm, so these are historical evidence, not a controlled benchmark.", + "arms": { + "lint": { + "seconds": 0.34, + "comparable": true, + "commands": [ + {"command": "uv run ruff check blockference tests GRTs scripts", "seconds": 0.34} + ] + }, + "types": { + "seconds": 10.35, + "comparable": false, + "note": "The baseline mypy run targeted Python 3.12; the arm now targets the requires-python floor (3.11).", + "commands": [ + {"command": "uv run mypy blockference", "seconds": 10.35} + ] + }, + "test-full": { + "seconds": 88.77, + "comparable": false, + "note": "The baseline was `uv run pytest` without `--cov` or `--cov-fail-under=90`; the arm runs coverage. Recorded result: 84 passed, 1 skipped.", + "commands": [ + {"command": "uv run pytest", "seconds": 88.77} + ] + }, + "docs": { + "seconds": 0.22, + "comparable": false, + "note": "The baseline ran only the link/anchor checker; the arm also verifies the generated metrics document.", + "commands": [ + {"command": "uv run python scripts/check_docs_links.py", "seconds": 0.22} + ] + }, + "manuscript": { + "seconds": 18.81, + "comparable": false, + "note": "The baseline ran validation, build, and the build check; the arm also verifies the generated variables file.", + "commands": [ + {"command": "uv run python scripts/validate_manuscript.py", "seconds": 3.85}, + {"command": "uv run python scripts/build_manuscript.py", "seconds": 7.24}, + {"command": "uv run python scripts/build_manuscript.py --check", "seconds": 7.72} + ] + }, + "gnn": { + "seconds": 14.8, + "comparable": true, + "commands": [ + {"command": "uv run python scripts/export_gnn_spec.py --check", "seconds": 3.08}, + {"command": "uv run python scripts/validate_gnn_spec.py --json", "seconds": 11.72} + ] + }, + "smoke": { + "seconds": 15.42, + "comparable": true, + "commands": [ + {"command": "uv run blockference pipeline --config configs/smoke.yml --output-root /output --run-name gate_smoke", "seconds": 7.57}, + {"command": "uv run blockference validation --run-dir /output/gate_smoke", "seconds": 3.66}, + {"command": "uv run blockference run --config configs/smoke.yml --output /run.csv", "seconds": 4.19} + ] + }, + "package": { + "seconds": 17.67, + "comparable": true, + "commands": [ + {"command": "uv build", "seconds": 4.98}, + {"command": "uv venv --python /.venv/bin/python /venv", "seconds": 0.04}, + {"command": "uv pip install --python /venv/bin/python dist/*.whl", "seconds": 1.01}, + {"command": "/venv/bin/python -c \"from blockference import GridWorld; ...\"", "seconds": 11.64} + ] + } + } +} diff --git a/docs/_generated/gate-metrics.md b/docs/_generated/gate-metrics.md index 399a4bf..b7f15ae 100644 --- a/docs/_generated/gate-metrics.md +++ b/docs/_generated/gate-metrics.md @@ -2,30 +2,43 @@ -Measured numbers from one recorded run of the canonical gate on `macOS-27.0.1-arm64-arm-64bit` with Python 3.12.16 at `2026-11-12T13:09:06Z`. +Measured numbers from one recorded run of the canonical gate on `macOS-27.0.1-arm64-arm-64bit` with Python 3.12.16 at `2026-11-12T13:20:11Z`. Refresh with: ```bash -uv run python scripts/release_check.py --arm all --arm test \ +uv run python scripts/release_check.py --arm all --arm test --arm render \ --timings docs/_generated/gate-timings.json uv run python scripts/gate_metrics.py ``` -## Arm wall times - -| Arm | Wall seconds | Status | First command | -| --- | --- | --- | --- | -| `lint` | 0.07 | passed | `uv run ruff check blockference tests GRTs scripts` | -| `types` | 0.35 | passed | `uv run mypy blockference` | -| `test` | 12.88 | passed | `uv run pytest -m not slow` | -| `test-full` | 58.06 | passed | `uv run pytest --cov=blockference --cov-report=term --cov-report=xml --cov-fail-under=90` | -| `docs` | 11.02 | passed | `uv run python scripts/check_docs_links.py` | -| `manuscript` | 7.84 | passed | `uv run python scripts/manuscript_variables.py --check` | -| `gnn` | 5.61 | passed | `uv run python scripts/export_gnn_spec.py --check` | -| `smoke` | 5.23 | passed | `uv run blockference pipeline --config configs/smoke.yml --output-root /var/folders/qs/rxkntx5d4hlgz0c7x8v15gym0000gn/T/activeblockference-gate-avw6d67s/output --run-name gate_smoke` | -| `package` | 12.91 | passed | `uv build` | -| `render` | not recorded | — | — | +## Before and after + +`n/a` means the arm or selection did not exist before the composable gate. + +| Arm | Before (s) | After (s) | After status | After command | +| --- | --- | --- | --- | --- | +| `lint` | 0.34 | 0.05 | passed | `uv run ruff check blockference tests GRTs scripts` | +| `types` | 10.35 | 0.28 | passed | `uv run mypy blockference` | +| `test` | n/a | 15.18 | passed | `uv run pytest -m not slow` | +| `test-full` | 88.77 | 77.03 | passed | `uv run pytest --cov=blockference --cov-report=term --cov-report=xml --cov-fail-under=90` | +| `docs` | 0.22 | 10.25 | passed | `uv run python scripts/check_docs_links.py` | +| `manuscript` | 18.81 | 7.73 | passed | `uv run python scripts/manuscript_variables.py --check` | +| `gnn` | 14.8 | 5.74 | passed | `uv run python scripts/export_gnn_spec.py --check` | +| `smoke` | 15.42 | 6.1 | passed | `uv run blockference pipeline --config configs/smoke.yml --output-root /var/folders/qs/rxkntx5d4hlgz0c7x8v15gym0000gn/T/activeblockference-gate-1k9b70t9/output --run-name gate_smoke` | +| `package` | 17.67 | 12.87 | passed | `uv build` | +| `render` | n/a | 11.65 | passed | `uv run python scripts/build_manuscript.py --render pdf` | + +Baseline workloads that are not equivalent to their arms: + +- `types`: The baseline mypy run targeted Python 3.12; the arm now targets the requires-python floor (3.11). +- `test-full`: The baseline was `uv run pytest` without `--cov` or `--cov-fail-under=90`; the arm runs coverage. Recorded result: 84 passed, 1 skipped. +- `docs`: The baseline ran only the link/anchor checker; the arm also verifies the generated metrics document. +- `manuscript`: The baseline ran validation, build, and the build check; the arm also verifies the generated variables file. + +Baseline conditions: One manual run per command before the composable gate landed, on the same host. Other estate jobs drove the load average to about 100 and all caches were warm, so these are historical evidence, not a controlled benchmark. + +The baseline and current receipts are historical evidence. Timings include host load and warm caches; they are not a controlled benchmark. ## Test selection @@ -39,6 +52,6 @@ Recorded run: 85 passed, 0 skipped. `test-full` recorded 90.94% line coverage of `blockference` (floor 90%). -## Receipt +## Receipts -The source of these numbers is [`gate-timings.json`](gate-timings.json), written by `scripts/release_check.py --timings`. An arm absent from that receipt is reported as not recorded, never as passing. +The current numbers come from [`gate-timings.json`](gate-timings.json), written by `scripts/release_check.py --timings`; the before numbers come from [`gate-baseline.json`](gate-baseline.json), recorded once before the composable gate landed. An arm absent from a receipt is reported as not recorded, never as passing. diff --git a/docs/_generated/gate-timings.json b/docs/_generated/gate-timings.json index 2bbdbd4..970375c 100644 --- a/docs/_generated/gate-timings.json +++ b/docs/_generated/gate-timings.json @@ -5,7 +5,7 @@ "uv run python scripts/check_docs_links.py", "uv run python scripts/gate_metrics.py --check" ], - "seconds": 11.02, + "seconds": 10.25, "status": "passed" }, "gnn": { @@ -13,14 +13,14 @@ "uv run python scripts/export_gnn_spec.py --check", "uv run python scripts/validate_gnn_spec.py --json" ], - "seconds": 5.61, + "seconds": 5.74, "status": "passed" }, "lint": { "commands": [ "uv run ruff check blockference tests GRTs scripts" ], - "seconds": 0.07, + "seconds": 0.05, "status": "passed" }, "manuscript": { @@ -30,51 +30,58 @@ "uv run python scripts/build_manuscript.py", "uv run python scripts/build_manuscript.py --check" ], - "seconds": 7.84, + "seconds": 7.73, "status": "passed" }, "package": { "commands": [ "uv build", - "uv venv --python /Users/friedman/Documents/Git/gitvault/Username/ActiveInferenceInstitute/Public/ActiveBlockference/.venv/bin/python /var/folders/qs/rxkntx5d4hlgz0c7x8v15gym0000gn/T/activeblockference-package-fn41c71n/venv", - "uv pip install --python /var/folders/qs/rxkntx5d4hlgz0c7x8v15gym0000gn/T/activeblockference-package-fn41c71n/venv/bin/python /Users/friedman/Documents/Git/gitvault/Username/ActiveInferenceInstitute/Public/ActiveBlockference/dist/active_blockference-1.0.0-py3-none-any.whl", - "/var/folders/qs/rxkntx5d4hlgz0c7x8v15gym0000gn/T/activeblockference-package-fn41c71n/venv/bin/python -c from blockference import GridWorld; assert GridWorld(2, {'agent': (0, 0)}).current_state == {'agent': (0, 0)}" + "uv venv --python /Users/friedman/Documents/Git/gitvault/Username/ActiveInferenceInstitute/Public/ActiveBlockference/.venv/bin/python /var/folders/qs/rxkntx5d4hlgz0c7x8v15gym0000gn/T/activeblockference-package-f6ywi5t7/venv", + "uv pip install --python /var/folders/qs/rxkntx5d4hlgz0c7x8v15gym0000gn/T/activeblockference-package-f6ywi5t7/venv/bin/python /Users/friedman/Documents/Git/gitvault/Username/ActiveInferenceInstitute/Public/ActiveBlockference/dist/active_blockference-1.0.0-py3-none-any.whl", + "/var/folders/qs/rxkntx5d4hlgz0c7x8v15gym0000gn/T/activeblockference-package-f6ywi5t7/venv/bin/python -c from blockference import GridWorld; assert GridWorld(2, {'agent': (0, 0)}).current_state == {'agent': (0, 0)}" ], - "seconds": 12.91, + "seconds": 12.87, + "status": "passed" + }, + "render": { + "commands": [ + "uv run python scripts/build_manuscript.py --render pdf" + ], + "seconds": 11.65, "status": "passed" }, "smoke": { "commands": [ - "uv run blockference pipeline --config configs/smoke.yml --output-root /var/folders/qs/rxkntx5d4hlgz0c7x8v15gym0000gn/T/activeblockference-gate-avw6d67s/output --run-name gate_smoke", - "uv run blockference validation --run-dir /var/folders/qs/rxkntx5d4hlgz0c7x8v15gym0000gn/T/activeblockference-gate-avw6d67s/output/gate_smoke", - "uv run blockference run --config configs/smoke.yml --output /var/folders/qs/rxkntx5d4hlgz0c7x8v15gym0000gn/T/activeblockference-gate-avw6d67s/run.csv" + "uv run blockference pipeline --config configs/smoke.yml --output-root /var/folders/qs/rxkntx5d4hlgz0c7x8v15gym0000gn/T/activeblockference-gate-1k9b70t9/output --run-name gate_smoke", + "uv run blockference validation --run-dir /var/folders/qs/rxkntx5d4hlgz0c7x8v15gym0000gn/T/activeblockference-gate-1k9b70t9/output/gate_smoke", + "uv run blockference run --config configs/smoke.yml --output /var/folders/qs/rxkntx5d4hlgz0c7x8v15gym0000gn/T/activeblockference-gate-1k9b70t9/run.csv" ], - "seconds": 5.23, + "seconds": 6.1, "status": "passed" }, "test": { "commands": [ "uv run pytest -m not slow" ], - "seconds": 12.88, + "seconds": 15.18, "status": "passed" }, "test-full": { "commands": [ "uv run pytest --cov=blockference --cov-report=term --cov-report=xml --cov-fail-under=90" ], - "seconds": 58.06, + "seconds": 77.03, "status": "passed" }, "types": { "commands": [ "uv run mypy blockference" ], - "seconds": 0.35, + "seconds": 0.28, "status": "passed" } }, - "generated": "2026-11-12T13:09:06Z", + "generated": "2026-11-12T13:20:11Z", "host": "macOS-27.0.1-arm64-arm-64bit", "python": "3.12.16", "schema": "activeblockference-gate/v1", diff --git a/docs/development.md b/docs/development.md index ea90226..7481ac4 100644 --- a/docs/development.md +++ b/docs/development.md @@ -17,7 +17,9 @@ smoke runs, and the wheel import check. mypy targets the `requires-python` floor (`python_version = "3.11"` in `pyproject.toml`), so one run covers the minimum supported stubs. Notebook execution, real-pipeline integration, CLI smoke, and figure-byte checks carry -the `slow` marker and are excluded from `--arm test`. +the `slow` marker and are excluded from `--arm test`. The recorded before/after +arm times, including the render arm, are in +[`_generated/gate-metrics.md`](_generated/gate-metrics.md). The pinned GNN parser conformance check (`scripts/validate_gnn_spec.py`) is part of the `gnn` arm only when `GNN_ROOT` or `GNN_RUNTIME_PYTHON` is set, diff --git a/docs/gate.md b/docs/gate.md index 710470c..a83790d 100644 --- a/docs/gate.md +++ b/docs/gate.md @@ -12,8 +12,10 @@ uv run python scripts/release_check.py --list # arms and their commands ``` Measured wall times and test counts live in the generated -[`_generated/gate-metrics.md`](_generated/gate-metrics.md). Do not hand-edit -those numbers: refresh them with the commands the generated page records. +[`_generated/gate-metrics.md`](_generated/gate-metrics.md), which shows the +recorded before/after arm times and labels baseline workloads that are not +equivalent to their arms. Do not hand-edit those numbers: refresh them with the +commands the generated page records. ## Arms diff --git a/scripts/gate_metrics.py b/scripts/gate_metrics.py index dd5e58b..95506d4 100644 --- a/scripts/gate_metrics.py +++ b/scripts/gate_metrics.py @@ -1,13 +1,18 @@ """Write the generated gate metrics document. -The measured numbers in ``docs/_generated/gate-metrics.md`` come from one -recorded run of the canonical gate. This script never invents a runtime: an arm -missing from the receipt is reported as not recorded. Test counts are derived -from the current suite with ``pytest --collect-only``. +The numbers in ``docs/_generated/gate-metrics.md`` come from recorded receipts: + +* ``gate-timings.json`` -- one run of the canonical gate (``--timings``); +* ``gate-baseline.json`` -- the pre-change arm commands and their measured wall + times, recorded once when the composable gate landed. + +The producer never invents a runtime: an arm missing from a receipt is reported +as not recorded. Test counts are derived from the current suite with +``pytest --collect-only``. Refresh after changing the gate or the test selection:: - uv run python scripts/release_check.py --arm all --arm test \ + uv run python scripts/release_check.py --arm all --arm test --arm render \ --timings docs/_generated/gate-timings.json uv run python scripts/gate_metrics.py """ @@ -22,8 +27,10 @@ ROOT = Path(__file__).resolve().parents[1] RECEIPT = ROOT / "docs" / "_generated" / "gate-timings.json" +BASELINE = ROOT / "docs" / "_generated" / "gate-baseline.json" OUTPUT = ROOT / "docs" / "_generated" / "gate-metrics.md" SCHEMA = "activeblockference-gate/v1" +BASELINE_SCHEMA = "activeblockference-gate-baseline/v1" FAST_SELECTION = "not slow" ARM_ORDER = ( @@ -67,10 +74,22 @@ def _receipt() -> dict[str, object]: return payload -def render(payload: dict[str, object]) -> str: - """Render the metrics document for one gate receipt.""" +def _baseline() -> dict[str, object]: + """Return the recorded pre-change baseline, or an empty record.""" + + if not BASELINE.is_file(): + return {} + payload = json.loads(BASELINE.read_text(encoding="utf-8")) + if payload.get("schema") != BASELINE_SCHEMA: + raise ValueError(f"unsupported baseline schema {payload.get('schema')!r}") + return payload + + +def render(payload: dict[str, object], baseline: dict[str, object]) -> str: + """Render the metrics document from the recorded receipts.""" arms = payload.get("arms") or {} + before = baseline.get("arms") or {} tests = payload.get("tests") or {} lines = [ "# Gate metrics", @@ -84,24 +103,49 @@ def render(payload: dict[str, object]) -> str: "Refresh with:", "", "```bash", - "uv run python scripts/release_check.py --arm all --arm test \\", + "uv run python scripts/release_check.py --arm all --arm test --arm render \\", " --timings docs/_generated/gate-timings.json", "uv run python scripts/gate_metrics.py", "```", "", - "## Arm wall times", + "## Before and after", + "", + "`n/a` means the arm or selection did not exist before the composable gate.", "", - "| Arm | Wall seconds | Status | First command |", - "| --- | --- | --- | --- |", + "| Arm | Before (s) | After (s) | After status | After command |", + "| --- | --- | --- | --- | --- |", ] for arm in ARM_ORDER: - entry = arms.get(arm) - if not entry: - lines.append(f"| `{arm}` | not recorded | — | — |") - continue - commands = entry.get("commands") or [] + recorded = before.get(arm) or {} + current = arms.get(arm) or {} + before_seconds = recorded.get("seconds", "n/a") + after_seconds = current.get("seconds", "not recorded") + status = current.get("status", "—") + commands = current.get("commands") or [] command = f"`{commands[0]}`" if commands else "—" - lines.append(f"| `{arm}` | {entry.get('seconds')} | {entry.get('status')} | {command} |") + lines.append( + f"| `{arm}` | {before_seconds} | {after_seconds} | {status} | {command} |" + ) + notes = [ + f"- `{arm}`: {recorded['note']}" + for arm, recorded in before.items() + if recorded.get("note") and not recorded.get("comparable", True) + ] + if notes: + lines += [ + "", + "Baseline workloads that are not equivalent to their arms:", + "", + *notes, + ] + if baseline: + lines += [ + "", + "Baseline conditions: " + str(baseline.get("conditions", "")), + "", + "The baseline and current receipts are historical evidence. Timings " + "include host load and warm caches; they are not a controlled benchmark.", + ] lines += ["", "## Test selection", "", "| Selection | Collected tests |", "| --- | --- |"] total = _collected() slow = _collected("-m", "slow") @@ -123,11 +167,13 @@ def render(payload: dict[str, object]) -> str: ) lines += [ "", - "## Receipt", + "## Receipts", "", - "The source of these numbers is [`gate-timings.json`](gate-timings.json), " - "written by `scripts/release_check.py --timings`. An arm absent from that " - "receipt is reported as not recorded, never as passing.", + "The current numbers come from [`gate-timings.json`](gate-timings.json), " + "written by `scripts/release_check.py --timings`; the before numbers come " + "from [`gate-baseline.json`](gate-baseline.json), recorded once before the " + "composable gate landed. An arm absent from a receipt is reported as not " + "recorded, never as passing.", "", ] return "\n".join(lines) @@ -140,7 +186,7 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--check", action="store_true", help="fail when the document is stale") args = parser.parse_args(argv) try: - content = render(_receipt()) + content = render(_receipt(), _baseline()) except (FileNotFoundError, ValueError, subprocess.CalledProcessError) as exc: print(f"gate metrics failed: {exc}", file=sys.stderr) return 1