diff --git a/docs/_generated/gate-metrics.md b/docs/_generated/gate-metrics.md index 8561d11..e3a9d13 100644 --- a/docs/_generated/gate-metrics.md +++ b/docs/_generated/gate-metrics.md @@ -43,8 +43,8 @@ The baseline and current receipts are historical evidence. Timings include host | Selection | Collected tests | | --- | --- | -| full suite | 112 | -| fast (`-m "not slow"`) | 99 | +| full suite | 115 | +| fast (`-m "not slow"`) | 102 | | slow (`-m slow`) | 13 | Recorded run: 97 passed, 0 skipped. diff --git a/docs/gate.md b/docs/gate.md index dddad45..e5dfe32 100644 --- a/docs/gate.md +++ b/docs/gate.md @@ -34,7 +34,7 @@ measurements. | `test-full` | `uv run pytest --cov=blockference --cov-report=term --cov-report=xml --cov-fail-under=90` | yes | | `docs` | `scripts/check_docs_links.py`, `scripts/gate_metrics.py --check` | yes | | `manuscript` | `scripts/manuscript_variables.py --check`, `scripts/validate_manuscript.py`, `scripts/build_manuscript.py`, `scripts/build_manuscript.py --check`; the `gnn_*` variables come from `gnn.importers` in the pinned runtime | yes | -| `gnn` | `scripts/export_gnn_spec.py --check`, the shared `python -m gnn.importers check --repo .` pin and pointer check, and real pinned `scripts/validate_gnn_spec.py --verify-tag-remote` validation (parser digests before parsing, the pin's tag resolved upstream to the pinned commit; needs network); missing runtime fails | yes | +| `gnn` | `scripts/export_gnn_spec.py --check`, then pinned `scripts/validate_gnn_spec.py --verify-tag-remote` conformance (parser digests verified before anything parses, the pin's tag resolved upstream to the pinned commit; needs network), then the shared `python -m gnn.importers check --repo .` pin and pointer check; missing runtime fails | yes | | `smoke` | pipeline, validation, and standalone `run` JSON smoke checks in a temporary directory | yes | | `package` | `uv build`, locked `uv export` constraints for clean-wheel dependencies, and isolated import outside the source tree | yes | | `render` | `scripts/build_manuscript.py --render pdf` (Pandoc + XeLaTeX) | no | @@ -60,9 +60,13 @@ The conformance job's key is `gnn-conformance` and its display name is The pin, the pointer and the specification are read by the estate's shared reader, `gnn.importers`, so this repository carries no copy of the pin shape. The -`gnn` arm runs `python -m gnn.importers check --repo . --require-pointer +`gnn` arm runs the offline drift check, then real pinned-parser conformance, then +`python -m gnn.importers check --repo . --require-pointer --workflow .github/workflows/ci.yml --job gnn-conformance` from the pinned -runtime, then the offline drift check, then real pinned-parser conformance. +runtime. Conformance precedes that reader check because `gnn.importers check` +parses the document with the installed parser, while the validator verifies the +pin's parser digests before it parses anything: no arm command reaches the +parser before its digests are verified. Select an installed pinned runtime with `--gnn-runtime` or `GNN_RUNTIME_PYTHON`, or supply a pinned source checkout via `GNN_ROOT` as diff --git a/docs/gnn.md b/docs/gnn.md index 685586d..d5be30a 100644 --- a/docs/gnn.md +++ b/docs/gnn.md @@ -102,7 +102,10 @@ and exact CI map. The pin's `tag` must resolve to the pinned `commit` (`tag-commit-mismatch`): locally in a `GNN_ROOT` checkout, or upstream with `--verify-tag-remote` (`git ls-remote`, the peeled commit of an annotated tag; the release gate's - `gnn` arm passes the flag, so it needs network access). The commit the + `gnn` arm passes the flag, so it needs network access). The upstream answer is + authoritative: when the flag is given, upstream is always queried, so a tag + that moved after a checkout fetched it fails even though the local tag still + names the pinned commit. The commit the installer recorded for the runtime (`direct_url.json` `vcs_info.commit_id`) must equal the pin's commit. Without a checkout or the flag, and for an install with no recorded VCS commit, the receipt reports `tag_binding` / diff --git a/scripts/README.md b/scripts/README.md index 80b4ff9..e0105d1 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -13,7 +13,7 @@ Repository gates and generators, all run from the repository root: | `execute_notebooks.py` | Parses and executes every tracked notebook from the repository root. | | `validate_notebooks.py` | Notebook hygiene checks before execution. | | `export_gnn_spec.py` | Writes or verifies `specs/gridworld_3x3.gnn.md` from the implementation. | -| `validate_gnn_spec.py` | Validates the committed specification with the pinned parser, using a separate interpreter selected by `--gnn-runtime` or the documented environment variables. Verifies the parser digests before parsing, binds the pin's tag to its commit (`--gnn-root` checkout or `--verify-tag-remote`; mismatch fails as `tag-commit-mismatch`), and compares the installed runtime's recorded commit with the pin. | +| `validate_gnn_spec.py` | Validates the committed specification with the pinned parser, using a separate interpreter selected by `--gnn-runtime` or the documented environment variables. Verifies the parser digests before parsing, binds the pin's tag to its commit (`--gnn-root` checkout or `--verify-tag-remote`, which always queries upstream, so a stale local tag cannot pass; mismatch fails as `tag-commit-mismatch`), and compares the installed runtime's recorded commit with the pin. | | `gnn_parser_probe.py` | Imports the upstream `gnn` parser; runs as a subprocess under the pin. | | `gnn_importer_report.py` | Imports the upstream `gnn.importers` reader and prints this repository's normalized pin and `gnn-importer-variables/v1` map for the two scripts above. | diff --git a/scripts/release_check.py b/scripts/release_check.py index 158c556..01e454d 100644 --- a/scripts/release_check.py +++ b/scripts/release_check.py @@ -192,6 +192,17 @@ def _plan( ] if arm == "gnn": commands = [["uv", "run", "python", "scripts/export_gnn_spec.py", "--check"]] + # The validator verifies the pinned parser file digests before it parses + # anything, so it runs before the shared reader's specification check: + # ``gnn.importers check`` parses the document with the installed parser + # and must not reach it before the digests are verified. It also resolves + # the pin's tag upstream, so a tag moved off the pinned commit fails. + validator = [ + "uv", "run", "python", "scripts/validate_gnn_spec.py", "--verify-tag-remote", "--json", + ] + if gnn_runtime: + validator[-2:-2] = ["--gnn-runtime", gnn_runtime] + commands.append(validator) if gnn_runtime: commands.append( [ @@ -201,14 +212,6 @@ def _plan( "--job", "gnn-conformance", ] ) - # The pin's tag is resolved upstream (needs network), so a tag that was - # moved off the pinned commit fails the release gate. - validator = [ - "uv", "run", "python", "scripts/validate_gnn_spec.py", "--verify-tag-remote", "--json", - ] - if gnn_runtime: - validator[-2:-2] = ["--gnn-runtime", gnn_runtime] - commands.append(validator) return commands if arm == "smoke": output_root = workdir / "output" diff --git a/scripts/validate_gnn_spec.py b/scripts/validate_gnn_spec.py index c8f5767..4afb61e 100644 --- a/scripts/validate_gnn_spec.py +++ b/scripts/validate_gnn_spec.py @@ -19,8 +19,10 @@ The pin's ``tag`` and ``commit`` name one revision. A GNN checkout resolves ``refs/tags/^{commit}`` locally; ``--verify-tag-remote`` resolves it upstream -with ``git ls-remote`` (the peeled commit of an annotated tag). A tag that -resolves elsewhere fails as ``tag-commit-mismatch``. Without a checkout or the +with ``git ls-remote`` (the peeled commit of an annotated tag), and always asks +upstream: a tag that moved after a checkout fetched it fails as +``tag-commit-mismatch`` instead of passing on the stale local tag. A tag that +resolves elsewhere fails the same way. Without a checkout or the flag the binding is recorded in the receipt as ``unresolvable-offline``, never as a pass, and no network is used. With the flag, a tag that cannot be resolved ends the run as *not run* (exit ``2``). @@ -274,8 +276,11 @@ def tag_binding( """Bind the pin's ``tag`` to its ``commit`` in an available object store. A GNN checkout resolves the tag locally; ``remote`` resolves it upstream with - ``git ls-remote`` (also when the checkout lacks the tag). When neither is - available the status says so instead of assuming the binding holds. + ``git ls-remote``. An explicit upstream request is never answered by a local + tag: a tag that moved upstream after this checkout fetched it is stale here, + so ``remote`` overrides the local reading instead of accepting it. When + neither store can answer the status says so instead of assuming the binding + holds. """ tag = pin["tag"] @@ -286,9 +291,10 @@ def tag_binding( "resolved": None, "detail": "no GNN object store was supplied and --verify-tag-remote was not requested", } + local: dict[str, Any] | None = None if gnn_root is not None and (gnn_root / ".git").exists(): try: - local = subprocess.run( + local_run = subprocess.run( [ "git", "-C", str(gnn_root), "rev-parse", "--verify", "-q", f"refs/tags/{tag}^{{commit}}", @@ -300,15 +306,16 @@ def tag_binding( except OSError as exc: record["detail"] = f"git could not run: {exc}" else: - if local.returncode == 0 and local.stdout.strip(): - record.update( - status="verified", - resolved=local.stdout.strip(), - detail=f"local object store {gnn_root}", - ) - return record - record["detail"] = f"tag {tag} is not present in {gnn_root}" + if local_run.returncode == 0 and local_run.stdout.strip(): + local = { + "resolved": local_run.stdout.strip(), + "detail": f"local object store {gnn_root}", + } + else: + record["detail"] = f"tag {tag} is not present in {gnn_root}" if not remote: + if local is not None: + record.update(status="verified", **local) return record try: completed = subprocess.run( diff --git a/tests/test_gnn_pin_binding.py b/tests/test_gnn_pin_binding.py index 9c597a2..3437d30 100644 --- a/tests/test_gnn_pin_binding.py +++ b/tests/test_gnn_pin_binding.py @@ -84,6 +84,68 @@ def test_local_tag_pointing_elsewhere_is_a_named_failure(checkout) -> None: assert head in failures[0] and OTHER in failures[0] +def _stale_checkout(tmp_path: Path) -> tuple[Path, Path, str, str]: + """Return a checkout whose local tag is stale against a local upstream. + + The upstream tag is moved to a new commit after the checkout cloned it, so + the checkout keeps a tag that names the pinned commit while upstream names + another one. Everything is local, so no test here uses the network. + """ + + work = tmp_path / "work" + work.mkdir() + _git(work, "init", "-q") + (work / "f").write_text("A", encoding="utf-8") + _git(work, "add", "f") + _git(work, "commit", "-q", "-m", "A") + pinned = _git(work, "rev-parse", "HEAD") + _git(work, "tag", "v9.9.9") + upstream = tmp_path / "upstream.git" + subprocess.run(["git", "clone", "-q", "--bare", str(work), str(upstream)], check=True) + checkout = tmp_path / "checkout" + subprocess.run(["git", "clone", "-q", str(upstream), str(checkout)], check=True) + (work / "f").write_text("B", encoding="utf-8") + _git(work, "commit", "-q", "-am", "B") + moved = _git(work, "rev-parse", "HEAD") + _git(work, "tag", "-f", "-a", "-m", "moved", "v9.9.9", moved) + _git(work, "push", "-q", "--force", str(upstream), "refs/tags/v9.9.9") + return checkout, upstream, pinned, moved + + +def test_remote_is_consulted_when_the_local_tag_resolves(tmp_path: Path) -> None: + checkout, upstream, pinned, moved = _stale_checkout(tmp_path) + pin = _pin(repository=str(upstream), commit=pinned) + local = vgs.tag_binding(checkout, pin) + + assert local["status"] == "verified" + assert local["resolved"] == pinned + + binding = vgs.tag_binding(checkout, pin, remote=True) + + assert binding["detail"] == "git ls-remote" + assert binding["resolved"] == moved + assert [f.split(":")[0] for f in vgs.tag_failures(binding, pin)] == [vgs.TAG_COMMIT_MISMATCH] + + +def test_remote_that_cannot_answer_never_accepts_the_local_tag( + monkeypatch: pytest.MonkeyPatch, checkout +) -> None: + repo, head = checkout + _git(repo, "tag", "v9.9.9") + real_run = subprocess.run + + def run(command, **kwargs): + if command[:2] == ["git", "ls-remote"]: + return subprocess.CompletedProcess(command, 128, "", "unreachable") + return real_run(command, **kwargs) + + monkeypatch.setattr(vgs.subprocess, "run", run) + binding = vgs.tag_binding(repo, _pin(commit=head), remote=True) + + assert binding["status"] == "unresolvable-offline" + assert "ls-remote failed" in binding["detail"] + + def test_offline_without_a_store_is_explicitly_unresolved( monkeypatch: pytest.MonkeyPatch, checkout ) -> None: diff --git a/tests/test_release_gate.py b/tests/test_release_gate.py index f79837b..add521e 100644 --- a/tests/test_release_gate.py +++ b/tests/test_release_gate.py @@ -16,13 +16,23 @@ ROOT = Path(__file__).resolve().parents[1] -def test_gnn_arm_runs_the_shared_pin_reader_before_conformance(): +def test_gnn_arm_runs_the_shared_pin_reader_after_digest_verification(): runtime = "/opt/gnn-runtime/bin/python" plan = gate._plan("gnn", workdir=Path(""), gnn_runtime=runtime) assert plan is not None assert plan[0] == ["uv", "run", "python", "scripts/export_gnn_spec.py", "--check"] assert plan[1] == [ + "uv", + "run", + "python", + "scripts/validate_gnn_spec.py", + "--gnn-runtime", + runtime, + "--verify-tag-remote", + "--json", + ] + assert plan[2] == [ runtime, "-m", "gnn.importers", @@ -35,16 +45,29 @@ def test_gnn_arm_runs_the_shared_pin_reader_before_conformance(): "--job", "gnn-conformance", ] - assert plan[2] == [ - "uv", - "run", - "python", - "scripts/validate_gnn_spec.py", - "--gnn-runtime", - runtime, - "--verify-tag-remote", - "--json", - ] + + +def test_gnn_arm_verifies_the_pinned_parser_digests_before_it_parses(): + """``gnn.importers check`` parses the document; the digests must be verified first. + + ``check`` calls ``gnn.importers._validate_spec``, which imports the installed + parser and parses every ``*.gnn.md``. The validator is the command that + verifies the pin's parser digests before it parses anything, so it has to + come first or a tampered parser is executed before its digests are checked. + """ + + plan = gate._plan("gnn", workdir=Path(""), gnn_runtime="/opt/gnn/bin/python") + + assert plan is not None + verify = next( + index + for index, command in enumerate(plan) + if any("validate_gnn_spec.py" in part for part in command) + ) + parses = next( + index for index, command in enumerate(plan) if "gnn.importers" in command + ) + assert verify < parses def test_gnn_arm_omits_the_reader_without_a_runtime(): @@ -137,5 +160,8 @@ def test_gnn_arm_always_asks_the_validator_to_resolve_the_tag_upstream(): plan = gate._plan("gnn", workdir=Path(""), gnn_runtime=runtime) assert plan is not None - assert "--verify-tag-remote" in plan[-1] - assert plan[-1][-1] == "--json" + validator = next( + command for command in plan if any("validate_gnn_spec.py" in part for part in command) + ) + assert "--verify-tag-remote" in validator + assert validator[-1] == "--json"