From 43ae25b34af2723a27846f42136a97b3d15eecbe Mon Sep 17 00:00:00 2001 From: Daniel Ari Friedman Date: Thu, 12 Nov 2026 10:59:40 -0800 Subject: [PATCH] Verify GNN digests before parsing and bind the pin tag to its commit validate_gnn_spec.py ran the parser probe before verifying the installed parser digests and never checked that the pin's tag names the pinned commit. For an installed runtime the package is now located with importlib.util.find_spec and its digests are verified before the probe runs; the parser must be imported from that package. The installer's recorded VCS commit (direct_url.json) is compared with the pin. tag_binding() resolves the tag from a --gnn-root checkout or, with --verify-tag-remote, from git ls-remote (peeled object preferred; the normalized owner/name repository from gnn.importers is expanded to a GitHub URL); a tag that resolves elsewhere fails as tag-commit-mismatch, and an unanswerable --verify-tag-remote ends the run as "not run" (exit 2). The receipt gains tag_binding and revision. The --gnn-root path is kept. No language-version check is added: this pin carries no language_version key. scripts/release_check.py's gnn arm passes --verify-tag-remote. Offline tests in tests/test_gnn_pin_binding.py cover each failure mode; gate-metrics.md is regenerated for the new test count. Co-Authored-By: Claude Opus 5.5 --- .github/AGENTS.md | 4 +- docs/_generated/gate-metrics.md | 4 +- docs/gate.md | 2 +- docs/gnn.md | 24 ++- scripts/README.md | 2 +- scripts/release_check.py | 8 +- scripts/validate_gnn_spec.py | 275 +++++++++++++++++++++++++++++++- tests/test_gnn_pin_binding.py | 255 +++++++++++++++++++++++++++++ tests/test_release_gate.py | 10 ++ 9 files changed, 565 insertions(+), 19 deletions(-) create mode 100644 tests/test_gnn_pin_binding.py diff --git a/.github/AGENTS.md b/.github/AGENTS.md index e99e74d..384d70a 100644 --- a/.github/AGENTS.md +++ b/.github/AGENTS.md @@ -7,7 +7,9 @@ contains only Python 3.11 and runs `--arm test-full`. The `gnn-conformance` job on Python 3.12 with `GNN_RUNTIME_PYTHON` set to the separate runtime provisioned from `specs/gnn-pin.json`. There is no separate `gate` job. Pinned conformance always runs in the `gnn` arm, where the shared `python -m gnn.importers check --repo .` reads the pin and -the pointer; missing runtime dependencies must fail, never skip. PDF rendering remains +the pointer, and `scripts/validate_gnn_spec.py --verify-tag-remote` checks the +pinned v5.1.0 tag resolves to commit `75c834bd` and verifies the parser digests +before parsing; missing runtime dependencies must fail, never skip. PDF rendering remains an explicit local `--arm render` selection. The workflow uses pinned `astral-sh/setup-uv` directly, not the Institute composite; see [`../docs/gate.md`](../docs/gate.md) for the exact action and command map. diff --git a/docs/_generated/gate-metrics.md b/docs/_generated/gate-metrics.md index 0ad72e7..8561d11 100644 --- a/docs/_generated/gate-metrics.md +++ b/docs/_generated/gate-metrics.md @@ -43,8 +43,8 @@ The baseline and current receipts are historical evidence. Timings include host | Selection | Collected tests | | --- | --- | -| full suite | 97 | -| fast (`-m "not slow"`) | 84 | +| full suite | 112 | +| fast (`-m "not slow"`) | 99 | | slow (`-m slow`) | 13 | Recorded run: 97 passed, 0 skipped. diff --git a/docs/gate.md b/docs/gate.md index bf04992..dddad45 100644 --- a/docs/gate.md +++ b/docs/gate.md @@ -34,7 +34,7 @@ measurements. | `test-full` | `uv run pytest --cov=blockference --cov-report=term --cov-report=xml --cov-fail-under=90` | yes | | `docs` | `scripts/check_docs_links.py`, `scripts/gate_metrics.py --check` | yes | | `manuscript` | `scripts/manuscript_variables.py --check`, `scripts/validate_manuscript.py`, `scripts/build_manuscript.py`, `scripts/build_manuscript.py --check`; the `gnn_*` variables come from `gnn.importers` in the pinned runtime | yes | -| `gnn` | `scripts/export_gnn_spec.py --check`, the shared `python -m gnn.importers check --repo .` pin and pointer check, and real pinned `scripts/validate_gnn_spec.py` validation; missing runtime fails | yes | +| `gnn` | `scripts/export_gnn_spec.py --check`, the shared `python -m gnn.importers check --repo .` pin and pointer check, and real pinned `scripts/validate_gnn_spec.py --verify-tag-remote` validation (parser digests before parsing, the pin's tag resolved upstream to the pinned commit; needs network); missing runtime fails | yes | | `smoke` | pipeline, validation, and standalone `run` JSON smoke checks in a temporary directory | yes | | `package` | `uv build`, locked `uv export` constraints for clean-wheel dependencies, and isolated import outside the source tree | yes | | `render` | `scripts/build_manuscript.py --render pdf` (Pandoc + XeLaTeX) | no | diff --git a/docs/gnn.md b/docs/gnn.md index 4f20754..685586d 100644 --- a/docs/gnn.md +++ b/docs/gnn.md @@ -63,8 +63,10 @@ GNN_ROOT=/path/to/Generalized_Notation_Notation \ GNN_PYTHON=/path/to/parser-runtime/bin/python \ uv run python scripts/validate_gnn_spec.py --json -# Installed pinned runtime (what CI uses) -uv run python scripts/validate_gnn_spec.py --gnn-runtime /path/to/venv/bin/python --json +# Installed pinned runtime (what CI uses); --verify-tag-remote resolves the +# pin's tag upstream with `git ls-remote` and needs network access +uv run python scripts/validate_gnn_spec.py --gnn-runtime /path/to/venv/bin/python \ + --verify-tag-remote --json ``` `GNN_ROOT` is a `Generalized_Notation_Notation` checkout: the validator resolves @@ -72,8 +74,8 @@ the pinned commit with `git archive`, verifies every recorded parser digest, and runs the parser from that materialized tree, so a live working copy under development is never imported. `GNN_PYTHON` (or `--gnn-runtime`) selects the interpreter that has the parser dependencies; the GNN checkout's own `.venv` -works. Exit codes are `0` pass, `1` conformance failure, and `2` pinned runtime -or checkout unavailable — an unavailable run is reported as not run, never as a +works. Exit codes are `0` pass, `1` conformance failure, and `2` pinned runtime, +checkout, or requested remote tag resolution unavailable — an unavailable run is reported as not run, never as a pass. With `GNN_ROOT` set and no runtime named, the gate uses the checkout's own `.venv/bin/python` for both the reader and the pin. @@ -94,7 +96,19 @@ and exact CI map. revision, `.github/workflows/ci.yml`, and job `gnn-conformance`. 2. The document parses with the pinned `gnn` parser with no errors or warnings, and the parser version equals the pinned tag; every recorded parser digest - matches the installed (or archived) parser files. + matches the installed (or archived) parser files, and for an installed + runtime that check runs *before* the document is parsed and the parser must be + imported from the verified package. + The pin's `tag` must resolve to the pinned `commit` (`tag-commit-mismatch`): + locally in a `GNN_ROOT` checkout, or upstream with `--verify-tag-remote` + (`git ls-remote`, the peeled commit of an annotated tag; the release gate's + `gnn` arm passes the flag, so it needs network access). The commit the + installer recorded for the runtime (`direct_url.json` `vcs_info.commit_id`) + must equal the pin's commit. Without a checkout or the flag, and for an + install with no recorded VCS commit, the receipt reports `tag_binding` / + `revision` as `unresolvable-offline` rather than a pass; with + `--verify-tag-remote`, a tag that cannot be resolved ends the run as *not + run* (exit `2`). 3. The declared `ModelParameters` are read back and used to build the runtime model, whose `A`/`B`/`C`/`D`/`E` arrays must equal the declared matrices exactly; declared `StateSpaceBlock` dimensions must match the array shapes. diff --git a/scripts/README.md b/scripts/README.md index a2306ce..80b4ff9 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -13,7 +13,7 @@ Repository gates and generators, all run from the repository root: | `execute_notebooks.py` | Parses and executes every tracked notebook from the repository root. | | `validate_notebooks.py` | Notebook hygiene checks before execution. | | `export_gnn_spec.py` | Writes or verifies `specs/gridworld_3x3.gnn.md` from the implementation. | -| `validate_gnn_spec.py` | Validates the committed specification with the pinned parser, using a separate interpreter selected by `--gnn-runtime` or the documented environment variables. | +| `validate_gnn_spec.py` | Validates the committed specification with the pinned parser, using a separate interpreter selected by `--gnn-runtime` or the documented environment variables. Verifies the parser digests before parsing, binds the pin's tag to its commit (`--gnn-root` checkout or `--verify-tag-remote`; mismatch fails as `tag-commit-mismatch`), and compares the installed runtime's recorded commit with the pin. | | `gnn_parser_probe.py` | Imports the upstream `gnn` parser; runs as a subprocess under the pin. | | `gnn_importer_report.py` | Imports the upstream `gnn.importers` reader and prints this repository's normalized pin and `gnn-importer-variables/v1` map for the two scripts above. | diff --git a/scripts/release_check.py b/scripts/release_check.py index 11f5e88..158c556 100644 --- a/scripts/release_check.py +++ b/scripts/release_check.py @@ -201,9 +201,13 @@ def _plan( "--job", "gnn-conformance", ] ) - validator = ["uv", "run", "python", "scripts/validate_gnn_spec.py", "--json"] + # The pin's tag is resolved upstream (needs network), so a tag that was + # moved off the pinned commit fails the release gate. + validator = [ + "uv", "run", "python", "scripts/validate_gnn_spec.py", "--verify-tag-remote", "--json", + ] if gnn_runtime: - validator[-1:-1] = ["--gnn-runtime", gnn_runtime] + validator[-2:-2] = ["--gnn-runtime", gnn_runtime] commands.append(validator) return commands if arm == "smoke": diff --git a/scripts/validate_gnn_spec.py b/scripts/validate_gnn_spec.py index 2a78744..c8f5767 100644 --- a/scripts/validate_gnn_spec.py +++ b/scripts/validate_gnn_spec.py @@ -11,7 +11,19 @@ the live working tree is never imported; * an installed pinned runtime (``--gnn-runtime`` or ``GNN_RUNTIME_PYTHON``): an interpreter whose ``gnn`` package is installed at the pinned revision; the - distribution version and the installed parser file digests are verified. + installed parser file digests are verified *before* the document is parsed + (the package is located with the interpreter's import machinery, never + imported to parse), the VCS commit the installer recorded + (``direct_url.json``) is compared with the pin, and the parser must be imported + from the verified package. + +The pin's ``tag`` and ``commit`` name one revision. A GNN checkout resolves +``refs/tags/^{commit}`` locally; ``--verify-tag-remote`` resolves it upstream +with ``git ls-remote`` (the peeled commit of an annotated tag). A tag that +resolves elsewhere fails as ``tag-commit-mismatch``. Without a checkout or the +flag the binding is recorded in the receipt as ``unresolvable-offline``, never +as a pass, and no network is used. With the flag, a tag that cannot be resolved +ends the run as *not run* (exit ``2``). Either way the document is parsed by ``scripts/gnn_parser_probe.py`` in a subprocess and checked for conformance: the declared fixture parameters, the @@ -19,8 +31,9 @@ by ``blockference.gnn``, the pinned semantic validator, the B-tensor orientation diagnostic, and the consistency checker. -Exit codes: ``0`` pass, ``1`` conformance failure, ``2`` pinned runtime or -checkout unavailable (the run is then explicitly *not run*, never a pass). +Exit codes: ``0`` pass, ``1`` conformance failure, ``2`` pinned runtime, +checkout, or requested remote tag resolution unavailable (the run is then +explicitly *not run*, never a pass). Usage: @@ -30,7 +43,8 @@ uv run python scripts/validate_gnn_spec.py --json # Pinned installed runtime (used by CI) - uv run python scripts/validate_gnn_spec.py --gnn-runtime /path/to/venv/bin/python + uv run python scripts/validate_gnn_spec.py --gnn-runtime /path/to/venv/bin/python \\ + --verify-tag-remote """ from __future__ import annotations @@ -63,6 +77,40 @@ DEFAULT_RECEIPT = ROOT / "docs" / "_build" / "gnn" / "validation-receipt.json" PACKAGE_PREFIX = "src/gnn/" +#: Named failure: the pin's tag and commit are two spellings of one revision. +TAG_COMMIT_MISMATCH = "tag-commit-mismatch" + +# Locating the installed package and reading its installer metadata never parses +# a document, so the digest and revision checks run before the probe. +LOCATE_EXPRESSION = ( + "import importlib.util, json; " + "spec = importlib.util.find_spec('gnn'); " + "print(json.dumps({'module': spec.origin if spec else None, " + "'locations': [str(path) for path in (spec.submodule_search_locations or [])] " + "if spec else []}))" +) +REVISION_PROGRAM = """ +import importlib.metadata as md +import json + +records = [] +for name in md.packages_distributions().get("gnn", []): + try: + distribution = md.distribution(name) + except md.PackageNotFoundError: + continue + record = {"name": name, "version": distribution.version, "commit": None} + raw = distribution.read_text("direct_url.json") + if raw: + try: + direct = json.loads(raw) + except ValueError: + direct = {} + record["commit"] = (direct.get("vcs_info") or {}).get("commit_id") + records.append(record) +print(json.dumps(records)) +""" + class Unavailable(RuntimeError): """Raised when the pinned parser runtime or checkout is not available.""" @@ -128,6 +176,176 @@ def _verify_digests(package_dir: Path, digests: dict[str, str], *, origin: str) return observed +def _run_text(command: list[str], *, what: str) -> str: + try: + completed = subprocess.run(command, capture_output=True, text=True, check=False) + except OSError as exc: + raise Unavailable(f"{what}: {exc}") from exc + if completed.returncode != 0: + detail = (completed.stderr or completed.stdout).strip().splitlines() + tail = " | ".join(detail[-3:]) if detail else "no output" + raise Unavailable(f"{what}: {tail}") + return completed.stdout + + +def locate_package(python: str) -> Path: + """Return the ``gnn`` package directory ``python`` would import, without importing it.""" + + output = _run_text( + [python, "-c", LOCATE_EXPRESSION], what=f"cannot locate the gnn package in {python}" + ) + try: + located = json.loads(output) + except json.JSONDecodeError as exc: + raise Unavailable(f"cannot locate the gnn package in {python}: {exc}") from exc + if not located.get("module"): + raise Unavailable( + f"the gnn package is not installed in {python}; provision the pinned revision " + "or pass --gnn-root" + ) + locations = located.get("locations") or [] + return ( + Path(locations[0]).resolve() if locations else Path(located["module"]).resolve().parent + ) + + +def installed_revision(python: str) -> dict[str, Any]: + """Return the VCS commit the installer recorded for the ``gnn`` distribution. + + ``direct_url.json`` carries ``vcs_info.commit_id`` only for an install from a + git URL; an editable or index install has no commit to compare and is + reported as ``unresolvable-offline`` rather than assumed to match. + """ + + output = _run_text( + [python, "-c", REVISION_PROGRAM], what=f"cannot read the installed gnn metadata in {python}" + ) + try: + records = json.loads(output) + except json.JSONDecodeError as exc: + raise Unavailable(f"cannot read the installed gnn metadata in {python}: {exc}") from exc + commits = sorted({record["commit"] for record in records if record.get("commit")}) + if not commits: + return { + "status": "unresolvable-offline", + "commit": None, + "detail": "the installer recorded no VCS commit (not installed from a git URL)", + } + if len(commits) > 1: + return { + "status": "conflicting", + "commit": None, + "detail": f"the installer recorded several commits {commits}", + } + return {"status": "verified", "commit": commits[0], "detail": "direct_url.json vcs_info.commit_id"} + + +def revision_failures(revision: dict[str, Any], pin: dict[str, Any]) -> list[str]: + """Return failures when the installed runtime's recorded commit is not the pin.""" + + if revision.get("status") == "verified" and revision.get("commit") != pin["commit"]: + return [ + f"{TAG_COMMIT_MISMATCH}: the installed runtime records commit " + f"{revision.get('commit')} but the pin records {pin['commit']}" + ] + if revision.get("status") == "conflicting": + return [ + f"{TAG_COMMIT_MISMATCH}: the installed runtime metadata is ambiguous: " + f"{revision.get('detail')}" + ] + return [] + + +def remote_url(repository: str) -> str: + """Return a git URL for the pin's repository. + + ``gnn.importers`` normalizes the pin's repository to ``owner/name`` for a + GitHub URL; ``git ls-remote`` needs the full URL. + """ + + if "://" in repository or repository.startswith("git@") or repository.startswith("/"): + return repository + return f"https://github.com/{repository}" + + +def tag_binding( + gnn_root: Path | None, pin: dict[str, Any], *, remote: bool = False +) -> dict[str, Any]: + """Bind the pin's ``tag`` to its ``commit`` in an available object store. + + A GNN checkout resolves the tag locally; ``remote`` resolves it upstream with + ``git ls-remote`` (also when the checkout lacks the tag). When neither is + available the status says so instead of assuming the binding holds. + """ + + tag = pin["tag"] + record: dict[str, Any] = { + "status": "unresolvable-offline", + "tag": tag, + "commit": pin["commit"], + "resolved": None, + "detail": "no GNN object store was supplied and --verify-tag-remote was not requested", + } + if gnn_root is not None and (gnn_root / ".git").exists(): + try: + local = subprocess.run( + [ + "git", "-C", str(gnn_root), "rev-parse", "--verify", "-q", + f"refs/tags/{tag}^{{commit}}", + ], + capture_output=True, + text=True, + check=False, + ) + except OSError as exc: + record["detail"] = f"git could not run: {exc}" + else: + if local.returncode == 0 and local.stdout.strip(): + record.update( + status="verified", + resolved=local.stdout.strip(), + detail=f"local object store {gnn_root}", + ) + return record + record["detail"] = f"tag {tag} is not present in {gnn_root}" + if not remote: + return record + try: + completed = subprocess.run( + ["git", "ls-remote", "--tags", remote_url(pin["repository"]), f"refs/tags/{tag}*"], + capture_output=True, + text=True, + check=False, + ) + except OSError as exc: + record["detail"] = f"git ls-remote could not run: {exc}" + return record + if completed.returncode != 0: + record["detail"] = f"git ls-remote failed: {completed.stderr.strip()}" + return record + pairs = [line.split() for line in completed.stdout.splitlines() if len(line.split()) == 2] + # An annotated tag lists the tag object and, peeled, the commit it names; the + # peeled entry is the commit. + for wanted in (f"refs/tags/{tag}^{{}}", f"refs/tags/{tag}"): + for sha, name in pairs: + if name == wanted: + record.update(status="verified", resolved=sha, detail="git ls-remote") + return record + record["detail"] = f"tag {tag} is not listed by {pin['repository']}" + return record + + +def tag_failures(tag: dict[str, Any] | None, pin: dict[str, Any]) -> list[str]: + """Return the named failure when the pin's tag resolves to a different commit.""" + + if tag is not None and tag.get("status") == "verified" and tag.get("resolved") != pin["commit"]: + return [ + f"{TAG_COMMIT_MISMATCH}: tag {pin['tag']} resolves to {tag.get('resolved')} " + f"but the pin records {pin['commit']}" + ] + return [] + + def materialize_pin(gnn_root: Path, pin: dict[str, Any], target: Path) -> tuple[Path, str]: """Materialize the pinned commit into ``target`` and return it with its origin. @@ -215,6 +433,8 @@ def conformance_failures( pin: dict[str, Any], source: Path | None, spec_text: str, + package_dir: Path | None = None, + tag: dict[str, Any] | None = None, ) -> list[str]: """Return every conformance failure between the document and the model.""" @@ -226,10 +446,17 @@ def conformance_failures( failures.append( f"parser version {payload['gnn_version']} differs from the pinned {expected_version}" ) + failures.extend(tag_failures(tag, pin)) if source is not None: module = Path(str(payload["gnn_module"])).resolve() if not module.is_relative_to((source / "src").resolve()): failures.append(f"parser was imported from {module}, outside the pinned tree {source}") + elif package_dir is not None: + module = Path(str(payload["gnn_module"])).resolve() + if not module.is_relative_to(package_dir): + failures.append( + f"parser was imported from {module}, outside the verified package {package_dir}" + ) if not parser["success"]: failures.append("pinned parser reported failure") for message in parser["errors"]: @@ -324,6 +551,11 @@ def main(argv: list[str] | None = None) -> int: default=os.environ.get("GNN_PYTHON"), help="parser interpreter used with --gnn-root (default: $GNN_PYTHON or this interpreter)", ) + parser.add_argument( + "--verify-tag-remote", + action="store_true", + help="resolve the pin's tag upstream with git ls-remote (needs network access)", + ) parser.add_argument("--spec", type=Path, default=GNN_SPEC_PATH) parser.add_argument("--receipt", type=Path, default=DEFAULT_RECEIPT) parser.add_argument("--json", action="store_true", help="print the receipt to stdout") @@ -347,6 +579,13 @@ def main(argv: list[str] | None = None) -> int: print(f"gnn validation not run: {exc}", file=sys.stderr) return 2 + tag = tag_binding(args.gnn_root, pin, remote=args.verify_tag_remote) + if args.verify_tag_remote and tag["status"] != "verified": + # An explicit request to verify the tag must not degrade into a silent + # pass when nothing could answer (offline, tag not listed). + print(f"gnn validation not run: --verify-tag-remote: {tag['detail']}", file=sys.stderr) + return 2 + with tempfile.TemporaryDirectory(prefix="activeblockference-gnn-") as directory: output = Path(directory) / "probe.json" try: @@ -354,14 +593,30 @@ def main(argv: list[str] | None = None) -> int: source, origin = materialize_pin( args.gnn_root, pin, Path(directory) / "gnn-pin" ) + package_dir = None + if (args.gnn_root / ".git").exists(): + revision = { + "status": "verified", + "commit": pin["commit"], + "detail": "git archive of the pinned commit", + } + else: + revision = { + "status": "unresolvable-offline", + "commit": None, + "detail": "digest-verified source tree without git metadata", + } payload = run_probe( args.python or sys.executable, args.spec, output, source=source ) else: source = None - payload = run_probe(args.gnn_runtime, args.spec, output, source=None) - package_dir = Path(str(payload["gnn_module"])).resolve().parent + # The digest and revision checks come before the parser runs, so a + # tampered parser is never reached by parsing first. + package_dir = locate_package(args.gnn_runtime) _verify_digests(package_dir, pin["parser_files"], origin=args.gnn_runtime) + revision = installed_revision(args.gnn_runtime) + payload = run_probe(args.gnn_runtime, args.spec, output, source=None) origin = f"installed runtime {args.gnn_runtime} ({package_dir})" except Unavailable as exc: print(f"gnn validation not run: {exc}", file=sys.stderr) @@ -372,7 +627,10 @@ def main(argv: list[str] | None = None) -> int: pin=pin, source=source, spec_text=args.spec.read_text(encoding="utf-8"), + package_dir=package_dir, + tag=tag, ) + failures.extend(revision_failures(revision, pin)) receipt = { "ok": not failures, @@ -381,6 +639,8 @@ def main(argv: list[str] | None = None) -> int: "pin_source": origin, "gnn_version": payload["gnn_version"], "gnn_module": payload["gnn_module"], + "tag_binding": tag, + "revision": revision, "parser_success": payload["parser"]["success"], "semantic_valid": payload["semantic"].get("is_valid"), "orientation": [ @@ -400,7 +660,8 @@ def main(argv: list[str] | None = None) -> int: return 1 print( f"gnn validation passed: {args.spec} parsed by gnn {payload['gnn_version']} " - f"({origin}); receipt {args.receipt}" + f"({origin}); tag {tag['tag']} -> {tag['status']}; " + f"revision {revision['status']}; receipt {args.receipt}" ) return 0 diff --git a/tests/test_gnn_pin_binding.py b/tests/test_gnn_pin_binding.py new file mode 100644 index 0000000..9c597a2 --- /dev/null +++ b/tests/test_gnn_pin_binding.py @@ -0,0 +1,255 @@ +"""Offline tests for the pin-binding checks in ``scripts/validate_gnn_spec.py``. + +No test here uses the network or an installed GNN runtime: remote tag resolution +is exercised against a stubbed ``git ls-remote``, the checkout path uses a +throwaway local git repository, and the runtime is a stub. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +import pytest + +import scripts.validate_gnn_spec as vgs + +COMMIT = "a" * 40 +OTHER = "b" * 40 + + +def _pin(**overrides) -> dict: + pin = { + "repository": "https://example.invalid/Generalized_Notation_Notation", + "tag": "v9.9.9", + "commit": COMMIT, + "parser_files": {"src/gnn/__init__.py": "0" * 64}, + } + pin.update(overrides) + return pin + + +def _git(cwd: Path, *args: str) -> str: + completed = subprocess.run( + ["git", "-C", str(cwd), "-c", "user.name=t", "-c", "user.email=t@example.invalid", *args], + capture_output=True, + text=True, + check=True, + ) + return completed.stdout.strip() + + +@pytest.fixture +def checkout(tmp_path: Path) -> tuple[Path, str]: + repo = tmp_path / "gnn" + repo.mkdir() + _git(repo, "init", "-q") + (repo / "f").write_text("x", encoding="utf-8") + _git(repo, "add", "f") + _git(repo, "commit", "-q", "-m", "c") + return repo, _git(repo, "rev-parse", "HEAD") + + +def _stub_ls_remote(monkeypatch: pytest.MonkeyPatch, stdout: str, returncode: int = 0) -> list: + calls: list[list[str]] = [] + + def fake_run(command, **_kwargs): + calls.append(list(command)) + return subprocess.CompletedProcess(command, returncode, stdout=stdout, stderr="boom") + + monkeypatch.setattr(vgs.subprocess, "run", fake_run) + return calls + + +def test_local_annotated_tag_resolves_to_the_peeled_commit(checkout) -> None: + repo, head = checkout + _git(repo, "tag", "-a", "-m", "annotated", "v9.9.9") + pin = _pin(commit=head) + binding = vgs.tag_binding(repo, pin) + + assert binding["status"] == "verified" + assert binding["resolved"] == head + assert vgs.tag_failures(binding, pin) == [] + + +def test_local_tag_pointing_elsewhere_is_a_named_failure(checkout) -> None: + repo, head = checkout + _git(repo, "tag", "v9.9.9") + pin = _pin(commit=OTHER) + failures = vgs.tag_failures(vgs.tag_binding(repo, pin), pin) + + assert len(failures) == 1 + assert failures[0].startswith(vgs.TAG_COMMIT_MISMATCH) + assert head in failures[0] and OTHER in failures[0] + + +def test_offline_without_a_store_is_explicitly_unresolved( + monkeypatch: pytest.MonkeyPatch, checkout +) -> None: + repo, _head = checkout # a checkout that lacks the tag + + def no_network(command, **_kwargs): + assert "ls-remote" not in command, "offline binding must not use the network" + return subprocess.CompletedProcess(command, 1, stdout="", stderr="") + + monkeypatch.setattr(vgs.subprocess, "run", no_network) + for store in (None, repo): + binding = vgs.tag_binding(store, _pin()) + assert binding["status"] == "unresolvable-offline" + assert binding["resolved"] is None + assert vgs.tag_failures(binding, _pin()) == [] + + +def test_remote_prefers_the_peeled_commit(monkeypatch: pytest.MonkeyPatch) -> None: + calls = _stub_ls_remote( + monkeypatch, f"{OTHER}\trefs/tags/v9.9.9\n{COMMIT}\trefs/tags/v9.9.9^{{}}\n" + ) + binding = vgs.tag_binding(None, _pin(), remote=True) + + assert calls[0][:3] == ["git", "ls-remote", "--tags"] + assert binding["status"] == "verified" + assert binding["resolved"] == COMMIT + assert vgs.tag_failures(binding, _pin()) == [] + + +def test_remote_falls_back_when_the_checkout_lacks_the_tag( + monkeypatch: pytest.MonkeyPatch, checkout +) -> None: + repo, _head = checkout + real_run = subprocess.run + + def run(command, **kwargs): + if command[:2] == ["git", "ls-remote"]: + return subprocess.CompletedProcess(command, 0, f"{COMMIT}\trefs/tags/v9.9.9\n", "") + return real_run(command, **kwargs) + + monkeypatch.setattr(vgs.subprocess, "run", run) + binding = vgs.tag_binding(repo, _pin(), remote=True) + + assert binding["status"] == "verified" + assert binding["detail"] == "git ls-remote" + + +def test_remote_tag_resolving_elsewhere_fails_by_name(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_ls_remote(monkeypatch, f"{OTHER}\trefs/tags/v9.9.9\n") + binding = vgs.tag_binding(None, _pin(), remote=True) + + assert [f.split(":")[0] for f in vgs.tag_failures(binding, _pin())] == [vgs.TAG_COMMIT_MISMATCH] + + +def test_remote_ignores_look_alike_tags(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_ls_remote(monkeypatch, f"{OTHER}\trefs/tags/v9.9.9-rc1\n") + binding = vgs.tag_binding(None, _pin(), remote=True) + + assert binding["status"] == "unresolvable-offline" + assert "not listed" in binding["detail"] + + +def test_failed_remote_is_unresolved_never_a_pass(monkeypatch: pytest.MonkeyPatch) -> None: + _stub_ls_remote(monkeypatch, "", returncode=128) + binding = vgs.tag_binding(None, _pin(), remote=True) + + assert binding["status"] == "unresolvable-offline" + assert "ls-remote failed" in binding["detail"] + + +def test_installed_commit_must_match_the_pin() -> None: + pin = _pin() + + assert vgs.revision_failures({"status": "verified", "commit": COMMIT}, pin) == [] + assert vgs.revision_failures({"status": "unresolvable-offline", "commit": None}, pin) == [] + assert vgs.revision_failures({"status": "verified", "commit": OTHER}, pin)[0].startswith( + vgs.TAG_COMMIT_MISMATCH + ) + assert vgs.revision_failures({"status": "conflicting", "detail": "two"}, pin)[0].startswith( + vgs.TAG_COMMIT_MISMATCH + ) + + +def _run_main(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, *extra: str): + probed: list[str] = [] + + def fake_probe(*_args, **_kwargs): + probed.append("probe") + raise AssertionError("the parser must not run before the digests verify") + + monkeypatch.setattr(vgs, "run_probe", fake_probe) + monkeypatch.setattr( + vgs, "read_importer_report", lambda *_a, **_k: {"pin": _pin(), "variables": {}} + ) + code = vgs.main( + ["--gnn-runtime", sys.executable, "--receipt", str(tmp_path / "r.json"), *extra] + ) + return code, probed + + +def test_digests_are_verified_before_the_document_is_parsed( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + package = tmp_path / "gnn" + package.mkdir() + (package / "__init__.py").write_text("tampered", encoding="utf-8") + monkeypatch.setattr(vgs, "locate_package", lambda *_a, **_k: package) + + code, probed = _run_main(monkeypatch, tmp_path) + + assert code == 2 + assert probed == [] + assert "digest mismatch" in capsys.readouterr().err + + +def test_a_missing_pinned_package_is_not_run_before_parsing( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path +) -> None: + def absent(*_args, **_kwargs): + raise vgs.Unavailable("the gnn package is not installed") + + monkeypatch.setattr(vgs, "locate_package", absent) + code, probed = _run_main(monkeypatch, tmp_path) + + assert code == 2 + assert probed == [] + + +def test_verify_tag_remote_that_cannot_answer_is_not_run_not_a_pass( + monkeypatch: pytest.MonkeyPatch, tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + _stub_ls_remote(monkeypatch, "", returncode=128) + code, probed = _run_main(monkeypatch, tmp_path, "--verify-tag-remote") + + assert code == 2 + assert probed == [] + assert "--verify-tag-remote" in capsys.readouterr().err + + +def test_a_remote_tag_moved_off_the_pinned_commit_reaches_the_conformance_failures() -> None: + binding = {"status": "verified", "tag": "v9.9.9", "commit": COMMIT, "resolved": OTHER} + payload = { + "gnn_version": "9.9.9", + "gnn_module": "/nowhere/gnn/__init__.py", + "parser": { + "success": True, + "errors": [], + "warnings": [], + "model_name": "x", + "variables": [], + "version": "1.1", + }, + "semantic": {"is_valid": True}, + "orientation": {}, + "consistency": {"consistent": True}, + } + failures = vgs.conformance_failures( + payload, pin=_pin(), source=None, spec_text="", tag=binding + ) + + assert any(f.startswith(vgs.TAG_COMMIT_MISMATCH) for f in failures) + + +def test_normalized_owner_name_pins_become_github_urls(monkeypatch: pytest.MonkeyPatch) -> None: + assert vgs.remote_url("Org/Repo") == "https://github.com/Org/Repo" + assert vgs.remote_url("https://example.invalid/x") == "https://example.invalid/x" + calls = _stub_ls_remote(monkeypatch, "") + vgs.tag_binding(None, _pin(repository="Org/Repo"), remote=True) + assert "https://github.com/Org/Repo" in calls[0] diff --git a/tests/test_release_gate.py b/tests/test_release_gate.py index a6320b3..f79837b 100644 --- a/tests/test_release_gate.py +++ b/tests/test_release_gate.py @@ -42,6 +42,7 @@ def test_gnn_arm_runs_the_shared_pin_reader_before_conformance(): "scripts/validate_gnn_spec.py", "--gnn-runtime", runtime, + "--verify-tag-remote", "--json", ] @@ -129,3 +130,12 @@ def test_render_provenance_requires_the_pdf(tmp_path, monkeypatch): monkeypatch.setitem(gate.RENDER_TARGETS, "pdf", tmp_path / "absent.pdf") with pytest.raises(FileNotFoundError): gate._render_artifact() + + +def test_gnn_arm_always_asks_the_validator_to_resolve_the_tag_upstream(): + for runtime in (None, "/opt/gnn-runtime/bin/python"): + plan = gate._plan("gnn", workdir=Path(""), gnn_runtime=runtime) + + assert plan is not None + assert "--verify-tag-remote" in plan[-1] + assert plan[-1][-1] == "--json"