diff --git a/.github/GNN.md b/.github/GNN.md index a8897e1..7e2ba87 100644 --- a/.github/GNN.md +++ b/.github/GNN.md @@ -20,6 +20,16 @@ What it validates, in order: exemplar at the pin's revision and validates that instead, and the JSON receipt says `"source": "pinned-exemplar"`. +The type checker's verdict is tri-state, not boolean: `validate_gnn_files` +returns `True` when every file validated, `2` when one or more files had +type/consistency problems **or when no GNN file was found**, and `False` on a +hard failure. Only the literal `True` passes the arm. `2` is truthy, so a +truthiness test would report an invalid model set as `"result": "verified"`; +instead any other value exits non-zero and the refusal names the value the +checker returned. `.github/scripts/tests/test_gnn_conformance.py` pins all four +outcomes (`True`, `2`, `False`, `None`/`0`) without needing the parser +installed. + The arm needs network access and a Python version the pinned release supports (`>=3.11,<3.15`); CI uses 3.12. The local recipe, including a throwaway virtual environment, is in [`RUN_GUIDE.md`](../docs/RUN_GUIDE.md#running-the-gnn-arm-locally). diff --git a/.github/scripts/gnn_conformance.py b/.github/scripts/gnn_conformance.py index 05330bc..322f39e 100755 --- a/.github/scripts/gnn_conformance.py +++ b/.github/scripts/gnn_conformance.py @@ -109,8 +109,18 @@ def _check_types(sketches: list[Path]) -> None: model_dir.mkdir() for index, sketch in enumerate(sketches): shutil.copy2(sketch, model_dir / f"{index:03d}_{sketch.name}") - if not GNNTypeChecker().validate_gnn_files(model_dir, Path(output_dir)): - raise SystemExit("GNN type checker rejected the checked sketches") + checked = GNNTypeChecker().validate_gnn_files(model_dir, Path(output_dir)) + # ``validate_gnn_files`` is tri-state, not boolean: ``True`` when every + # file validated, ``2`` when one or more files were invalid *or* no GNN + # file was found, ``False`` on a hard failure. ``2`` is truthy, so a + # truthiness test passes an invalid model set; only ``True`` may pass. + if checked is not True: + raise SystemExit( + "GNN type checker did not verify the checked sketches: " + f"validate_gnn_files returned {checked!r} " + "(True = every file validated, 2 = invalid files or none found, " + "False = hard failure)" + ) def verify(root: Path, pin: dict) -> dict: diff --git a/.github/scripts/tests/test_gnn_conformance.py b/.github/scripts/tests/test_gnn_conformance.py new file mode 100644 index 0000000..f3f4703 --- /dev/null +++ b/.github/scripts/tests/test_gnn_conformance.py @@ -0,0 +1,158 @@ +"""Verdict tests for the pinned GNN conformance arm. + +``GNNTypeChecker.validate_gnn_files`` is tri-state, not boolean: ``True`` when +every file validated, ``2`` when one or more files were invalid or no GNN file +was found, ``False`` on a hard failure. Only ``True`` may pass the arm, so the +arm must refuse ``2`` even though ``2`` is truthy — the regression these tests +pin down, because a truthiness test printed ``"result": "verified"`` for an +invalid model set. + +The pinned parser is a third-party install the ``tests`` arm deliberately does +not require, so every case here stubs ``gnn`` in ``sys.modules``. The arm's own +code — the copying, the call, the verdict, the receipt — runs unchanged. +""" + +from __future__ import annotations + +import contextlib +import io +import json +import sys +import tempfile +import types +import unittest +from pathlib import Path +from unittest import mock + +SCRIPTS = Path(__file__).resolve().parents[1] +ROOT = SCRIPTS.parents[1] +if str(SCRIPTS) not in sys.path: + sys.path.insert(0, str(SCRIPTS)) + +import gnn_conformance + +PIN = json.loads((ROOT / gnn_conformance.PIN_PATH).read_text(encoding="utf-8")) + + +class _Parsed: + """Stand-in for a parsed sketch with no parse errors.""" + + parse_errors: list[str] = [] + + +def _stub_modules(verdict: object, *, version: str = "5.0.3") -> dict[str, object]: + """Return ``sys.modules`` entries implementing the pinned parser surface.""" + gnn = types.ModuleType("gnn") + gnn.__version__ = version # type: ignore[attr-defined] + + parsers = types.ModuleType("gnn.parsers") + basic = types.ModuleType("gnn.parsers.basic") + basic.parse_gnn_formal = lambda path: _Parsed() # type: ignore[attr-defined] + basic.validate_gnn_syntax = lambda text, **kwargs: (True, []) # type: ignore[attr-defined] + + checker_module = types.ModuleType("gnn.type_checker") + + class _Checker: + """Records the model set it was handed, then returns the verdict.""" + + def __init__(self) -> None: + _Checker.last_calls = self.calls = [] + + def validate_gnn_files(self, target_dir, output_dir, *args, **kwargs): + Path(output_dir).mkdir(parents=True, exist_ok=True) + self.calls.append(sorted(path.name for path in Path(target_dir).iterdir())) + return verdict + + _Checker.last_calls = [] # type: ignore[attr-defined] + checker_module.GNNTypeChecker = _Checker # type: ignore[attr-defined] + + return { + "gnn": gnn, + "gnn.parsers": parsers, + "gnn.parsers.basic": basic, + "gnn.type_checker": checker_module, + } + + +class TypeCheckVerdictTests(unittest.TestCase): + """``_check_types`` may pass only on the literal ``True``.""" + + def setUp(self) -> None: + self._temp = tempfile.TemporaryDirectory() + self.root = Path(self._temp.name).resolve() + self.sketch = self.root / "m.gnn.md" + self.sketch.write_text("# sketch\n", encoding="utf-8") + + def tearDown(self) -> None: + self._temp.cleanup() + + def _run(self, verdict: object) -> None: + with mock.patch.dict(sys.modules, _stub_modules(verdict)): + gnn_conformance._check_types([self.sketch]) + + def test_true_passes(self) -> None: + stubs = _stub_modules(True) + with mock.patch.dict(sys.modules, stubs): + gnn_conformance._check_types([self.sketch]) + calls = stubs["gnn.type_checker"].GNNTypeChecker.last_calls # type: ignore[attr-defined] + self.assertEqual(calls, [["000_m.gnn.md"]]) + + def test_two_is_refused_and_named(self) -> None: + with self.assertRaises(SystemExit) as raised: + self._run(2) + self.assertIn("returned 2", str(raised.exception.code)) + + def test_false_is_refused_and_named(self) -> None: + with self.assertRaises(SystemExit) as raised: + self._run(False) + self.assertIn("returned False", str(raised.exception.code)) + + def test_none_is_refused_and_named(self) -> None: + with self.assertRaises(SystemExit) as raised: + self._run(None) + self.assertIn("returned None", str(raised.exception.code)) + + def test_zero_is_refused_and_named(self) -> None: + with self.assertRaises(SystemExit) as raised: + self._run(0) + self.assertIn("returned 0", str(raised.exception.code)) + + +class RunVerdictTests(unittest.TestCase): + """The arm must not print a ``verified`` receipt for a refused verdict.""" + + def setUp(self) -> None: + self._temp = tempfile.TemporaryDirectory() + self.root = Path(self._temp.name).resolve() + pin_path = self.root / gnn_conformance.PIN_PATH + pin_path.parent.mkdir(parents=True) + pin_path.write_text(json.dumps(PIN), encoding="utf-8") + (self.root / "model.gnn.md").write_text("# sketch\n", encoding="utf-8") + + def tearDown(self) -> None: + self._temp.cleanup() + + def _run(self, verdict: object) -> tuple[int | None, str]: + stdout, stderr = io.StringIO(), io.StringIO() + code: int | None = 0 + with mock.patch.dict(sys.modules, _stub_modules(verdict)): + with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr): + code = gnn_conformance.run(self.root) + return code, stdout.getvalue() + stderr.getvalue() + + def test_invalid_models_fail_the_arm_without_a_receipt(self) -> None: + with self.assertRaises(SystemExit) as raised: + self._run(2) + self.assertIn("returned 2", str(raised.exception.code)) + + def test_verified_models_print_the_receipt(self) -> None: + code, output = self._run(True) + self.assertEqual(code, 0) + receipt = json.loads(output.strip().splitlines()[-1]) + self.assertEqual(receipt["result"], "verified") + self.assertEqual(receipt["source"], "repository") + self.assertEqual(receipt["models"], 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/docs/RUN_GUIDE.md b/docs/RUN_GUIDE.md index 158f117..fdbdec7 100644 --- a/docs/RUN_GUIDE.md +++ b/docs/RUN_GUIDE.md @@ -97,7 +97,10 @@ schema and repository identity, installs with `--no-deps` plus the runtime dependencies `defusedxml`, `psutil`, and `pyyaml`, then checks that the installed `gnn.__version__` equals the tag without its `v`. Parsing then covers every `*.gnn.md` sketch in the repository -(full parse, syntax validation, type checking). This archive carries none, so +(full parse, syntax validation, type checking). The type-check verdict passes +only when `validate_gnn_files` returns the literal `True`; its `2` +(`SUCCESS_WITH_WARNINGS`, which covers invalid files and an empty model set) is +a failure, as `.github/GNN.md` records. This archive carries none, so the arm validates the pinned upstream exemplar `input/gnn_files/basics/static_perception.md` at the pinned revision and prints a JSON receipt naming `"source": "pinned-exemplar"`.