Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions .github/GNN.md
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,16 @@ What it validates, in order:
exemplar at the pin's revision and validates that instead, and the JSON
receipt says `"source": "pinned-exemplar"`.

The type checker's verdict is tri-state, not boolean: `validate_gnn_files`
returns `True` when every file validated, `2` when one or more files had
type/consistency problems **or when no GNN file was found**, and `False` on a
hard failure. Only the literal `True` passes the arm. `2` is truthy, so a
truthiness test would report an invalid model set as `"result": "verified"`;
instead any other value exits non-zero and the refusal names the value the
checker returned. `.github/scripts/tests/test_gnn_conformance.py` pins all four
outcomes (`True`, `2`, `False`, `None`/`0`) without needing the parser
installed.

The arm needs network access and a Python version the pinned release supports
(`>=3.11,<3.15`); CI uses 3.12. The local recipe, including a throwaway virtual
environment, is in [`RUN_GUIDE.md`](../docs/RUN_GUIDE.md#running-the-gnn-arm-locally).
Expand Down
14 changes: 12 additions & 2 deletions .github/scripts/gnn_conformance.py
Original file line number Diff line number Diff line change
Expand Up @@ -109,8 +109,18 @@ def _check_types(sketches: list[Path]) -> None:
model_dir.mkdir()
for index, sketch in enumerate(sketches):
shutil.copy2(sketch, model_dir / f"{index:03d}_{sketch.name}")
if not GNNTypeChecker().validate_gnn_files(model_dir, Path(output_dir)):
raise SystemExit("GNN type checker rejected the checked sketches")
checked = GNNTypeChecker().validate_gnn_files(model_dir, Path(output_dir))
# ``validate_gnn_files`` is tri-state, not boolean: ``True`` when every
# file validated, ``2`` when one or more files were invalid *or* no GNN
# file was found, ``False`` on a hard failure. ``2`` is truthy, so a
# truthiness test passes an invalid model set; only ``True`` may pass.
if checked is not True:
raise SystemExit(
"GNN type checker did not verify the checked sketches: "
f"validate_gnn_files returned {checked!r} "
"(True = every file validated, 2 = invalid files or none found, "
"False = hard failure)"
)


def verify(root: Path, pin: dict) -> dict:
Expand Down
158 changes: 158 additions & 0 deletions .github/scripts/tests/test_gnn_conformance.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,158 @@
"""Verdict tests for the pinned GNN conformance arm.

``GNNTypeChecker.validate_gnn_files`` is tri-state, not boolean: ``True`` when
every file validated, ``2`` when one or more files were invalid or no GNN file
was found, ``False`` on a hard failure. Only ``True`` may pass the arm, so the
arm must refuse ``2`` even though ``2`` is truthy — the regression these tests
pin down, because a truthiness test printed ``"result": "verified"`` for an
invalid model set.

The pinned parser is a third-party install the ``tests`` arm deliberately does
not require, so every case here stubs ``gnn`` in ``sys.modules``. The arm's own
code — the copying, the call, the verdict, the receipt — runs unchanged.
"""

from __future__ import annotations

import contextlib
import io
import json
import sys
import tempfile
import types
import unittest
from pathlib import Path
from unittest import mock

SCRIPTS = Path(__file__).resolve().parents[1]
ROOT = SCRIPTS.parents[1]
if str(SCRIPTS) not in sys.path:
sys.path.insert(0, str(SCRIPTS))

import gnn_conformance

PIN = json.loads((ROOT / gnn_conformance.PIN_PATH).read_text(encoding="utf-8"))


class _Parsed:
"""Stand-in for a parsed sketch with no parse errors."""

parse_errors: list[str] = []


def _stub_modules(verdict: object, *, version: str = "5.0.3") -> dict[str, object]:
"""Return ``sys.modules`` entries implementing the pinned parser surface."""
gnn = types.ModuleType("gnn")
gnn.__version__ = version # type: ignore[attr-defined]

parsers = types.ModuleType("gnn.parsers")
basic = types.ModuleType("gnn.parsers.basic")
basic.parse_gnn_formal = lambda path: _Parsed() # type: ignore[attr-defined]
basic.validate_gnn_syntax = lambda text, **kwargs: (True, []) # type: ignore[attr-defined]

checker_module = types.ModuleType("gnn.type_checker")

class _Checker:
"""Records the model set it was handed, then returns the verdict."""

def __init__(self) -> None:
_Checker.last_calls = self.calls = []

def validate_gnn_files(self, target_dir, output_dir, *args, **kwargs):
Path(output_dir).mkdir(parents=True, exist_ok=True)
self.calls.append(sorted(path.name for path in Path(target_dir).iterdir()))
return verdict

_Checker.last_calls = [] # type: ignore[attr-defined]
checker_module.GNNTypeChecker = _Checker # type: ignore[attr-defined]

return {
"gnn": gnn,
"gnn.parsers": parsers,
"gnn.parsers.basic": basic,
"gnn.type_checker": checker_module,
}


class TypeCheckVerdictTests(unittest.TestCase):
"""``_check_types`` may pass only on the literal ``True``."""

def setUp(self) -> None:
self._temp = tempfile.TemporaryDirectory()
self.root = Path(self._temp.name).resolve()
self.sketch = self.root / "m.gnn.md"
self.sketch.write_text("# sketch\n", encoding="utf-8")

def tearDown(self) -> None:
self._temp.cleanup()

def _run(self, verdict: object) -> None:
with mock.patch.dict(sys.modules, _stub_modules(verdict)):
gnn_conformance._check_types([self.sketch])

def test_true_passes(self) -> None:
stubs = _stub_modules(True)
with mock.patch.dict(sys.modules, stubs):
gnn_conformance._check_types([self.sketch])
calls = stubs["gnn.type_checker"].GNNTypeChecker.last_calls # type: ignore[attr-defined]
self.assertEqual(calls, [["000_m.gnn.md"]])

def test_two_is_refused_and_named(self) -> None:
with self.assertRaises(SystemExit) as raised:
self._run(2)
self.assertIn("returned 2", str(raised.exception.code))

def test_false_is_refused_and_named(self) -> None:
with self.assertRaises(SystemExit) as raised:
self._run(False)
self.assertIn("returned False", str(raised.exception.code))

def test_none_is_refused_and_named(self) -> None:
with self.assertRaises(SystemExit) as raised:
self._run(None)
self.assertIn("returned None", str(raised.exception.code))

def test_zero_is_refused_and_named(self) -> None:
with self.assertRaises(SystemExit) as raised:
self._run(0)
self.assertIn("returned 0", str(raised.exception.code))


class RunVerdictTests(unittest.TestCase):
"""The arm must not print a ``verified`` receipt for a refused verdict."""

def setUp(self) -> None:
self._temp = tempfile.TemporaryDirectory()
self.root = Path(self._temp.name).resolve()
pin_path = self.root / gnn_conformance.PIN_PATH
pin_path.parent.mkdir(parents=True)
pin_path.write_text(json.dumps(PIN), encoding="utf-8")
(self.root / "model.gnn.md").write_text("# sketch\n", encoding="utf-8")

def tearDown(self) -> None:
self._temp.cleanup()

def _run(self, verdict: object) -> tuple[int | None, str]:
stdout, stderr = io.StringIO(), io.StringIO()
code: int | None = 0
with mock.patch.dict(sys.modules, _stub_modules(verdict)):
with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr):
code = gnn_conformance.run(self.root)
return code, stdout.getvalue() + stderr.getvalue()

def test_invalid_models_fail_the_arm_without_a_receipt(self) -> None:
with self.assertRaises(SystemExit) as raised:
self._run(2)
self.assertIn("returned 2", str(raised.exception.code))

def test_verified_models_print_the_receipt(self) -> None:
code, output = self._run(True)
self.assertEqual(code, 0)
receipt = json.loads(output.strip().splitlines()[-1])
self.assertEqual(receipt["result"], "verified")
self.assertEqual(receipt["source"], "repository")
self.assertEqual(receipt["models"], 1)


if __name__ == "__main__":
unittest.main()
5 changes: 4 additions & 1 deletion docs/RUN_GUIDE.md
Original file line number Diff line number Diff line change
Expand Up @@ -97,7 +97,10 @@ schema and repository identity, installs
with `--no-deps` plus the runtime dependencies `defusedxml`, `psutil`, and
`pyyaml`, then checks that the installed `gnn.__version__` equals the tag
without its `v`. Parsing then covers every `*.gnn.md` sketch in the repository
(full parse, syntax validation, type checking). This archive carries none, so
(full parse, syntax validation, type checking). The type-check verdict passes
only when `validate_gnn_files` returns the literal `True`; its `2`
(`SUCCESS_WITH_WARNINGS`, which covers invalid files and an empty model set) is
a failure, as `.github/GNN.md` records. This archive carries none, so
the arm validates the pinned upstream exemplar
`input/gnn_files/basics/static_perception.md` at the pinned revision and prints
a JSON receipt naming `"source": "pinned-exemplar"`.
Expand Down
Loading