From 2846b655b47c553cf413ac7e5bcd1e7b82f62ded Mon Sep 17 00:00:00 2001 From: James Nightingale Date: Mon, 24 Aug 2026 21:23:34 +0000 Subject: [PATCH] fix: witness map missed 5 tested repos and split 2 more across keys MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow-up to #267/#268, which fixed the PyAutoNerves half of the Refactor Agent's witness map. This is the audit that prompt asked for: Nerves was not a one-off. `test_witness` had no row for PyAutoCTI or for five organs that all have real suites, so `behaviour_preservation` reported them `unwitnessed` and the conductor advised "strengthen tests first" for repos that are already tested. Two more repos carried the same two-keys-per-repo defect #268 fixed for Nerves: `@PyAutoCTI`/`@autocti` and `@PyAutoReduce`/`@autoreduce` normalised to different keys with no alias joining them, and the Reduce row was keyed on the prefixed spelling — so the bare package spelling, the one prompts actually use, missed the witness entirely. Every row was verified against the repository itself (tree read of PyAutoLabs/@main), not inferred from the naming convention: test_autocti and test_autoreduce exist; Brain, Heart, Hands, Memory and Mind each keep a plain `tests/`. PyAutoGut has no suite and is deliberately left out with a comment, so the conductor's advice is correct there rather than noise. The keying rule is now stated where the map lives: the canonical key is the bare package name where the repo ships one, the repo name where it does not. Three guards, all deriving from the body map so they hold for an adopting fork and name no instance fact: - every library/organ needs a witness row or a reasoned WITNESS_EXEMPT entry, and an exemption naming a repo the body map dropped is itself a failure; - a row's key must be what the repo it names normalises to — the invariant `autoconf: PyAutoConf/test_autoconf` violated while looking plausible; - where a row's value shows the repo ships a package, both spellings must reach the same key. Reverting any of the three fixes above now fails one or more of these; the pre-existing mis-keyed Reduce row fails all three. Closes #269 Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01MPhEXMoPYa7H1sNHWR2GNW --- config/policy.yaml | 25 +++++++-- tests/test_policy_seams.py | 109 +++++++++++++++++++++++++++---------- 2 files changed, 102 insertions(+), 32 deletions(-) diff --git a/config/policy.yaml b/config/policy.yaml index eb3dbfea..05e69a1a 100644 --- a/config/policy.yaml +++ b/config/policy.yaml @@ -14,6 +14,9 @@ repo_aliases: pyautogalaxy: autogalaxy pyautolens: autolens pyautonerves: autonerves + pyautocti: autocti + pyautoreduce: autoreduce + autohands: pyautohands # organs have no package, so the repo name is canonical autoconf: autonerves # back-compat: the Nerves package was renamed autoconf → autonerves pyautoconf: autonerves # back-compat: the Nerves repo was renamed PyAutoConf → PyAutoNerves pyautobuild: pyautohands # back-compat: the Hands repo was renamed PyAutoBuild → PyAutoHands @@ -71,15 +74,29 @@ target_default_wiki: autoarray: methods autonerves: methods # the PyAutoConf/autoconf spellings alias here -# Refactor conductor: normalised repo -> the test dir that witnesses -# behaviour preservation. +# Refactor conductor: normalised repo -> the test dir that witnesses behaviour +# preservation. Keyed by the canonical key repo_aliases resolves to: the bare +# package name where the repo ships one, the repo name where it does not. +# Every library and organ in the body map needs a row here or an entry in the +# test's WITNESS_EXEMPT — a missing row makes the conductor advise "strengthen +# tests first" for a repo that is already tested. test_witness: - autonerves: PyAutoNerves/test_autonerves + # libraries (package name is canonical) autofit: PyAutoFit/test_autofit autoarray: PyAutoArray/test_autoarray autogalaxy: PyAutoGalaxy/test_autogalaxy autolens: PyAutoLens/test_autolens - pyautoreduce: PyAutoReduce/test_autoreduce + autocti: PyAutoCTI/test_autocti + autoreduce: PyAutoReduce/test_autoreduce + # organs (repo name is canonical; Nerves ships a package, so it keys bare) + autonerves: PyAutoNerves/test_autonerves + pyautobrain: PyAutoBrain/tests + pyautoheart: PyAutoHeart/tests + pyautohands: PyAutoHands/tests + pyautomemory: PyAutoMemory/tests + pyautomind: PyAutoMind/tests + # PyAutoGut has no test suite — deliberately absent, so the conductor's + # "strengthen tests first" advice is correct there rather than noise. # Release policy (NOT identity: the released set is a deliberate subset of # the body map's libraries — e.g. PyAutoReduce is not in the nightly matrix). diff --git a/tests/test_policy_seams.py b/tests/test_policy_seams.py index 63bd5bc2..fde1e6ca 100644 --- a/tests/test_policy_seams.py +++ b/tests/test_policy_seams.py @@ -97,37 +97,90 @@ def test_refactor_test_witness_loads(): assert _sizing.normalise_repo(mention) in _refactor.TEST_WITNESS, mention -def test_release_policy_loads(): - sys.path.insert(0, str(BRAIN_HOME / "agents" / "conductors" / "release")) - import activity_gate +# Repos allowed to have no witness row, with the reason. An entry here is a +# statement that the repo has no test suite — not a licence to skip one. +WITNESS_EXEMPT = { + "PyAutoGut": "no test suite (bin/ + docs only, verified at PyAutoBrain#269)", +} + + +def test_every_library_and_organ_is_witnessed(): + """PyAutoBrain#269: a tested repo missing from the map reads as untested. + + ``behaviour_preservation`` reports any repo absent from ``test_witness`` as + ``unwitnessed`` and advises "strengthen tests first". For a repo that has a + suite that advice is wrong, and it is wrong silently — which is how one + library and five organs went unnoticed until the #267 rename work looked. + Pin coverage to the body map so a new library or organ cannot be added + without either a witness row or a reasoned exemption. + """ + sys.path.insert(0, str(BRAIN_HOME / "agents" / "conductors" / "refactor")) + import _refactor + + cats = _sizing._body_map_categories() + missing = [ + name for name, cat in cats.items() + if cat in ("library", "organ") + and name not in WITNESS_EXEMPT + and _sizing.normalise_repo(name) not in _refactor.TEST_WITNESS + ] + assert not missing, ( + "library/organ repos with no test_witness row — add the row, or add the " + f"repo to WITNESS_EXEMPT with its reason: {missing}" + ) + - assert "PyAutoLens" in activity_gate.RELEASE_RELEVANT_REPOS - assert "PyAutoMind" not in activity_gate.RELEASE_RELEVANT_REPOS +def test_witness_exemptions_are_still_real_repos(): + """An exemption for a repo that left the body map is stale, not a waiver.""" + cats = _sizing._body_map_categories() + stale = [name for name in WITNESS_EXEMPT if name not in cats] + assert not stale, f"WITNESS_EXEMPT names repos not in the body map: {stale}" -def test_release_relevant_repos_all_exist_in_the_body_map(): - """The drift guard PyAutoBrain#267 was missing. +def test_every_witness_key_is_what_its_repo_normalises_to(): + """A row keyed on something the normaliser never produces is a dead row. - ``nightly.sh`` fetches ``repos//$repo/commits`` for every name in - this list verbatim. A repo renamed in the body map but not here leaves the - gate polling a name that only a GitHub rename redirect could answer — which - is how ``PyAutoConf`` survived the Nerves rename. Pin the set to identity. + That is exactly what ``autoconf: PyAutoConf/test_autoconf`` had become + (PyAutoBrain#267): reachable only through a stale alias, pointing at a repo + path that no longer existed, and invisible because nothing checked the key + against the repo it names. Derive the expectation from the row's own value + so this holds for any body map, not just ours. """ - sys.path.insert(0, str(BRAIN_HOME / "agents" / "conductors" / "release")) - import activity_gate - - known = set(_sizing._body_map_categories()) - unknown = [r for r in activity_gate.RELEASE_RELEVANT_REPOS if r not in known] - assert not unknown, f"not in PyAutoMind/repos.yaml: {unknown}" - - -def test_nightly_tag_repo_resolves(): - here = BRAIN_HOME / "agents" / "conductors" / "release" - out = subprocess.run( - [sys.executable, "-c", - "import yaml, pathlib; print(yaml.safe_load((pathlib.Path(" - f"'{here}').parents[2] / 'config' / 'policy.yaml').read_text())" - "['release']['tag_repo'])"], - capture_output=True, text=True, + sys.path.insert(0, str(BRAIN_HOME / "agents" / "conductors" / "refactor")) + import _refactor + + wrong = { + key: witness + for key, witness in _refactor.TEST_WITNESS.items() + if _sizing.normalise_repo(witness.split("/")[0]) != key + } + assert not wrong, ( + "test_witness rows whose key is not what their repo normalises to — " + f"the conductor can never look them up: {wrong}" + ) + + +def test_witness_repos_resolve_from_the_package_spelling_too(): + """Prompts name a library by its package, not by its repo. + + Where a witness row's repo ships a package — inferable from the row itself, + since a ``/test_`` value names one and a ``/tests`` + value does not — both spellings must reach the same key, or which one the + prompt happened to use silently decides whether the witness resolves + (PyAutoBrain#268, #269). + """ + sys.path.insert(0, str(BRAIN_HOME / "agents" / "conductors" / "refactor")) + import _refactor + + split = {} + for key, witness in _refactor.TEST_WITNESS.items(): + repo, _, test_dir = witness.partition("/") + if not test_dir.startswith("test_"): + continue # organ suite in a plain `tests/` dir — no package spelling + package = test_dir[len("test_"):] + if _sizing.normalise_repo(package) != key: + split[repo] = (package, _sizing.normalise_repo(package), key) + assert not split, ( + "package spelling normalises to a different key than the repo spelling " + f"— add a repo_aliases entry joining them: {split}" ) - assert out.returncode == 0 and "/" in out.stdout