From b7be703f1046c0f79c3c33ac0e675d45e9dfb598 Mon Sep 17 00:00:00 2001 From: "Reference Field, Inc." Date: Wed, 2 Sep 2026 19:37:23 -0400 Subject: [PATCH 1/5] Add validator regression self-tests --- tools/test_validate_protocol.py | 97 +++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 tools/test_validate_protocol.py diff --git a/tools/test_validate_protocol.py b/tools/test_validate_protocol.py new file mode 100644 index 0000000..93f594c --- /dev/null +++ b/tools/test_validate_protocol.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Regression self-tests for tools/validate_protocol.py. + +These tests exercise the validator as a black box against temporary copies of +this repository. They intentionally cover a small number of high-value +invariants rather than duplicating the validator implementation. +""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +class ValidatorRegressionTests(unittest.TestCase): + def make_copy(self) -> Path: + temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(temp_dir.cleanup) + target = Path(temp_dir.name) / "repo" + shutil.copytree( + ROOT, + target, + ignore=shutil.ignore_patterns(".git", "__pycache__", ".pytest_cache"), + ) + return target + + def run_validator(self, root: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "tools/validate_protocol.py"], + cwd=root, + text=True, + capture_output=True, + check=False, + ) + + def assert_fails_with(self, result: subprocess.CompletedProcess[str], text: str) -> None: + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, output) + self.assertIn(text, output) + self.assertIn("RESULT: FAIL", output) + + def test_current_repository_passes(self) -> None: + root = self.make_copy() + result = self.run_validator(root) + output = result.stdout + result.stderr + self.assertEqual(result.returncode, 0, output) + self.assertIn("RESULT: PASS", output) + + def test_documented_bootloader_drift_fails(self) -> None: + root = self.make_copy() + setup_path = root / "SETUP.md" + text = setup_path.read_text(encoding="utf-8") + needle = "\n> Operational Memory:" + replacement = "\n> Altered Operational Memory:" + self.assertIn(needle, text) + setup_path.write_text(text.replace(needle, replacement, 1), encoding="utf-8") + + result = self.run_validator(root) + self.assert_fails_with(result, "documented bootloader does not match") + + def test_missing_main_push_trigger_fails(self) -> None: + root = self.make_copy() + workflow_path = root / ".github" / "workflows" / "protocol-validation.yml" + text = workflow_path.read_text(encoding="utf-8") + needle = " - main\n" + self.assertIn(needle, text) + workflow_path.write_text(text.replace(needle, "", 1), encoding="utf-8") + + result = self.run_validator(root) + self.assert_fails_with(result, "push trigger must include canonical main") + + def test_supported_plan_manifest_drift_fails(self) -> None: + root = self.make_copy() + manifest_path = root / "PROTOCOL.yaml" + text = manifest_path.read_text(encoding="utf-8") + needle = "minimum_supported_chatgpt_plan: plus" + self.assertIn(needle, text) + manifest_path.write_text( + text.replace(needle, "minimum_supported_chatgpt_plan: free", 1), + encoding="utf-8", + ) + + result = self.run_validator(root) + self.assert_fails_with( + result, + "compatibility.minimum_supported_chatgpt_plan must be 'plus'", + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2) From ad1680eee6967aa8d19d431aa848ba059eb00795 Mon Sep 17 00:00:00 2001 From: "Reference Field, Inc." Date: Wed, 2 Sep 2026 19:37:30 -0400 Subject: [PATCH 2/5] Run validator self-tests in CI --- .github/workflows/protocol-validation.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/protocol-validation.yml b/.github/workflows/protocol-validation.yml index e13d572..fffa964 100644 --- a/.github/workflows/protocol-validation.yml +++ b/.github/workflows/protocol-validation.yml @@ -26,13 +26,18 @@ jobs: - name: Install validator dependency run: python -m pip install "PyYAML==6.0.2" + - name: Run validator regression self-tests + run: python tools/test_validate_protocol.py + - name: Validate protocol structure run: python tools/validate_protocol.py # This workflow is intentionally advisory. It runs for pull requests, every # push to canonical main (including normal operational-memory writes), or when -# manually dispatched. Main-push validation is a post-write repository-integrity -# check; immediate persistence claims still depend on write readback/verification, +# manually dispatched. Validator regression self-tests run before repository +# validation so high-value failure detectors are exercised on every workflow +# run. Main-push validation is a post-write repository-integrity check; +# immediate persistence claims still depend on write readback/verification, # not on waiting for this workflow. PROTOCOL.yaml declares # blocking_required_check: false, and the recommended main-branch ruleset does # not require this status check. From aebd3e9a4e844f80b363e154aa2268a59fccb580 Mon Sep 17 00:00:00 2001 From: "Reference Field, Inc." Date: Wed, 2 Sep 2026 19:38:27 -0400 Subject: [PATCH 3/5] Require behavioral qualification before template-copy acceptance --- MIGRATIONS.md | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/MIGRATIONS.md b/MIGRATIONS.md index bd24d37..6881cd4 100644 --- a/MIGRATIONS.md +++ b/MIGRATIONS.md @@ -117,19 +117,33 @@ From that point through Gates 2-6, do not make opportunistic improvements. Any a Against the frozen candidate: +- run `tools/test_validate_protocol.py` and require all validator regression self-tests to pass; - run `tools/validate_protocol.py` and require `RESULT: PASS` or consciously adjudicate every `Watch` signal; - verify repository visibility/template status, default branch, branch set, and intended public-repository ruleset/settings; - verify no setup/probe files or unintended artifacts remain; - verify `START_HERE.md` and other declared soft budgets; - verify the public template-source repository ID resolves correctly. -Record the exact candidate SHA used. A validator result from an earlier commit does not satisfy this gate. +Record the exact candidate SHA used. A validator or self-test result from an earlier commit does not satisfy this gate. + +### Gate 2B — Independent behavioral qualification + +Before any zero-reading template-copy acceptance session, qualify the exact frozen candidate against the maintained behavioral/adversarial suite in `EVALS.md`. + +- use fresh conversation/context for the evaluated model/surface and do not expose the corresponding `Expected` section before the model acts; +- exercise every scenario applicable to the public release claims; any scenario classified as not applicable must be explicitly recorded with the reason rather than silently omitted; +- judge observed behavior after the fact and do not count ambiguous outcomes as passes; +- distinguish protocol/model behavior failures from tool outage or capability unavailability; +- record the qualifying run in `EVAL_RESULTS.md` with date, model/surface, candidate SHA, scenario coverage, pass/fail/blocked or not-applicable classifications, and material failure modes; +- same-session self-review after the expected answers have been seen may inform development but does **not** satisfy Gate 2B. + +Gate 2B passes only when there is no unresolved behavioral failure that contradicts a public release claim or a required safety/authority/persistence invariant. If qualification exposes a repository change that should be made, invalidate the candidate and enter the Gate 6 corrective loop before proceeding to Gate 3. ### Gate 3 — Zero-reading template-copy acceptance test Create a new **private** repository using GitHub **Use this template**, give it an arbitrary name, and approach it as a first-time non-expert user who has not read the repository files. -The minimum public ChatGPT support claim for this release is **ChatGPT Plus**. Free and ChatGPT Go are explicitly unsupported and are not eligible substitutes for this acceptance test. Run Gate 3 on a normal Plus account/surface, not a maintainer-only, administrator-only, internal, development, or unusually privileged environment. Record the exact plan/surface and GitHub plugin/app path actually used. +The minimum public ChatGPT support claim for this release is **ChatGPT Plus**. Free and ChatGPT Go are explicitly unsupported and are not eligible substitutes for this acceptance test. Run Gate 3 on a normal Plus account/surface, not a maintainer-only, administrator-only, internal, development, or unusually privileged environment. Before activation, prove that the user-facing prerequisite path itself works: `@GitHub` can be installed/selected in ChatGPT, authenticated to GitHub, authorized for the exact private template copy, and exposes the repository create/update/delete actions required by activation. Plugin visibility alone is not enough. If any of those prerequisites is unavailable on the tested Plus surface, Gate 3 is **BLOCKED** and the release cannot claim Plus support until the capability path is established or the public support boundary is deliberately changed. @@ -144,6 +158,8 @@ Using a fresh ChatGPT conversation and only the documented beginner **Create → 7. rename the private repository; 8. start another fresh conversation and verify the same repository ID resolves to the renamed repository, the state is recovered, and a verified write succeeds without changing the bootloader. +For the Gate 3 evidence record, capture at minimum: date/time with timezone, frozen candidate SHA and tree SHA, ChatGPT model, plan, surface, GitHub integration/plugin path actually used, derived repository ID, activation READY/BLOCKED outcome, durable-state write commit SHA, and the derived-copy validator workflow run ID/result. Keep the record focused on reproducibility; do not copy private memory content merely for instrumentation. + Failure or user confusion is release evidence. Fix the smallest root cause, return the public template to development, and, when the existing acceptance authorization remains in force, re-enter with a new candidate after corrective work. ### Gate 4 — Surface smoke tests @@ -167,17 +183,18 @@ Do not accept or reject a finding merely because it agrees or disagrees with pri ### Gate 6 — Post-fix regression gate -If Gates 2–5 identify any repository change that should be made: +If Gates 2–5, including Gate 2B, identify any repository change that should be made: 1. invalidate the current candidate and restore `protocol_status: development` as part of corrective work; 2. make and verify the accepted fixes in development; 3. if the existing acceptance authorization remains in force and the work stayed within the corrective loop, change `protocol_status` back to `acceptance_candidate` without a redundant approval prompt; otherwise obtain a new explicit entry decision; 4. freeze the resulting new canonical `main` SHA/tree; -5. rerun deterministic validation against that exact candidate; -6. rerun the portions of the zero-reading/surface tests affected by the changes; -7. rerun the independent audit as a regression/delta audit, explicitly checking accepted fixes for newly introduced failures. +5. rerun validator regression self-tests and deterministic validation against that exact candidate; +6. rerun the behavioral scenarios affected by the changes, plus any scenarios needed to establish that the new candidate still satisfies Gate 2B; +7. rerun the portions of the zero-reading/surface tests affected by the changes; +8. rerun the independent audit as a regression/delta audit, explicitly checking accepted fixes for newly introduced failures. -The candidate passes only when there are **zero unresolved blockers**, every remaining warning is consciously classified, and no new material regression is found. +The candidate passes only when there are **zero unresolved blockers**, no unresolved applicable behavioral failure contradicts the release claims/invariants, every remaining warning is consciously classified, and no new material regression is found. ### Gate 7 — Release transition @@ -187,7 +204,7 @@ Only after Gates 1–6 pass: - replace the **pre-first-release bootstrap lifecycle** in `PROTOCOL.yaml`, `AGENTS.md`, and `tools/validate_protocol.py` with a post-release lifecycle that can represent the last/current released protocol and a future development/acceptance target without reverting a released protocol ambiguously to `unreleased`; define and validate that transition before tagging the first release; - execute the controlled one-time history cleanup if still desired; - restore and verify the intended public-repository protection/settings; -- rerun deterministic validation against the final release tree; +- rerun validator regression self-tests and deterministic validation against the final release tree; - tag/publish the release; - create one final private copy using **Use this template** and repeat the activation smoke test against what GitHub actually ships. @@ -227,8 +244,8 @@ When the first public release is actually cut: 5. remove or replace pre-release-only wording explicitly in **`README.md` (Development status), `MIGRATIONS.md` (Current pre-release rule), and `AGENTS.md` (the public-template unreleased-version instruction)**, then sweep for any other remaining `unreleased` wording that would confuse a working copy created from the release; 6. verify that setup/activation produces and uses the repository-ID bootloader and that a working-repository rename does not require bootloader changes; 7. verify that `template_source.repository_id` resolves the public upstream and update discovery does not depend on its current owner/name; -8. run deterministic structural validation and the semantic repository health check; -9. run the behavioral/adversarial eval set relevant to routing, persistence, update discovery, maintenance, failure handling, plan/plugin readiness, and release lifecycle; +8. run validator regression self-tests, deterministic structural validation, and the semantic repository health check; +9. confirm the Gate 2B behavioral evidence remains applicable to the release tree and rerun any behavioral/adversarial scenarios affected by the prescribed Gate 7 lifecycle/release transition; 10. verify README/SETUP/START_HERE/OPERATIONS/SECURITY/MIGRATIONS all describe the same release behavior; 11. perform the controlled one-time history cleanup above if a clean release baseline is still desired; 12. only then treat the public template as the migration source for user-created copies. From cda77e43b05ce093f251875da1aa4d627056d530 Mon Sep 17 00:00:00 2001 From: "Reference Field, Inc." Date: Wed, 2 Sep 2026 19:39:22 -0400 Subject: [PATCH 4/5] Keep behavioral evidence outside frozen candidate tree --- MIGRATIONS.md | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/MIGRATIONS.md b/MIGRATIONS.md index 6881cd4..239d036 100644 --- a/MIGRATIONS.md +++ b/MIGRATIONS.md @@ -134,7 +134,8 @@ Before any zero-reading template-copy acceptance session, qualify the exact froz - exercise every scenario applicable to the public release claims; any scenario classified as not applicable must be explicitly recorded with the reason rather than silently omitted; - judge observed behavior after the fact and do not count ambiguous outcomes as passes; - distinguish protocol/model behavior failures from tool outage or capability unavailability; -- record the qualifying run in `EVAL_RESULTS.md` with date, model/surface, candidate SHA, scenario coverage, pass/fail/blocked or not-applicable classifications, and material failure modes; +- record the qualification in an acceptance evidence record **outside the frozen candidate tree**, including date, model/surface, candidate SHA, scenario coverage, pass/fail/blocked or not-applicable classifications, and material failure modes; +- do **not** update `EVAL_RESULTS.md` while Gates 1–6 freeze is active, because that would mutate and invalidate the candidate; publish the compact qualifying summary there during the prescribed Gate 7 release transition; - same-session self-review after the expected answers have been seen may inform development but does **not** satisfy Gate 2B. Gate 2B passes only when there is no unresolved behavioral failure that contradicts a public release claim or a required safety/authority/persistence invariant. If qualification exposes a repository change that should be made, invalidate the candidate and enter the Gate 6 corrective loop before proceeding to Gate 3. @@ -158,7 +159,7 @@ Using a fresh ChatGPT conversation and only the documented beginner **Create → 7. rename the private repository; 8. start another fresh conversation and verify the same repository ID resolves to the renamed repository, the state is recovered, and a verified write succeeds without changing the bootloader. -For the Gate 3 evidence record, capture at minimum: date/time with timezone, frozen candidate SHA and tree SHA, ChatGPT model, plan, surface, GitHub integration/plugin path actually used, derived repository ID, activation READY/BLOCKED outcome, durable-state write commit SHA, and the derived-copy validator workflow run ID/result. Keep the record focused on reproducibility; do not copy private memory content merely for instrumentation. +For the Gate 3 acceptance evidence record, capture at minimum: date/time with timezone, frozen candidate SHA and tree SHA, ChatGPT model, plan, surface, GitHub integration/plugin path actually used, derived repository ID, activation READY/BLOCKED outcome, durable-state write commit SHA, and the derived-copy validator workflow run ID/result. Keep the record outside the frozen candidate tree during Gates 1–6 and focused on reproducibility; do not copy private memory content merely for instrumentation. Failure or user confusion is release evidence. Fix the smallest root cause, return the public template to development, and, when the existing acceptance authorization remains in force, re-enter with a new candidate after corrective work. @@ -202,13 +203,14 @@ Only after Gates 1–6 pass: - assign the first real protocol release identifier and set `protocol_status: released`; - replace the **pre-first-release bootstrap lifecycle** in `PROTOCOL.yaml`, `AGENTS.md`, and `tools/validate_protocol.py` with a post-release lifecycle that can represent the last/current released protocol and a future development/acceptance target without reverting a released protocol ambiguously to `unreleased`; define and validate that transition before tagging the first release; +- publish the compact Gate 2B qualifying behavioral summary from the external acceptance evidence record into `EVAL_RESULTS.md`, rerunning and updating any scenarios materially affected by the prescribed release/lifecycle transition; - execute the controlled one-time history cleanup if still desired; - restore and verify the intended public-repository protection/settings; - rerun validator regression self-tests and deterministic validation against the final release tree; - tag/publish the release; - create one final private copy using **Use this template** and repeat the activation smoke test against what GitHub actually ships. -The prescribed release/version/lifecycle/history changes in Gate 7 are the release transition after the frozen candidate has passed; they are not an acceptance-candidate defect that sends the repository back through Gate 6. Any unrelated or opportunistic change discovered during Gate 7 still invalidates the release transition and must be handled through the normal corrective path. +The prescribed release/version/lifecycle/evidence/history changes in Gate 7 are the release transition after the frozen candidate has passed; they are not an acceptance-candidate defect that sends the repository back through Gate 6. Any unrelated or opportunistic change discovered during Gate 7 still invalidates the release transition and must be handled through the normal corrective path. That final private-copy activation is the release artifact check. A successful test against the source repository alone is not sufficient. @@ -245,7 +247,7 @@ When the first public release is actually cut: 6. verify that setup/activation produces and uses the repository-ID bootloader and that a working-repository rename does not require bootloader changes; 7. verify that `template_source.repository_id` resolves the public upstream and update discovery does not depend on its current owner/name; 8. run validator regression self-tests, deterministic structural validation, and the semantic repository health check; -9. confirm the Gate 2B behavioral evidence remains applicable to the release tree and rerun any behavioral/adversarial scenarios affected by the prescribed Gate 7 lifecycle/release transition; +9. publish the Gate 2B behavioral summary into `EVAL_RESULTS.md` and confirm the evidence remains applicable to the release tree, rerunning any behavioral/adversarial scenarios affected by the prescribed Gate 7 lifecycle/release transition; 10. verify README/SETUP/START_HERE/OPERATIONS/SECURITY/MIGRATIONS all describe the same release behavior; 11. perform the controlled one-time history cleanup above if a clean release baseline is still desired; 12. only then treat the public template as the migration source for user-created copies. From 9b5c73dae34a0757446390d303b6e51579deb4b2 Mon Sep 17 00:00:00 2001 From: "Reference Field, Inc." Date: Wed, 2 Sep 2026 19:39:39 -0400 Subject: [PATCH 5/5] Document frozen-candidate behavioral evidence handling --- EVAL_RESULTS.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/EVAL_RESULTS.md b/EVAL_RESULTS.md index 8cb854c..ea4dbac 100644 --- a/EVAL_RESULTS.md +++ b/EVAL_RESULTS.md @@ -19,6 +19,12 @@ A behavioral run should be recorded here only when the evaluation is meaningfull A same-session self-review in which the model has already read the expected answers is useful for development but is **not** a publishable independent behavioral result. +## Frozen-candidate handling + +During the Gates 1–6 acceptance freeze, qualifying behavioral evidence is recorded outside the frozen candidate tree so the act of recording results does not itself mutate and invalidate the candidate. `MIGRATIONS.md` Gate 2B defines the required qualification and evidence fields. + +After the frozen candidate passes Gates 1–6, Gate 7 publishes the compact qualifying summary into this file as part of the prescribed release transition. If Gate 7's lifecycle/release changes materially affect an evaluated behavior, rerun the affected scenarios and publish the updated result rather than carrying forward stale evidence. + ## Published runs _None yet._