diff --git a/.github/workflows/protocol-validation.yml b/.github/workflows/protocol-validation.yml index e13d572..fffa964 100644 --- a/.github/workflows/protocol-validation.yml +++ b/.github/workflows/protocol-validation.yml @@ -26,13 +26,18 @@ jobs: - name: Install validator dependency run: python -m pip install "PyYAML==6.0.2" + - name: Run validator regression self-tests + run: python tools/test_validate_protocol.py + - name: Validate protocol structure run: python tools/validate_protocol.py # This workflow is intentionally advisory. It runs for pull requests, every # push to canonical main (including normal operational-memory writes), or when -# manually dispatched. Main-push validation is a post-write repository-integrity -# check; immediate persistence claims still depend on write readback/verification, +# manually dispatched. Validator regression self-tests run before repository +# validation so high-value failure detectors are exercised on every workflow +# run. Main-push validation is a post-write repository-integrity check; +# immediate persistence claims still depend on write readback/verification, # not on waiting for this workflow. PROTOCOL.yaml declares # blocking_required_check: false, and the recommended main-branch ruleset does # not require this status check. diff --git a/EVAL_RESULTS.md b/EVAL_RESULTS.md index 8cb854c..ea4dbac 100644 --- a/EVAL_RESULTS.md +++ b/EVAL_RESULTS.md @@ -19,6 +19,12 @@ A behavioral run should be recorded here only when the evaluation is meaningfull A same-session self-review in which the model has already read the expected answers is useful for development but is **not** a publishable independent behavioral result. +## Frozen-candidate handling + +During the Gates 1–6 acceptance freeze, qualifying behavioral evidence is recorded outside the frozen candidate tree so the act of recording results does not itself mutate and invalidate the candidate. `MIGRATIONS.md` Gate 2B defines the required qualification and evidence fields. + +After the frozen candidate passes Gates 1–6, Gate 7 publishes the compact qualifying summary into this file as part of the prescribed release transition. If Gate 7's lifecycle/release changes materially affect an evaluated behavior, rerun the affected scenarios and publish the updated result rather than carrying forward stale evidence. + ## Published runs _None yet._ diff --git a/MIGRATIONS.md b/MIGRATIONS.md index bd24d37..239d036 100644 --- a/MIGRATIONS.md +++ b/MIGRATIONS.md @@ -117,19 +117,34 @@ From that point through Gates 2-6, do not make opportunistic improvements. Any a Against the frozen candidate: +- run `tools/test_validate_protocol.py` and require all validator regression self-tests to pass; - run `tools/validate_protocol.py` and require `RESULT: PASS` or consciously adjudicate every `Watch` signal; - verify repository visibility/template status, default branch, branch set, and intended public-repository ruleset/settings; - verify no setup/probe files or unintended artifacts remain; - verify `START_HERE.md` and other declared soft budgets; - verify the public template-source repository ID resolves correctly. -Record the exact candidate SHA used. A validator result from an earlier commit does not satisfy this gate. +Record the exact candidate SHA used. A validator or self-test result from an earlier commit does not satisfy this gate. + +### Gate 2B — Independent behavioral qualification + +Before any zero-reading template-copy acceptance session, qualify the exact frozen candidate against the maintained behavioral/adversarial suite in `EVALS.md`. + +- use fresh conversation/context for the evaluated model/surface and do not expose the corresponding `Expected` section before the model acts; +- exercise every scenario applicable to the public release claims; any scenario classified as not applicable must be explicitly recorded with the reason rather than silently omitted; +- judge observed behavior after the fact and do not count ambiguous outcomes as passes; +- distinguish protocol/model behavior failures from tool outage or capability unavailability; +- record the qualification in an acceptance evidence record **outside the frozen candidate tree**, including date, model/surface, candidate SHA, scenario coverage, pass/fail/blocked or not-applicable classifications, and material failure modes; +- do **not** update `EVAL_RESULTS.md` while Gates 1–6 freeze is active, because that would mutate and invalidate the candidate; publish the compact qualifying summary there during the prescribed Gate 7 release transition; +- same-session self-review after the expected answers have been seen may inform development but does **not** satisfy Gate 2B. + +Gate 2B passes only when there is no unresolved behavioral failure that contradicts a public release claim or a required safety/authority/persistence invariant. If qualification exposes a repository change that should be made, invalidate the candidate and enter the Gate 6 corrective loop before proceeding to Gate 3. ### Gate 3 — Zero-reading template-copy acceptance test Create a new **private** repository using GitHub **Use this template**, give it an arbitrary name, and approach it as a first-time non-expert user who has not read the repository files. -The minimum public ChatGPT support claim for this release is **ChatGPT Plus**. Free and ChatGPT Go are explicitly unsupported and are not eligible substitutes for this acceptance test. Run Gate 3 on a normal Plus account/surface, not a maintainer-only, administrator-only, internal, development, or unusually privileged environment. Record the exact plan/surface and GitHub plugin/app path actually used. +The minimum public ChatGPT support claim for this release is **ChatGPT Plus**. Free and ChatGPT Go are explicitly unsupported and are not eligible substitutes for this acceptance test. Run Gate 3 on a normal Plus account/surface, not a maintainer-only, administrator-only, internal, development, or unusually privileged environment. Before activation, prove that the user-facing prerequisite path itself works: `@GitHub` can be installed/selected in ChatGPT, authenticated to GitHub, authorized for the exact private template copy, and exposes the repository create/update/delete actions required by activation. Plugin visibility alone is not enough. If any of those prerequisites is unavailable on the tested Plus surface, Gate 3 is **BLOCKED** and the release cannot claim Plus support until the capability path is established or the public support boundary is deliberately changed. @@ -144,6 +159,8 @@ Using a fresh ChatGPT conversation and only the documented beginner **Create → 7. rename the private repository; 8. start another fresh conversation and verify the same repository ID resolves to the renamed repository, the state is recovered, and a verified write succeeds without changing the bootloader. +For the Gate 3 acceptance evidence record, capture at minimum: date/time with timezone, frozen candidate SHA and tree SHA, ChatGPT model, plan, surface, GitHub integration/plugin path actually used, derived repository ID, activation READY/BLOCKED outcome, durable-state write commit SHA, and the derived-copy validator workflow run ID/result. Keep the record outside the frozen candidate tree during Gates 1–6 and focused on reproducibility; do not copy private memory content merely for instrumentation. + Failure or user confusion is release evidence. Fix the smallest root cause, return the public template to development, and, when the existing acceptance authorization remains in force, re-enter with a new candidate after corrective work. ### Gate 4 — Surface smoke tests @@ -167,17 +184,18 @@ Do not accept or reject a finding merely because it agrees or disagrees with pri ### Gate 6 — Post-fix regression gate -If Gates 2–5 identify any repository change that should be made: +If Gates 2–5, including Gate 2B, identify any repository change that should be made: 1. invalidate the current candidate and restore `protocol_status: development` as part of corrective work; 2. make and verify the accepted fixes in development; 3. if the existing acceptance authorization remains in force and the work stayed within the corrective loop, change `protocol_status` back to `acceptance_candidate` without a redundant approval prompt; otherwise obtain a new explicit entry decision; 4. freeze the resulting new canonical `main` SHA/tree; -5. rerun deterministic validation against that exact candidate; -6. rerun the portions of the zero-reading/surface tests affected by the changes; -7. rerun the independent audit as a regression/delta audit, explicitly checking accepted fixes for newly introduced failures. +5. rerun validator regression self-tests and deterministic validation against that exact candidate; +6. rerun the behavioral scenarios affected by the changes, plus any scenarios needed to establish that the new candidate still satisfies Gate 2B; +7. rerun the portions of the zero-reading/surface tests affected by the changes; +8. rerun the independent audit as a regression/delta audit, explicitly checking accepted fixes for newly introduced failures. -The candidate passes only when there are **zero unresolved blockers**, every remaining warning is consciously classified, and no new material regression is found. +The candidate passes only when there are **zero unresolved blockers**, no unresolved applicable behavioral failure contradicts the release claims/invariants, every remaining warning is consciously classified, and no new material regression is found. ### Gate 7 — Release transition @@ -185,13 +203,14 @@ Only after Gates 1–6 pass: - assign the first real protocol release identifier and set `protocol_status: released`; - replace the **pre-first-release bootstrap lifecycle** in `PROTOCOL.yaml`, `AGENTS.md`, and `tools/validate_protocol.py` with a post-release lifecycle that can represent the last/current released protocol and a future development/acceptance target without reverting a released protocol ambiguously to `unreleased`; define and validate that transition before tagging the first release; +- publish the compact Gate 2B qualifying behavioral summary from the external acceptance evidence record into `EVAL_RESULTS.md`, rerunning and updating any scenarios materially affected by the prescribed release/lifecycle transition; - execute the controlled one-time history cleanup if still desired; - restore and verify the intended public-repository protection/settings; -- rerun deterministic validation against the final release tree; +- rerun validator regression self-tests and deterministic validation against the final release tree; - tag/publish the release; - create one final private copy using **Use this template** and repeat the activation smoke test against what GitHub actually ships. -The prescribed release/version/lifecycle/history changes in Gate 7 are the release transition after the frozen candidate has passed; they are not an acceptance-candidate defect that sends the repository back through Gate 6. Any unrelated or opportunistic change discovered during Gate 7 still invalidates the release transition and must be handled through the normal corrective path. +The prescribed release/version/lifecycle/evidence/history changes in Gate 7 are the release transition after the frozen candidate has passed; they are not an acceptance-candidate defect that sends the repository back through Gate 6. Any unrelated or opportunistic change discovered during Gate 7 still invalidates the release transition and must be handled through the normal corrective path. That final private-copy activation is the release artifact check. A successful test against the source repository alone is not sufficient. @@ -227,8 +246,8 @@ When the first public release is actually cut: 5. remove or replace pre-release-only wording explicitly in **`README.md` (Development status), `MIGRATIONS.md` (Current pre-release rule), and `AGENTS.md` (the public-template unreleased-version instruction)**, then sweep for any other remaining `unreleased` wording that would confuse a working copy created from the release; 6. verify that setup/activation produces and uses the repository-ID bootloader and that a working-repository rename does not require bootloader changes; 7. verify that `template_source.repository_id` resolves the public upstream and update discovery does not depend on its current owner/name; -8. run deterministic structural validation and the semantic repository health check; -9. run the behavioral/adversarial eval set relevant to routing, persistence, update discovery, maintenance, failure handling, plan/plugin readiness, and release lifecycle; +8. run validator regression self-tests, deterministic structural validation, and the semantic repository health check; +9. publish the Gate 2B behavioral summary into `EVAL_RESULTS.md` and confirm the evidence remains applicable to the release tree, rerunning any behavioral/adversarial scenarios affected by the prescribed Gate 7 lifecycle/release transition; 10. verify README/SETUP/START_HERE/OPERATIONS/SECURITY/MIGRATIONS all describe the same release behavior; 11. perform the controlled one-time history cleanup above if a clean release baseline is still desired; 12. only then treat the public template as the migration source for user-created copies. diff --git a/tools/test_validate_protocol.py b/tools/test_validate_protocol.py new file mode 100644 index 0000000..93f594c --- /dev/null +++ b/tools/test_validate_protocol.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""Regression self-tests for tools/validate_protocol.py. + +These tests exercise the validator as a black box against temporary copies of +this repository. They intentionally cover a small number of high-value +invariants rather than duplicating the validator implementation. +""" + +from __future__ import annotations + +import shutil +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + + +class ValidatorRegressionTests(unittest.TestCase): + def make_copy(self) -> Path: + temp_dir = tempfile.TemporaryDirectory() + self.addCleanup(temp_dir.cleanup) + target = Path(temp_dir.name) / "repo" + shutil.copytree( + ROOT, + target, + ignore=shutil.ignore_patterns(".git", "__pycache__", ".pytest_cache"), + ) + return target + + def run_validator(self, root: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "tools/validate_protocol.py"], + cwd=root, + text=True, + capture_output=True, + check=False, + ) + + def assert_fails_with(self, result: subprocess.CompletedProcess[str], text: str) -> None: + output = result.stdout + result.stderr + self.assertNotEqual(result.returncode, 0, output) + self.assertIn(text, output) + self.assertIn("RESULT: FAIL", output) + + def test_current_repository_passes(self) -> None: + root = self.make_copy() + result = self.run_validator(root) + output = result.stdout + result.stderr + self.assertEqual(result.returncode, 0, output) + self.assertIn("RESULT: PASS", output) + + def test_documented_bootloader_drift_fails(self) -> None: + root = self.make_copy() + setup_path = root / "SETUP.md" + text = setup_path.read_text(encoding="utf-8") + needle = "\n> Operational Memory:" + replacement = "\n> Altered Operational Memory:" + self.assertIn(needle, text) + setup_path.write_text(text.replace(needle, replacement, 1), encoding="utf-8") + + result = self.run_validator(root) + self.assert_fails_with(result, "documented bootloader does not match") + + def test_missing_main_push_trigger_fails(self) -> None: + root = self.make_copy() + workflow_path = root / ".github" / "workflows" / "protocol-validation.yml" + text = workflow_path.read_text(encoding="utf-8") + needle = " - main\n" + self.assertIn(needle, text) + workflow_path.write_text(text.replace(needle, "", 1), encoding="utf-8") + + result = self.run_validator(root) + self.assert_fails_with(result, "push trigger must include canonical main") + + def test_supported_plan_manifest_drift_fails(self) -> None: + root = self.make_copy() + manifest_path = root / "PROTOCOL.yaml" + text = manifest_path.read_text(encoding="utf-8") + needle = "minimum_supported_chatgpt_plan: plus" + self.assertIn(needle, text) + manifest_path.write_text( + text.replace(needle, "minimum_supported_chatgpt_plan: free", 1), + encoding="utf-8", + ) + + result = self.run_validator(root) + self.assert_fails_with( + result, + "compatibility.minimum_supported_chatgpt_plan must be 'plus'", + ) + + +if __name__ == "__main__": + unittest.main(verbosity=2)