Harden the codebase: types, module layers and checks catch wrong code

.github/workflows/ci.yml

@@ -0,0 +1,45 @@
+# This repository's own CI. Deterministic checks only: no model calls, no secrets.
+# (ci/github/*.yml are templates for the repositories that use shall.)
+name: ci
+
+on:
+  pull_request:
+  push:
+    branches: [main]
+
+jobs:
+  tooling:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with: { fetch-depth: 0 }
+      - uses: astral-sh/setup-uv@v6
+
+      - name: Unit and boundary tests
+        run: make test
+
+      - name: Corpus lint, index is current, types (pyright), module layers (import-linter)
+        run: make check
+
+      - name: Published statement ids are stable
+        if: github.event_name == 'pull_request'
+        run: |
+          if git show "origin/${{ github.base_ref }}:corpus/index/index.json" > /tmp/base-index.json 2>/dev/null; then
+            plugins/shall/bin/shall --config shall.yaml lint --quiet --against /tmp/base-index.json
+          fi
+
+      - name: This repo's own rules (.shall/, prefix JTOOL)
+        run: make self-check
+
+  plugin:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with: { node-version: 22 }
+
+      - name: Install the claude CLI
+        run: npm install -g @anthropic-ai/claude-code@2.1.296   # pinned: a new CLI must not turn CI red
+
+      - name: Plugin manifest and hook tests
+        run: make test-plugin

.shall/index/index.json

@@ -1,5 +1,5 @@
 {
-  "corpus_hash": "f96990cddff7f113",
+  "corpus_hash": "107b6b4976368374",
   "counts": {
     "domains": 3,
     "rfcs": 4,
@@ -194,7 +194,7 @@
       "level": "MUST",
       "not_applies_when": "The content does not call Jev and does not use Jev answers.",
       "rfc": "JTOOL-0001",
-      "source": ".shall/rfcs/JTOOL-0001-jev-usage.md#L32",
+      "source": ".shall/rfcs/JTOOL-0001-jev-usage.md#L40",
       "status": "enforced",
       "text": "Code MUST call Jev only through `shall.jev.Jev` (which caches answers, packs requests under the token budget and counts usage) and MUST NOT create a `TypeSafeClient` or `AsyncTypeSafeClient` anywhere else.",
       "title": "One client, one wrapper"
@@ -217,7 +217,7 @@
       "level": "MUST",
       "not_applies_when": "The content does not call Jev and does not use Jev answers.",
       "rfc": "JTOOL-0001",
-      "source": ".shall/rfcs/JTOOL-0001-jev-usage.md#L40",
+      "source": ".shall/rfcs/JTOOL-0001-jev-usage.md#L48",
       "status": "enforced",
       "text": "Jev state MUST contain only the text the questions judge; code MUST NOT put a whole corpus or list into the state and refer to one item by index (such as `existing[i]`) in the question text.",
       "title": "Small state, no indirection"
@@ -240,7 +240,7 @@
       "level": "MUST",
       "not_applies_when": "The content does not call Jev and does not use Jev answers.",
       "rfc": "JTOOL-0001",
-      "source": ".shall/rfcs/JTOOL-0001-jev-usage.md#L48",
+      "source": ".shall/rfcs/JTOOL-0001-jev-usage.md#L56",
       "status": "enforced",
       "text": "A pass/fail decision based on Jev MUST be gated on Noul probabilities or on the summed probability of all failing Choice options, not on the probability of the single top Choice option.",
       "title": "Absolute gates"
@@ -263,7 +263,7 @@
       "level": "MUST",
       "not_applies_when": "The content does not call Jev and does not use Jev answers.",
       "rfc": "JTOOL-0001",
-      "source": ".shall/rfcs/JTOOL-0001-jev-usage.md#L56",
+      "source": ".shall/rfcs/JTOOL-0001-jev-usage.md#L64",
       "status": "enforced",
       "text": "Model identifiers in configuration and code MUST be pinned versions, not moving aliases such as `jev-latest`, `jev-preview`, `sonnet` or `opus`.",
       "title": "Pinned models"
@@ -286,7 +286,7 @@
       "level": "MUST",
       "not_applies_when": "No agent runs, no agent output is used, and nothing is written to an external system.",
       "rfc": "JTOOL-0002",
-      "source": ".shall/rfcs/JTOOL-0002-agent-boundaries.md#L26",
+      "source": ".shall/rfcs/JTOOL-0002-agent-boundaries.md#L36",
       "status": "enforced",
       "text": "The review agent invoked by the pipeline MUST run with no tools (`--tools \"\"`), so that it can only return text.",
       "title": "Agents run without tools in the pipeline"
@@ -309,7 +309,7 @@
       "level": "MUST",
       "not_applies_when": "No agent runs, no agent output is used, and nothing is written to an external system.",
       "rfc": "JTOOL-0002",
-      "source": ".shall/rfcs/JTOOL-0002-agent-boundaries.md#L33",
+      "source": ".shall/rfcs/JTOOL-0002-agent-boundaries.md#L43",
       "status": "enforced",
       "text": "Findings returned by an agent MUST pass `validate_findings` (statement selected for the chunk, line or quote present in the artifact) before they reach a report or a comment.",
       "title": "Agent output is validated before use"
@@ -332,7 +332,7 @@
       "level": "MUST",
       "not_applies_when": "No agent runs, no agent output is used, and nothing is written to an external system.",
       "rfc": "JTOOL-0002",
-      "source": ".shall/rfcs/JTOOL-0002-agent-boundaries.md#L40",
+      "source": ".shall/rfcs/JTOOL-0002-agent-boundaries.md#L50",
       "status": "enforced",
       "text": "Writes to external systems (GitHub reviews, comments, thread resolution) MUST be performed by deterministic code, never by an agent, and the command that performs them MUST offer a `--dry-run` that performs no write.",
       "title": "External writes are deterministic and previewable"
@@ -355,7 +355,7 @@
       "level": "MUST",
       "not_applies_when": "No agent runs, no agent output is used, and nothing is written to an external system.",
       "rfc": "JTOOL-0002",
-      "source": ".shall/rfcs/JTOOL-0002-agent-boundaries.md#L48",
+      "source": ".shall/rfcs/JTOOL-0002-agent-boundaries.md#L58",
       "status": "enforced",
       "text": "Whether a finding is blocking MUST be computed by code from the corpus (RFC status and statement level) and the verification verdict; agent output MUST only lower a finding's severity, never raise it.",
       "title": "Severity comes from the corpus and verification"
@@ -378,7 +378,7 @@
       "level": "MUST",
       "not_applies_when": "No agent runs, no agent output is used, and nothing is written to an external system.",
       "rfc": "JTOOL-0002",
-      "source": ".shall/rfcs/JTOOL-0002-agent-boundaries.md#L56",
+      "source": ".shall/rfcs/JTOOL-0002-agent-boundaries.md#L66",
       "status": "enforced",
       "text": "A finding MUST NOT block a merge unless the verifier confirmed it from code or configuration it was shown, and a finding MUST only be dropped as refuted when the verdict cites evidence that exists.",
       "title": "Blocking needs seen evidence"
@@ -401,7 +401,7 @@
       "level": "MUST",
       "not_applies_when": "The content does not change CLI commands, exit codes or generated files.",
       "rfc": "JTOOL-0003",
-      "source": ".shall/rfcs/JTOOL-0003-cli-contract.md#L25",
+      "source": ".shall/rfcs/JTOOL-0003-cli-contract.md#L36",
       "status": "enforced",
       "text": "Commands MUST exit with 0 on success, 1 on errors, 2 on blocking findings and 3 when a Jev request fails after retries, and new failure cases MUST reuse one of these codes.",
       "title": "Stable exit codes"
@@ -424,7 +424,7 @@
       "level": "MUST",
       "not_applies_when": "The content does not change CLI commands, exit codes or generated files.",
       "rfc": "JTOOL-0003",
-      "source": ".shall/rfcs/JTOOL-0003-cli-contract.md#L32",
+      "source": ".shall/rfcs/JTOOL-0003-cli-contract.md#L43",
       "status": "enforced",
       "text": "The commands `lint`, `build`, `new`, `show`, `list`, `catalog`, `init`, `validate`, `bundle` and `publish` MUST NOT call Jev or Claude.",
       "title": "Deterministic commands stay model-free"
@@ -447,7 +447,7 @@
       "level": "MUST",
       "not_applies_when": "The content does not change CLI commands, exit codes or generated files.",
       "rfc": "JTOOL-0003",
-      "source": ".shall/rfcs/JTOOL-0003-cli-contract.md#L39",
+      "source": ".shall/rfcs/JTOOL-0003-cli-contract.md#L50",
       "status": "enforced",
       "text": "Generated files MUST be byte-for-byte reproducible from their inputs: no timestamps, no absolute paths, and sorted keys in JSON.",
       "title": "Reproducible generated files"
@@ -469,7 +469,7 @@
       "level": "MUST",
       "not_applies_when": "The content is not a skill, agent definition or plugin manifest.",
       "rfc": "JTOOL-0004",
-      "source": ".shall/rfcs/JTOOL-0004-plugin-content.md#L25",
+      "source": ".shall/rfcs/JTOOL-0004-plugin-content.md#L32",
       "status": "approved",
       "text": "The `description` of a skill or agent MUST state the situations in which it is used, not only what it does.",
       "title": "Descriptions say when to use"
@@ -491,7 +491,7 @@
       "level": "MUST",
       "not_applies_when": "The content is not a skill, agent definition or plugin manifest.",
       "rfc": "JTOOL-0004",
-      "source": ".shall/rfcs/JTOOL-0004-plugin-content.md#L32",
+      "source": ".shall/rfcs/JTOOL-0004-plugin-content.md#L39",
       "status": "approved",
       "text": "The review instructions MUST live only in `agents/shall-reviewer.md`, which both the in-session agent and the CI pipeline use; skills and code MUST NOT carry a second copy.",
       "title": "One reviewer prompt"
@@ -513,7 +513,7 @@
       "level": "SHOULD",
       "not_applies_when": "The content is not a skill, agent definition or plugin manifest.",
       "rfc": "JTOOL-0004",
-      "source": ".shall/rfcs/JTOOL-0004-plugin-content.md#L39",
+      "source": ".shall/rfcs/JTOOL-0004-plugin-content.md#L46",
       "status": "approved",
       "text": "Every `shall` command and flag shown in a skill SHOULD exist in the CLI (`shall <command> -h`).",
       "title": "Commands in skills exist"

.shall/rfcs/JTOOL-0001-jev-usage.md

@@ -27,6 +27,14 @@ unrelated state. The first conflict checker put the whole corpus in one state an
 about `existing[i]`: almost every pair came back as a conflict. A Choice split 0.51 / 0.49
 between two failing classes and hid a real weakening. These rules keep those lessons.
 
+## Checks in code
+
+Code checks part of these rules. Review checks the rest.
+
+- JTOOL-0001.1: `make check` (import-linter contract "Only shall.jev imports the TypeSafe SDK").
+- JTOOL-0001.4: `plugins/shall/tooling/tests/test_boundaries.py` (`test_default_models_are_pinned_and_defined_once`).
+- JTOOL-0001.2 and JTOOL-0001.3: review only.
+
 ## Requirements
 
 ### JTOOL-0001.1 One client, one wrapper

.shall/rfcs/JTOOL-0002-agent-boundaries.md

@@ -21,6 +21,16 @@ not_applies_when: >
 
 # JTOOL-0002: Agent boundaries
 
+## Checks in code
+
+Code checks part of these rules. Review checks the rest.
+
+- JTOOL-0002.1: `plugins/shall/tooling/tests/test_boundaries.py` (`test_every_claude_call_is_tool_less`, `test_only_owner_modules_start_processes`).
+- JTOOL-0002.2: `plugins/shall/tooling/tests/test_boundaries.py` (`test_findings_are_built_only_by_validation`, `test_findings_schema_matches_raw_finding`).
+- JTOOL-0002.4: `plugins/shall/tooling/tests/test_publish.py` (`test_dry_run_reads_but_never_writes`).
+- JTOOL-0002.5 and JTOOL-0002.6: `Finding.blocking` in `plugins/shall/tooling/src/shall/findings.py` is
+  computed, not stored. `plugins/shall/tooling/tests/test_verify.py` tests each verdict.
+
 ## Requirements
 
 ### JTOOL-0002.1 Agents run without tools in the pipeline

.shall/rfcs/JTOOL-0003-cli-contract.md

@@ -20,6 +20,17 @@ not_applies_when: >
 
 # JTOOL-0003: shall CLI contract
 
+## Checks in code
+
+Code checks part of these rules. Review checks the rest.
+
+- JTOOL-0003.1: `ExitCode` in `plugins/shall/tooling/src/shall/cli.py`, and `plugins/shall/tooling/tests/test_boundaries.py`
+  (`test_every_command_returns_an_exit_code`, `test_usage_error_exits_1_not_2`,
+  `test_scripts_exit_with_shall_codes_only`).
+- JTOOL-0003.2: `make check` (import-linter contract "Deterministic modules call no model"), and `plugins/shall/tooling/tests/test_boundaries.py`
+  (`test_deterministic_commands_reach_no_model_module`, `test_deterministic_commands_do_not_load_the_sdk`).
+- JTOOL-0003.3: `make check` and `make self-check` (`shall build --check`).
+
 ## Requirements
 
 ### JTOOL-0003.1 Stable exit codes

.shall/rfcs/JTOOL-0004-plugin-content.md

@@ -20,6 +20,13 @@ not_applies_when: >
 
 # JTOOL-0004: Plugin skills and agents
 
+## Checks in code
+
+Code checks part of these rules. Review checks the rest.
+
+- JTOOL-0004.3: `plugins/shall/tooling/tests/test_boundaries.py` (`test_docs_and_skills_use_existing_commands_and_flags`).
+- JTOOL-0004.1 and JTOOL-0004.2: review only.
+
 ## Requirements
 
 ### JTOOL-0004.1 Descriptions say when to use

AGENTS.md

@@ -20,8 +20,10 @@ Plain `shall` resolves to `.shall/` (this repo's `JTOOL-` rules). The example or
 
 ## Commands
 ```bash
-make test         # unit tests, no network
-make check        # org corpus lint + index current, no model calls
+make test         # unit + boundary tests, no network
+make typecheck    # pyright (Literal types, exhaustive match)
+make lint-imports # deterministic modules import no model module
+make check        # typecheck + lint-imports + org corpus lint + index current, no model calls
 make build        # after editing corpus/ (index is committed)
 make self-check   # this repo's own rules
 ```

Makefile

@@ -4,9 +4,12 @@ JRFC := plugins/shall/bin/shall
 # the root resolves to .shall/; organisation targets pass --config explicitly.
 ORG := $(JRFC) --config shall.yaml
 
-.PHONY: check lint build test eval eval-scale eval-facts eval-hooks eval-hooks-claude eval-triage eval-realpr review-example self-check self-conflicts self-review
+TOOLING := uv run --quiet --project plugins/shall/tooling
 
-check: lint          ## organisation corpus: deterministic CI checks (no model calls)
+.PHONY: check lint build test typecheck lint-imports test-plugin eval eval-scale eval-facts eval-hooks eval-hooks-claude eval-triage eval-realpr review-example self-check self-conflicts self-review
+
+# test-plugin is not part of check: it needs the claude CLI, and check must run with uv only.
+check: lint typecheck lint-imports  ## deterministic CI checks: corpus, index, types, module layers (no model calls)
 	$(ORG) build --check
 
 lint:
@@ -16,7 +19,17 @@ build:
 	$(ORG) build
 
 test:                ## tooling unit tests (no network)
-	uv run --quiet --project plugins/shall/tooling pytest -q plugins/shall/tooling/tests
+	$(TOOLING) pytest -q plugins/shall/tooling/tests
+
+typecheck:           ## pyright: Literal statuses/levels/verdicts, Optional narrowing, exhaustive match
+	$(TOOLING) --group check pyright -p plugins/shall/tooling/pyproject.toml
+
+lint-imports:        ## import-linter: deterministic modules never import a model module (JTOOL-0003.2)
+	$(TOOLING) --group check lint-imports --config plugins/shall/tooling/pyproject.toml
+
+test-plugin:         ## plugin manifest and TypeScript hook tests (needs the claude CLI)
+	claude plugin validate plugins/shall
+	claude plugin test plugins/shall
 
 eval:                ## selection recall/precision on eval/cases (needs TYPESAFE_API_KEY)
 	$(ORG) eval --out .shall-out/eval.json