From b1976c594af8db5f4e912b8679e30871c3727044 Mon Sep 17 00:00:00 2001 From: Jammy2211 Date: Tue, 1 Sep 2026 17:58:17 -0400 Subject: [PATCH 1/2] =?UTF-8?q?cortex:=20schema,=20docs,=20fixtures=20and?= =?UTF-8?q?=20workflows=20=E2=80=94=20the=20skeleton,=20part=20A=20(PyAuto?= =?UTF-8?q?Mind#379)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bootstrap of the run-and-ruling registry: AGENTS/README/REFERENCE in the Mind register, projects.yaml (restricted YAML subset, empty map, one commented worked example), epics.md with mind-half:, rulings/AGENTS.md (append-only, supersede-never-edit), the rolling-board batches docs with the one review grammar, policy copies, docs/schema_decisions.md (31 dated decisions), cortex_check.yml and ledger_merge.yml (trial-merge check), the repos_sync-generated CLAUDE.md / .claude/ and spliced AGENTS.md blocks, and the test fixtures (skeleton: one project, one phase per state, five rulings with one chain, one batch record + review; empty). scripts/ and tests/*.py follow in part B. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01SESCzU1FcwktEAdTZfmf3j --- .claude/hooks/session-start.sh | 568 ++++++++++++++++++ .claude/settings.json | 14 + .github/workflows/cortex_check.yml | 46 ++ .github/workflows/ledger_merge.yml | 256 ++++++++ AGENTS.md | 209 +++++++ CLAUDE.md | 5 + README.md | 40 +- REFERENCE.md | 484 +++++++++++++++ batches/AGENTS.md | 82 +++ batches/packets/AGENTS.md | 35 ++ batches/packets/TEMPLATE.md | 95 +++ batches/reviews/AGENTS.md | 40 ++ docs/schema_decisions.md | 123 ++++ epics.md | 27 + policy/never_rewrite_history.md | 6 + policy/remote_sessions.md | 44 ++ projects.yaml | 49 ++ rulings/AGENTS.md | 47 ++ tests/fixtures/empty/phases/.gitkeep | 0 tests/fixtures/empty/projects.yaml | 1 + tests/fixtures/empty/rulings/.gitkeep | 0 .../skeleton/batches/2026-09-01-pm.md | 26 + .../skeleton/batches/reviews/2026-09-01-pm.md | 44 ++ .../skeleton/phases/example/01_scope.md | 30 + .../phases/example/02_gated_on_dev.md | 31 + .../phases/example/03_ready_cleared.md | 31 + .../phases/example/04_submitted_override.md | 35 ++ .../phases/example/05_running_array.md | 39 ++ .../skeleton/phases/example/06_pulled.md | 33 + .../phases/example/07_awaiting_ruling.md | 35 ++ .../skeleton/phases/example/08_accepted.md | 37 ++ .../skeleton/phases/example/09_rerun.md | 35 ++ .../skeleton/phases/example/10_dropped.md | 35 ++ tests/fixtures/skeleton/projects.yaml | 14 + .../skeleton/rulings/2026/09/R-20260901-01.md | 18 + .../skeleton/rulings/2026/09/R-20260901-02.md | 22 + .../skeleton/rulings/2026/09/R-20260901-03.md | 18 + .../skeleton/rulings/2026/09/R-20260901-04.md | 19 + .../skeleton/rulings/2026/09/R-20260901-05.md | 19 + 39 files changed, 2688 insertions(+), 4 deletions(-) create mode 100755 .claude/hooks/session-start.sh create mode 100644 .claude/settings.json create mode 100644 .github/workflows/cortex_check.yml create mode 100644 .github/workflows/ledger_merge.yml create mode 100644 AGENTS.md create mode 100644 CLAUDE.md create mode 100644 REFERENCE.md create mode 100644 batches/AGENTS.md create mode 100644 batches/packets/AGENTS.md create mode 100644 batches/packets/TEMPLATE.md create mode 100644 batches/reviews/AGENTS.md create mode 100644 docs/schema_decisions.md create mode 100644 epics.md create mode 100644 policy/never_rewrite_history.md create mode 100644 policy/remote_sessions.md create mode 100644 projects.yaml create mode 100644 rulings/AGENTS.md create mode 100644 tests/fixtures/empty/phases/.gitkeep create mode 100644 tests/fixtures/empty/projects.yaml create mode 100644 tests/fixtures/empty/rulings/.gitkeep create mode 100644 tests/fixtures/skeleton/batches/2026-09-01-pm.md create mode 100644 tests/fixtures/skeleton/batches/reviews/2026-09-01-pm.md create mode 100644 tests/fixtures/skeleton/phases/example/01_scope.md create mode 100644 tests/fixtures/skeleton/phases/example/02_gated_on_dev.md create mode 100644 tests/fixtures/skeleton/phases/example/03_ready_cleared.md create mode 100644 tests/fixtures/skeleton/phases/example/04_submitted_override.md create mode 100644 tests/fixtures/skeleton/phases/example/05_running_array.md create mode 100644 tests/fixtures/skeleton/phases/example/06_pulled.md create mode 100644 tests/fixtures/skeleton/phases/example/07_awaiting_ruling.md create mode 100644 tests/fixtures/skeleton/phases/example/08_accepted.md create mode 100644 tests/fixtures/skeleton/phases/example/09_rerun.md create mode 100644 tests/fixtures/skeleton/phases/example/10_dropped.md create mode 100644 tests/fixtures/skeleton/projects.yaml create mode 100644 tests/fixtures/skeleton/rulings/2026/09/R-20260901-01.md create mode 100644 tests/fixtures/skeleton/rulings/2026/09/R-20260901-02.md create mode 100644 tests/fixtures/skeleton/rulings/2026/09/R-20260901-03.md create mode 100644 tests/fixtures/skeleton/rulings/2026/09/R-20260901-04.md create mode 100644 tests/fixtures/skeleton/rulings/2026/09/R-20260901-05.md diff --git a/.claude/hooks/session-start.sh b/.claude/hooks/session-start.sh new file mode 100755 index 0000000..f624d3e --- /dev/null +++ b/.claude/hooks/session-start.sh @@ -0,0 +1,568 @@ +#!/usr/bin/env bash +# GENERATED — canonical source: PyAutoMind/policy/session_start_hook.sh +# Installed into every checked-out repo as .claude/hooks/session-start.sh by +# `python3 PyAutoMind/scripts/repos_sync.py --write`, and drift-checked by +# `--check`. Edit the canonical file, never a copy. +# +# --------------------------------------------------------------------------- +# Python 3.12 is the default in Claude Code web/mobile sessions. +# +# The remote container ships /usr/local/bin/python{,3} -> /usr/bin/python3.11, +# a pip whose shebang is #!/usr/bin/python3, and a uv-managed tool set (pytest, +# ruff, black, mypy, pyright, flake8, poetry) every one of which was built on +# 3.11 — one minor version below the floor the organism set for itself in the +# Python 3.12 floor campaign, and below every CI leg it runs. The image's own +# `use-python 3.12` does not fix it: it moves the update-alternatives links +# under /usr/bin, which /usr/local/bin/python{,3} then shadow. +# +# This hook makes a session 3.12 on three surfaces: +# +# 1. a 3.12 virtualenv first on PATH — python, python3, pip, pytest; +# 2. the /usr/local/bin/python{,3} symlinks repointed at 3.12, so anything +# resolving PATH without this session's env (a subprocess with a scrubbed +# environment, a `#!/usr/bin/env python3` script) also gets 3.12; +# 3. the uv-managed tools rebuilt on 3.12 — mypy and flake8 read the +# interpreter's version, so on 3.11 they judged code against 3.11 rules — +# and then repaired, because (2) is what breaks each tool env's own +# `bin/python`, and a rebuild cannot fix a link it is handed. +# +# What it deliberately does NOT touch: the update-alternatives links under +# /usr/bin. Scripts with a literal `#!/usr/bin/python3` shebang follow those, +# and some of the image's own tools (conan) are installed for 3.11 only — a +# flip there breaks them for no gain the three surfaces above don't already +# give. +# +# Remote-only (a local checkout keeps whatever the developer's shell provides), +# idempotent (everything is skipped once it already reads 3.12, so the second +# repo's copy in the same session costs ~0.2s), and non-blocking: every step +# degrades to a logged warning rather than failing the session start. +# +# Per-repo dependencies: a repo that needs more than pytest + PyYAML declares it +# in .claude/session-python.txt — one pip argument per line (`-e .`, a package +# spec, `-r requirements.txt`; `#` comments ignored). Installed additively into +# the shared venv, so the file stays out of this generated hook and the hook +# stays byte-identical in every repo. +set -euo pipefail + +[ "${CLAUDE_CODE_REMOTE:-}" = "true" ] || exit 0 + +VENV="${PYAUTO_SESSION_VENV:-$HOME/.pyauto/session-py312}" +# pytest-xdist is a base dep, not a nicety: a remote container has 4 cores and +# the Brain suite takes 96s on one of them and 28s on four. A session that has +# to `pip install` before it can run tests fast will simply not run them fast. +BASE_DEPS=(pytest PyYAML pytest-xdist) + +# Which checkout is this copy of the hook installed in? +# +# NOT $CLAUDE_PROJECT_DIR. That is the session's project directory, which equals +# the repo only when the session holds exactly ONE repo. A session scoped to +# several organs clones them side by side under the project directory +# (/home/user/PyAutoMind, /home/user/PyAutoBrain, ...), and then +# $CLAUDE_PROJECT_DIR is their parent — so reading session-python.txt from it +# found nothing, silently, in exactly the sessions that hold the most repos. +# +# Derive it from where this script actually is, in both the installed and the +# canonical location. +HOOK_SELF="$(readlink -f "$0")" +case "$(dirname "$HOOK_SELF")" in + */.claude/hooks) REPO_DIR="$(cd "$(dirname "$HOOK_SELF")/../.." && pwd)" ;; + */policy) REPO_DIR="$(cd "$(dirname "$HOOK_SELF")/.." && pwd)" ;; + *) REPO_DIR="${CLAUDE_PROJECT_DIR:-$PWD}" ;; +esac +WORKSPACE_ROOT="$(dirname "$REPO_DIR")" +EXTRAS_FILE="$REPO_DIR/.claude/session-python.txt" + +# stderr, not stdout: a SessionStart hook's stdout is fed to the agent as +# session context. +log() { printf '[session-start] %s\n' "$*" >&2; } + +is_py312() { + [ -x "$1" ] && "$1" -c 'import sys; raise SystemExit(sys.version_info[:2] != (3, 12))' >/dev/null 2>&1 +} + +find_base_python() { + local candidate + for candidate in /usr/bin/python3.12 /usr/local/bin/python3.12 \ + "$(command -v python3.12 2>/dev/null || true)"; do + if [ -n "$candidate" ] && is_py312 "$candidate"; then + printf '%s\n' "$candidate" + return 0 + fi + done + # No system 3.12 (a future base image could drop it) — uv can fetch one. + if command -v uv >/dev/null 2>&1; then + log "no system python3.12; asking uv to install one" + uv python install 3.12 >&2 || return 1 + candidate="$(uv python find 3.12 2>/dev/null || true)" + if [ -n "$candidate" ] && is_py312 "$candidate"; then + printf '%s\n' "$candidate" + return 0 + fi + fi + return 1 +} + +venv_ready() { + is_py312 "$VENV/bin/python" \ + && [ -x "$VENV/bin/pip" ] \ + && "$VENV/bin/python" -c 'import pytest, yaml, xdist' >/dev/null 2>&1 +} + +# 1. The interpreter a session types: python, python3, pip, pytest. +ensure_venv() { + local base_python + if venv_ready; then + log "reusing $VENV ($("$VENV/bin/python" -V 2>&1))" + return 0 + fi + if ! base_python="$(find_base_python)"; then + log "WARNING: no Python 3.12 in this container; PATH left unchanged" + return 1 + fi + log "building $VENV on $base_python" + rm -rf "$VENV" + mkdir -p "$(dirname "$VENV")" + # --system-site-packages makes the venv a strict SUPERSET of the base + # interpreter. That is what lets `point_system_default` make this venv the + # session's `python3` without silently removing anything the image + # installed: an isolated venv would swap one set of missing modules for + # another, and the session would find out through a ModuleNotFoundError + # that reads like broken code. + if command -v uv >/dev/null 2>&1; then + # --seed puts pip inside the venv too, so `pip install` targets 3.12 + # rather than falling through to the container's 3.11 /usr/bin/pip. + uv venv --seed --system-site-packages --python "$base_python" "$VENV" >&2 + uv pip install --python "$VENV/bin/python" --quiet "${BASE_DEPS[@]}" >&2 + else + "$base_python" -m venv --system-site-packages "$VENV" >&2 + "$VENV/bin/python" -m pip install --quiet --upgrade pip >&2 + "$VENV/bin/python" -m pip install --quiet "${BASE_DEPS[@]}" >&2 + fi + venv_ready || { log "WARNING: could not build a 3.12 venv at $VENV"; return 1; } +} + +# This repo's own dependencies, if it declares any. Additive and marked, so a +# session holding several repos installs each repo's set exactly once. +ensure_repo_extras() { + [ -r "$EXTRAS_FILE" ] || return 0 + local marker args=() + marker="$VENV/.extras-$(cksum <"$EXTRAS_FILE" | tr -d ' /')" + [ -e "$marker" ] && return 0 + while IFS= read -r line; do + line="${line%%#*}" + line="$(printf '%s' "$line" | tr -d '\r' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" + [ -n "$line" ] && args+=("$line") + done <"$EXTRAS_FILE" + [ ${#args[@]} -gt 0 ] || return 0 + log "installing this repo's declared deps: ${args[*]}" + if (cd "$(dirname "$(dirname "$EXTRAS_FILE")")" && "$VENV/bin/python" -m pip install --quiet "${args[@]}" >&2); then + : >"$marker" + else + log "WARNING: $EXTRAS_FILE install failed; continuing without it" + fi +} + +# 2. What PATH means without this session's env file. +# +# This leg is the one that matters most, because the env file is exactly what a +# multi-repo session does NOT get: Claude Code registers project hooks from the +# session's project directory, that directory is the repos' parent, and no hook +# runs there (see leg 5). So every Bash call in such a session resolves whatever +# the image put on PATH, and these two names are the fallback that has to be +# right on its own. +# +# It used to point them at the BASE interpreter — `point_system_default +# "$(readlink -f "$VENV/bin/python")"` resolved the venv's python straight +# through to /usr/bin/python3.12. That satisfied the version question and lost +# everything else: `python3 -m pytest` answered "No module named pytest", and +# the session's own venv, sitting one directory away with pytest in it, was +# unreachable from any shell. +# +# Two details make the fix work: +# +# * A wrapper script, not a symlink. Python resolves a symlinked executable +# before looking for `pyvenv.cfg`, so a symlink to $VENV/bin/python lands on +# the base interpreter's prefix and the venv is lost again — silently, and +# in the same shape as the bug being fixed. `exec`ing it keeps the venv. +# * --system-site-packages on the venv (see ensure_venv), so pointing the +# system default here adds the session's packages without removing the +# image's. +# Bounded, because this predicate is what runs a candidate interpreter — and +# the one failure mode worth surviving is an interpreter that never returns. +venv_backed() { + [ -x "$1" ] || return 1 + [ "$(timeout 20 "$1" -c 'import sys; print(sys.prefix)' 2>/dev/null)" = "$VENV" ] +} + +# Does following $1's symlinks pass through $2? A venv's python legitimately +# RESOLVES to the same base interpreter a system default points at, so comparing +# endpoints refuses safe rewrites. The question that matters is narrower: is the +# path being rewritten a link in the target's own chain — because then the +# wrapper execs itself. +links_through() { + local node="$1" needle="$2" hops=0 + needle="$(cd "$(dirname "$needle")" 2>/dev/null && printf '%s/%s' "$(pwd -P)" "$(basename "$needle")")" + while [ -L "$node" ] && [ "$hops" -lt 40 ]; do + node="$(cd "$(dirname "$node")" && cd "$(dirname "$(readlink "$node")")" 2>/dev/null && printf '%s/%s' "$(pwd -P)" "$(basename "$(readlink "$node")")")" + [ "$node" = "$needle" ] && return 0 + hops=$((hops + 1)) + done + return 1 +} + +# `rm -f` first, and it is not tidiness. /usr/local/bin/python3 is a SYMLINK to +# the real interpreter, and a redirect opens the link's TARGET: `cat >` there +# overwrites /usr/bin/python3.12 itself with this wrapper. The venv's own +# python symlinks to that same file, so the wrapper then execs itself — the +# container loses its interpreter and every `python3` spins at 100% CPU. Writing +# a fresh file at the path replaces the link instead of following it. +# +# `links_through` is the second half of the same guard: a target that reaches +# the destination through its own symlink chain builds the identical loop by the +# other route, so refuse rather than write it. +write_venv_shim() { + local dest="$1" target="$2" + [ -w "$(dirname "$dest")" ] || { log "WARNING: cannot rewrite $dest (not writable)"; return 1; } + [ -x "$target" ] || { log "WARNING: $target is not executable; leaving $dest alone"; return 1; } + if links_through "$target" "$dest"; then + log "WARNING: refusing to point $dest at $target — it links back through $dest" + return 1 + fi + rm -f "$dest" + cat >"$dest" < $VENV/bin/python (venv, with pytest and yaml)" \ + || log "WARNING: $bin/python3 is still $(timeout 20 "$bin/python3" -V 2>&1)" +} + +# 2b. The `pytest` that PATH actually finds. +# +# $HOME/.local/bin precedes /usr/local/bin, and it holds uv's tool shims — so +# fixing python3 alone still leaves bare `pytest` resolving to uv's ISOLATED +# pytest, which by design cannot see PyYAML or a repo's own extras. In this +# workspace that made `pytest` exit on four collection ImportErrors that read +# like broken source, in a session where the suite was in fact green. +# +# A session should have exactly one pytest, and it should be the venv's. +# Confined to uv's own shim directory: a distro-packaged pytest is not ours to +# overwrite. Runs after retool_uv_tools, which rewrites these same shims. +point_pytest_at_venv() { + local shim_dir="${HOME}/.local/bin" name path + [ -x "$VENV/bin/pytest" ] || return 0 + for name in pytest py.test; do + path="$(command -v "$name" 2>/dev/null)" || continue + [ -n "$path" ] || continue + case "$path" in + "$VENV"/*) continue ;; + "$shim_dir"/*) ;; + *) log "WARNING: $name resolves to $path, outside uv's shim dir — left alone"; continue ;; + esac + write_venv_shim "$path" "$VENV/bin/$name" \ + && log "$path -> $VENV/bin/$name (so \`$name\` sees this repo's deps)" + done +} + +# 2c. Every OTHER console script the venv owns. +# +# `pytest` and `python3` survive without the venv on PATH because 2 and 2b give +# each one its own shim. Nothing else does — and PATH is exactly what a +# multi-repo session never gets, because the env file that exports it is written +# by Claude Code around a hook it never registers (see leg 5). So a dependency +# declared in `.claude/session-python.txt` that ships a CONSOLE SCRIPT rather +# than an importable module was installed into the venv and then invisible. +# +# Measured, not hypothetical: PyAutoHands shells out to `ipynb-py-convert`, and +# with the package installed its suite still failed five tests with +# `FileNotFoundError: ipynb-py-convert` — the binary sitting in $VENV/bin, +# unreachable, while `--check` called the session healthy. A missing binary and +# an unreachable one are the same symptom, and neither says "environment". +# +# The claim policy is 2b's, because the reasoning is 2b's: a name we can reach +# only through uv's shim dir or not at all is ours to provide; a name the image +# owns elsewhere is not ours to overwrite. +point_venv_scripts_at_venv() { + local shim_dir="${HOME}/.local/bin" entry name path + [ -d "$VENV/bin" ] || return 0 + mkdir -p "$shim_dir" 2>/dev/null || return 0 + for entry in "$VENV"/bin/*; do + [ -f "$entry" ] && [ -x "$entry" ] || continue + name="$(basename "$entry")" + case "$name" in + # python/pip are legs 2 and the venv's own plumbing; pytest is 2b. + # The activate family is meant to be sourced, never executed. + python*|pip*|pytest|py.test|activate*|deactivate*|*.bat|*.ps1|*.csh|*.fish|*.nu) continue ;; + esac + path="$(command -v "$name" 2>/dev/null)" || path="" + case "$path" in + "$VENV"/*) continue ;; + "") ;; + "$shim_dir"/*) ;; + *) log "WARNING: $name resolves to $path, outside uv's shim dir — left alone"; continue ;; + esac + write_venv_shim "$shim_dir/$name" "$entry" \ + && log "$shim_dir/$name -> $entry (venv console script)" + done +} + +# 3. The uv-managed tools — rebuilt on 3.12 only where they are not already. +# +# Pinned to the version already installed: this hook changes the INTERPRETER a +# tool runs on, and nothing else. Unpinned, `uv tool install --force` fetches +# the latest release, which quietly moved mypy across a major version (1.19 -> +# 2.3) the first time this ran — a lint-result change nobody asked for, riding +# in on a Python upgrade. Falls back to unpinned only if the pin cannot be +# resolved for 3.12. +retool_uv_tools() { + command -v uv >/dev/null 2>&1 || return 0 + local tools_dir tool name version spec + tools_dir="$(uv tool dir 2>/dev/null || echo "$HOME/.local/share/uv/tools")" + [ -d "$tools_dir" ] || return 0 + for tool in "$tools_dir"/*/; do + name="$(basename "$tool")" + is_py312 "$tool/bin/python" && continue + version="$(uv tool list 2>/dev/null | awk -v n="$name" '$1 == n {print substr($2, 2); exit}')" + spec="$name" + [ -n "$version" ] && spec="$name==$version" + if uv tool install --python 3.12 --force "$spec" >/dev/null 2>&1 \ + || uv tool install --python 3.12 --force "$name" >/dev/null 2>&1; then + log "rebuilt ${spec} on 3.12" + else + log "WARNING: could not rebuild $name on 3.12; it stays on $("$tool/bin/python" -V 2>&1)" + fi + done +} + +# 3b. The link uv's rebuild cannot fix from inside. +# +# uv creates each tool env with `bin/python` as a SYMLINK to whatever `python3` +# was at install time — here `/usr/local/bin/python3`, which leg 2 has already +# replaced with a wrapper that `exec`s the session venv. Every tool env's python +# then resolves its prefix to the VENV: `sys.prefix` is the venv, the tool's own +# site-packages never reaches `sys.path`, and the console script dies with +# `ModuleNotFoundError: No module named 'flake8'` — with flake8 sitting +# installed two directories away. +# +# TWO paths reach that state, which is why this runs after leg 3 rather than +# only on the envs leg 3 rebuilt: leg 3 rebuilds a 3.11 tool and hands the new +# env the hijacked path, and leg 2 separately breaks every PRE-EXISTING tool env +# that already pointed there and that leg 3 skips (`is_py312 … && continue`). +# Running here covers both, because leg 2 runs before leg 3. +# +# Measured 2026-08-27, post-bootstrap: mypy, flake8, black, poetry and pyright +# all dead this way; `ruff` survived (a native binary) and `pytest` survived +# (leg 2b points its shim straight at the venv, which has pytest). The +# bootstrap's `--check` called every one of them "3.12 OK", because the +# interpreter they reach IS 3.12 — it is simply the wrong one. A session then +# lints clean by not linting at all, and CI is the thing that finds out. +# +# The fix is one link: a venv's `bin/python` must resolve to a BASE interpreter, +# never to a path this hook hijacks. +# +# This lived in `scripts/session_bootstrap.sh` for one pass, which is the wrong +# home. The bootstrap is what a MULTI-repo session runs; a SINGLE-repo session +# registers this hook and never calls the bootstrap, so it got leg 2's breakage +# and none of the repair. +repair_uv_tools() { + local tools_dir base tool link prefix + base="$("$VENV/bin/python" -c 'import sys, os; print(os.path.join(sys.base_prefix, "bin", "python3.12"))' 2>/dev/null)" || base="" + [ -x "$base" ] || base="$(command -v python3.12 2>/dev/null)" || base="" + [ -x "$base" ] || return 0 + tools_dir="${PYAUTO_UV_TOOLS_DIR:-$(uv tool dir 2>/dev/null || echo "$HOME/.local/share/uv/tools")}" + [ -d "$tools_dir" ] || return 0 + for tool in "$tools_dir"/*/; do + link="${tool}bin/python" + [ -L "$link" ] || continue + # Ask the interpreter where it thinks it lives, rather than tracing the + # link: `/usr/local/bin/python3` is a WRAPPER SCRIPT (a symlink there + # would lose the venv — leg 2's whole note), so `readlink -f` stops at + # the wrapper and reports nothing about the venv behind it. sys.prefix + # is the outcome; anything else is the mechanism. + prefix="$("$link" -c 'import sys; print(sys.prefix)' 2>/dev/null)" || continue + [ -n "$prefix" ] || continue + [ "$prefix" = "${tool%/}" ] && continue # resolves to its own env: correct + # Non-fatal like every other leg: an unwritable tools dir is a warning, + # never a failed session start. + ln -sfn "$base" "$link" || { + log "WARNING: could not repoint $(basename "${tool%/}") at $base" + continue + } + prefix="$("$link" -c 'import sys; print(sys.prefix)' 2>/dev/null)" || prefix="" + if [ "$prefix" = "${tool%/}" ]; then + log "repointed $(basename "${tool%/}") at $base (it resolved into $VENV, not its own env)" + else + log "WARNING: $(basename "${tool%/}") still resolves to ${prefix:-nothing} — it will not run" + fi + done +} + +# 4. Honest git history. +# +# A remote session clones shallow. `git merge-base --is-ancestor` then LIES +# across the graft boundary: it reports "not an ancestor" for a commit whose +# ancestry is simply not in the clone, and the ship/close-out skills act on that +# answer. A completion record already logged this the hard way +# (complete/2026/08/status-sh-repos-missing-source.md, "environment note"). +# +# These repos are small (single-digit MB of .git), so unshallowing costs a +# couple of seconds once per container and removes a whole class of wrong +# answer. Bounded and non-fatal: a slow or blocked network leaves a shallow +# clone and a warning, never a failed session start. +ensure_full_clone() { + local repo + for repo in "$WORKSPACE_ROOT"/*/; do + [ -e "${repo}.git/shallow" ] || continue + log "unshallowing $(basename "$repo") (shallow clones make ancestry checks lie)" + if timeout 120 git -C "$repo" fetch --unshallow --quiet 2>/dev/null \ + || timeout 120 git -C "$repo" fetch --depth=2147483647 --quiet 2>/dev/null; then + log " $(basename "$repo"): full history ($(git -C "$repo" rev-list --count HEAD 2>/dev/null) commits)" + else + log " WARNING: $(basename "$repo") is still shallow — run 'git fetch --unshallow' before trusting any ancestry check" + fi + done +} + +# 5. Make the NEXT session in this container start correctly. +# +# Claude Code registers project hooks from the session's project directory. In a +# one-repo session that IS the repo, and `/.claude/settings.json` is found. +# In a session holding several organs the project directory is their parent, +# which is not a repo and has no `.claude/` — so none of the per-repo hooks are +# registered and none of this script runs. That is why a multi-repo session came +# up on the container's Python 3.11: not a broken hook, an unreachable one. +# +# A repo cannot ship a file into its own parent, so install it at run time. The +# settings we write fan out to every sibling repo's own hook, so the layout +# stays "each repo owns its hook" and this file stays generated-from-one-source. +# It takes effect on the next session start in this container. +install_workspace_settings() { + # Target the WORKSPACE ROOT, derived from this checkout — not + # $CLAUDE_PROJECT_DIR, and not only when the two differ. + # + # The early return this replaces ("only in the multi-repo layout") made the + # whole leg unreachable. Writing the fan-out requires the hook to be + # running; the hook runs only in a session whose project dir is a repo — + # that is, a SINGLE-repo session — and the early return then skipped it as + # having nothing to add. So the one session type that could seed the + # container never did, and the multi-repo session that needed it never ran + # the hook to find out. Observed directly: a container with two single-repo + # sessions behind it still had no workspace-root settings, and the next + # session — three organs, project dir /home/user — fired no hook, ran on + # PATH's pytest, and left both clones shallow for three minutes until an + # unrelated verb happened to knock on session_bootstrap.sh. + # + # A single-repo session has the same sibling layout one directory up, so it + # can seed the root for free. Skip only if the root is itself a repo (then + # it owns its own hook) or is not writable. + local root="$WORKSPACE_ROOT" + [ -n "$root" ] && [ -d "$root" ] || return 0 + [ -d "$root/.git" ] && return 0 + + local settings="$root/.claude/settings.json" + local fanout="$root/.claude/hooks/session-start.sh" + [ -w "$root" ] || { log "WARNING: $root not writable; multi-repo sessions will keep skipping the hook"; return 0; } + + mkdir -p "$(dirname "$fanout")" + rm -f "$fanout" # never write through a symlink; see write_venv_shim + cat >"$fanout" <<'FANOUT' +#!/usr/bin/env bash +# GENERATED at session start by a PyAuto repo's own session-start hook. +# Runs every sibling repo's hook, because the workspace root is not a repo and +# therefore has no hook of its own. Each repo's hook is idempotent, so the +# second and later ones cost ~0.2s. +set -u +ROOT="$(cd "$(dirname "$(readlink -f "$0")")/../.." && pwd)" +for repo in "$ROOT"/*/; do + hook="${repo}.claude/hooks/session-start.sh" + [ -x "$hook" ] || continue + CLAUDE_PROJECT_DIR="${repo%/}" "$hook" || \ + printf '[session-start] WARNING: %s failed\n' "$hook" >&2 +done +FANOUT + chmod 0755 "$fanout" + + if [ ! -f "$settings" ]; then + cat >"$settings" <<'SETTINGS' +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "$CLAUDE_PROJECT_DIR/.claude/hooks/session-start.sh" + } + ] + } + ] + } +} +SETTINGS + log "installed $settings — multi-repo sessions in this container now run the hook" + fi +} + +# Git history and hook reachability have nothing to do with the interpreter, so +# they run first and unconditionally: a container where the 3.12 venv cannot be +# built still wants honest ancestry and a hook that fires next session. +# PYAUTO_SESSION_DEFINE_ONLY=1 defines every function and performs no action — +# the seam this hook's own tests use to drive one leg at a time against a +# temporary directory, instead of against the container they run in. Sourcing +# returns; executing exits. +if [ "${PYAUTO_SESSION_DEFINE_ONLY:-}" = "1" ]; then + return 0 2>/dev/null || exit 0 +fi + +# Run leg 3b on its own, without a session start. The door +# `scripts/session_bootstrap.sh` knocks on after it has run every repo's hook — +# a subprocess rather than a source, so this script's `set -euo pipefail` never +# leaks into a caller that is contractually "a bootstrap, never a gate". +if [ "${1:-}" = "--repair-uv-tools" ]; then + repair_uv_tools + exit 0 +fi + +ensure_full_clone +install_workspace_settings + +# PYAUTO_SESSION_SKIP_PYTHON=1 stops here — the seam the hook's own tests use to +# exercise the legs above without building an interpreter. +if [ "${PYAUTO_SESSION_SKIP_PYTHON:-}" = "1" ]; then + log "PYAUTO_SESSION_SKIP_PYTHON=1 — leaving the interpreter alone" + exit 0 +fi + +if ensure_venv; then + ensure_repo_extras + point_system_default + retool_uv_tools + repair_uv_tools + point_pytest_at_venv + point_venv_scripts_at_venv + # Every repo in the session registers this hook, so the second copy must not + # prepend the venv a second time. + if [ -n "${CLAUDE_ENV_FILE:-}" ] && ! grep -qs 'PYAUTO_SESSION_PY312=' "$CLAUDE_ENV_FILE"; then + { + echo "export PYAUTO_SESSION_PY312=\"$VENV\"" + echo "export VIRTUAL_ENV=\"$VENV\"" + echo "export PATH=\"$VENV/bin:\$PATH\"" + # The workspace root, so the Brain's shell/python resolvers agree + # with the session instead of each re-deriving it. + echo "export PYAUTO_ROOT=\"$WORKSPACE_ROOT\"" + } >>"$CLAUDE_ENV_FILE" + fi + log "default python is now $("$VENV/bin/python" -V 2>&1) ($VENV/bin)" +fi diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..e06b033 --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,14 @@ +{ + "hooks": { + "SessionStart": [ + { + "hooks": [ + { + "type": "command", + "command": "$CLAUDE_PROJECT_DIR/.claude/hooks/session-start.sh" + } + ] + } + ] + } +} diff --git a/.github/workflows/cortex_check.yml b/.github/workflows/cortex_check.yml new file mode 100644 index 0000000..0961d34 --- /dev/null +++ b/.github/workflows/cortex_check.yml @@ -0,0 +1,46 @@ +name: Cortex Check + +# Guards the run-and-ruling registry: every phase file parses and its state is +# legal, every ruling id is unique and every Supersedes: resolves, every +# project a phase names is in projects.yaml, the gates/witness invariants hold +# (`scripts/cortex.py check`), then the script's own tests. No self-heal — +# there is no generated render here; a failing check is a real contradiction. + +on: + push: + branches: [main] + paths: + - "phases/**" + - "rulings/**" + - "batches/**" + - "projects.yaml" + - "epics.md" + - "scripts/**" + - "tests/**" + - ".github/workflows/cortex_check.yml" + pull_request: + paths: + - "phases/**" + - "rulings/**" + - "batches/**" + - "projects.yaml" + - "epics.md" + - "scripts/**" + - "tests/**" + - ".github/workflows/cortex_check.yml" + workflow_dispatch: + +permissions: + contents: read + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: cortex check + run: python3 scripts/cortex.py check + - name: tests + run: | + python3 -m pip install --quiet pytest pyyaml + python3 -m pytest tests -q diff --git a/.github/workflows/ledger_merge.yml b/.github/workflows/ledger_merge.yml new file mode 100644 index 0000000..7b1992e --- /dev/null +++ b/.github/workflows/ledger_merge.yml @@ -0,0 +1,256 @@ +name: Cortex Ledger Merge + +# Lands PyAutoCortex's own ledger work without anyone having to ask for it. +# +# THE PROBLEM. A branch-scoped session — the phone, claude.ai/code, any +# `claude/**` flow — pushes a phase move, a ruling, a batch record to a feature +# branch and it stops there: nothing downstream looks at a `claude/**` push. +# The Mind learned this first (mind_ledger_merge.yml); the Cortex is born with +# the answer. +# +# WHY A WORKFLOW AND NOT THE SESSION. A workflow's GITHUB_TOKEN is a different +# credential from the session's, and this repo trusts it with `contents: +# write`. Running the merge here means the ledger lands from any surface that +# can push — including a chat on a phone — whether or not the session that +# pushed it ever comes back. +# +# WHAT IT WILL AND WILL NOT MERGE. Only a branch whose whole diff is *ledger*: +# `phases/`, `rulings/`, `batches/`, `epics.md`. The line is drawn by +# scripts/ledger_merge.py — a script, so it is testable and a session can +# predict the verdict — and it is DEFAULT DENY: anything under `scripts/`, +# `tests/`, `.github/`, `policy/`, `docs/`, `projects.yaml`, the prose pages, +# or any path nobody has classified, stops the merge and leaves the branch for +# a human. So does a MODIFIED or DELETED file under `rulings/` — the ledger of +# record is append-only, and the classifier reads `git diff --name-status` to +# hold that line. This workflow's own file is on the code side, which is the +# intended self-consistency: it cannot merge a change to itself. +# +# THE TRIAL-MERGE CHECK. `cortex.py check` is run on the branch tip (so audit +# mode reports something) and AGAIN on the merged tree, after `git merge +# --no-ff` and before `git push`. That second run is what catches the +# ruling-id race: two branches that both assign `R--03` each pass alone +# and fail merged. A failing check resets the trial merge and leaves the +# branch for a human. + +on: + push: + branches: ["claude/**"] + workflow_dispatch: + inputs: + branch: + description: "Branch to judge (default: the caller's ref)" + type: string + default: "" + mode: + description: "audit = classify and report only · merge = land it" + type: choice + options: [audit, merge] + default: audit + delete_branch: + description: "Delete the branch after a direct (non-PR) merge" + type: boolean + default: true + +# contents: write → the merge push to main and the branch delete. +# pull-requests: write → merge through an open PR when the branch has one, so +# the PR records as MERGED rather than being orphaned by a direct push. +# actions: write → re-dispatch cortex_check.yml on main. A push made with +# GITHUB_TOKEN triggers nothing, so without this main's check never runs on +# the merge commit. +permissions: + contents: write + pull-requests: write + actions: write + +# Every run of this workflow ends in a push to main, so runs must not overlap: +# two branches merging concurrently would race, and the loser would spend its +# three attempts re-merging. Queue rather than cancel — a cancelled run can +# leave a branch merged and undeleted. +concurrency: + group: cortex-ledger-merge + cancel-in-progress: false + +jobs: + merge: + # Never in a fork: this job writes to main. + if: github.repository == 'PyAutoLabs/PyAutoCortex' && github.event.deleted != true + runs-on: ubuntu-latest + steps: + - name: Resolve the branch + id: target + run: | + BRANCH="${{ inputs.branch }}" + [ -n "$BRANCH" ] || BRANCH="${{ github.ref_name }}" + case "$BRANCH" in + main|"") echo "::error::refusing to operate on '$BRANCH'"; exit 1 ;; + archive/condemned/*) echo "::error::'$BRANCH' is a Gut transit ref"; exit 1 ;; + esac + MODE="${{ inputs.mode }}" + # A push is the automatic door and merges; the manual door defaults to + # audit, so a human dispatching it to look never merges by accident. + [ "${{ github.event_name }}" = "push" ] && MODE="merge" + echo "branch=$BRANCH" >> "$GITHUB_OUTPUT" + echo "mode=$MODE" >> "$GITHUB_OUTPUT" + echo "Branch: \`$BRANCH\` · mode: \`$MODE\`" >> "$GITHUB_STEP_SUMMARY" + + - uses: actions/checkout@v4 + with: + ref: ${{ steps.target.outputs.branch }} + # Full history: the classifier diffs against the merge base with main, + # and `merge-base --is-ancestor` cannot answer on a shallow clone — + # it reports "not an ancestor" for ancestry that is merely absent. + fetch-depth: 0 + + - name: Is there anything to merge? + id: ahead + run: | + git fetch --quiet origin main + HEAD_SHA=$(git rev-parse HEAD) + echo "head_sha=$HEAD_SHA" >> "$GITHUB_OUTPUT" + if git merge-base --is-ancestor "$HEAD_SHA" origin/main; then + echo "already in main — nothing to merge" >> "$GITHUB_STEP_SUMMARY" + echo "merged=already" >> "$GITHUB_OUTPUT" + else + echo "merged=no" >> "$GITHUB_OUTPUT" + fi + + - name: Classify the diff — ledger, or code? + id: classify + if: steps.ahead.outputs.merged == 'no' + run: | + set +e + OUT=$(python3 scripts/ledger_merge.py classify --base origin/main) + RC=$? + set -e + echo "$OUT" + { + echo "" + echo "
Classification" + echo "" + echo '```' + echo "$OUT" + echo '```' + echo "" + echo "
" + } >> "$GITHUB_STEP_SUMMARY" + # Exit 2 is the gate failing to run, which is not "a human's turn" — + # it is a broken gate, and it must fail the job rather than read as a + # quiet decline to merge. + if [ "$RC" -eq 2 ]; then + echo "::error::ledger_merge.py could not classify the diff" + exit 1 + fi + if [ "$RC" -ne 0 ]; then + echo "verdict=code" >> "$GITHUB_OUTPUT" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "**Left for a human** — this branch changes code, or edits a ruling, not just the ledger." >> "$GITHUB_STEP_SUMMARY" + else + echo "verdict=ledger" >> "$GITHUB_OUTPUT" + fi + + # Structural drift on the branch tip BLOCKS. A phase in a state its + # header contradicts, a dangling Supersedes:, a run line that does not + # parse — nothing heals that automatically, and merging it would put the + # contradiction on main. Also the only check audit mode gets to report. + - name: Structural drift on the branch (blocking) + if: steps.classify.outputs.verdict == 'ledger' + run: python3 scripts/cortex.py check + + - name: Merge into main + id: merge + if: steps.classify.outputs.verdict == 'ledger' && steps.target.outputs.mode == 'merge' + env: + GH_TOKEN: ${{ github.token }} + BRANCH: ${{ steps.target.outputs.branch }} + HEAD_SHA: ${{ steps.ahead.outputs.head_sha }} + run: | + git config user.name "github-actions[bot]" + git config user.email "41898282+github-actions[bot]@users.noreply.github.com" + + # An open PR is merged THROUGH the PR, so it records as MERGED and + # its review thread closes properly. A direct push would land the + # same commits and leave the PR reading "closed", which is a lie + # about what happened to the work. The trial-merge check still runs + # first, on a local merge of the same head, so a PR cannot bypass it. + PR=$(gh pr list --head "$BRANCH" --state open --json number --jq '.[0].number // empty') + + # No PR, or a PR: either way, build the merge locally first and run + # the check on the MERGED tree. Each attempt rebuilds on the current + # tip of main, so a concurrent push never needs a rebase — the same + # convergence the Mind's self-heal loops use. + for attempt in 1 2 3; do + git fetch --quiet origin main + if git merge-base --is-ancestor "$HEAD_SHA" origin/main; then + echo "landed by a concurrent run" + echo "via=concurrent" >> "$GITHUB_OUTPUT" + exit 0 + fi + git checkout -B _ledger_merge origin/main + if ! git merge --no-ff "$HEAD_SHA" \ + -m "cortex: auto-merge ledger branch $BRANCH" \ + -m "Ledger-only diff (scripts/ledger_merge.py), cortex check clean on the merged tree." ; then + git merge --abort || true + echo "::error::'$BRANCH' conflicts with main — resolve it by hand" + echo "**Conflict** — \`$BRANCH\` does not merge cleanly; left for a human." >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + # THE TRIAL-MERGE CHECK. The branch passed alone; does it pass + # merged? This is where two branches that both assigned the same + # ruling id, or both moved the same phase, are caught. + if ! python3 scripts/cortex.py check; then + git reset --hard origin/main + echo "::error::'$BRANCH' merges cleanly but 'cortex.py check' fails on the merged tree — left for a human" + echo "**Blocked** — \`$BRANCH\` fails \`cortex.py check\` once merged with main (a ruling-id or phase-state race?); left for a human." >> "$GITHUB_STEP_SUMMARY" + exit 1 + fi + if [ -n "$PR" ]; then + echo "open PR #$PR on this branch — merging through it" + gh pr merge "$PR" --merge + echo "via=pr#$PR" >> "$GITHUB_OUTPUT" + echo "Merged **#$PR** (\`$BRANCH\` → main)." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + if git push origin HEAD:main; then + echo "merged on attempt $attempt" + echo "via=direct" >> "$GITHUB_OUTPUT" + echo "Merged \`$BRANCH\` → main (no PR; direct merge commit)." >> "$GITHUB_STEP_SUMMARY" + exit 0 + fi + echo "push rejected (attempt $attempt) — retrying on the new tip of main" + done + echo "::error::could not push the merge after 3 attempts" + exit 1 + + # A PR merge deletes its own head where the repo has "Automatically + # delete head branches" on; a DIRECT merge has no such hook. Delete it + # here, on the same proof branch_sweep uses — main must actually contain + # the head sha — so a failed or partial merge never loses a branch. + - name: Delete the merged branch + if: steps.merge.outputs.via != '' && inputs.delete_branch != false + env: + BRANCH: ${{ steps.target.outputs.branch }} + HEAD_SHA: ${{ steps.ahead.outputs.head_sha }} + run: | + git fetch --quiet origin main + if ! git merge-base --is-ancestor "$HEAD_SHA" origin/main; then + echo "::warning::main does not contain $HEAD_SHA — keeping '$BRANCH'" + exit 0 + fi + if git push origin --delete "$BRANCH"; then + echo "Deleted \`$BRANCH\`." >> "$GITHUB_STEP_SUMMARY" + else + # Already gone (a PR merge with auto-delete on) is the common case + # and is not a failure. + echo "::warning::could not delete '$BRANCH' — it may already be gone" + fi + + # The merge push above was made with GITHUB_TOKEN, which triggers no + # workflows, so main's check must be asked explicitly or the merge + # commit is never checked on main. + - name: Run the check on main + if: steps.merge.outputs.via != '' + env: + GH_TOKEN: ${{ github.token }} + run: | + gh workflow run cortex_check.yml --repo "${{ github.repository }}" --ref main \ + || echo "::warning::could not dispatch cortex_check.yml — its next push-triggered run will check main" diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..acecf79 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,209 @@ +# PyAutoCortex — Agent Guidance + +This file is for AI coding agents (Claude Code, Codex, Cursor, etc.) and humans +discovering this repository. PyAutoCortex is the **Cortex** organ of the PyAuto +organism — where the organism learns what is true. + + +**You are one organ of the PyAuto organism** — an agentic ecosystem for +human-led, natural-language software development. The organs below are +peer repositories; this repo is one of them, not a part of another. +Canonical boundaries live in `PyAutoBrain/ORGANISM.md`; the full body map +(every repo, not just organs) is `PyAutoMind/repos.yaml`. + +| Organ | Repo | Role | +|-------|------|------| +| **Brain** | PyAutoBrain | Reasoning/orchestration layer; how work is decomposed and routed; the specialist agents. | +| **Mind** | PyAutoMind | Intent, goals, priorities, workflow state; every task starts as a markdown prompt here. | +| **Cortex** | PyAutoCortex | The Cortex — where the organism learns what is true: the science body map (`projects.yaml`) and the rulings of record for every science run; the science mirror of the Mind (runs and rulings, not prompts and PRs). | +| **Memory** | PyAutoMemory | Long-term scientific/software/project knowledge (see science pointer below). | +| **Heart** | PyAutoHeart | Health/readiness — the authoritative "is it safe to release?" verdict. | +| **Hands** | PyAutoHands | Packaging, tagging, notebook generation, PyPI release execution. | +| **Nerves** | PyAutoNerves | The Nerves — the configuration/serialization layer connecting workspace conventions to libraries (layered config, version handshake, test_mode), delivered as the `autonerves` package. | +| **Gut** | PyAutoGut | Owns the lifecycle of condemned self-material (stale branches, stashes, dead code/tests): holds it as durable, recoverable git refs through a transit window and voids it on a sweep. The storage mirror of Memory (retention vs release). | + +Call chain (always this order): **Brain → Heart (gate) → Build (execute)**. Brain agents are **conductors** (front-door; a human drives them; they decide *and* act) or **faculties** (read-only opinions the conductors consult; they judge and stop). New capability grows as a faculty, not a new organ, unless it owns state or effects no existing organ can. + +Generated from `PyAutoMind/repos.yaml` + `PyAutoBrain/ORGANISM.md`; edit there, then run `python3 PyAutoMind/scripts/repos_sync.py --write`. + + +## What this repo is + +**The Mind decides what to build, the Brain routes the work and executes +nothing, the Cortex learns what is true.** The Cortex is a **run-and-ruling +registry**: `project → phase → runs → rulings`. It owns two pieces of state no +other organ holds — the **science body map** (`projects.yaml`: every science +project, where it lives on the laptop and on RAL, how it syncs, where its own +commentary ledger is) and the **rulings ledger of record** (`rulings/`: every +verdict a human has passed on a science run, append-only). + +It is **not a second PyAutoMind.** The Mind is PR-shaped all the way down — +prompts, issues, branches, completion records, "delivered = a PR with a diff +and checks". The Cortex's unit is a **phase** of a **project**, which spawns +**runs** (SLURM job ids) and ends in a **ruling**. Development tasks a science +phase waits on stay in the Mind and are named here only as **gates** — GitHub +issue/PR refs in a phase's `Gates:` header, one grammar, one direction. The +Mind learns nothing about the Cortex beyond a render-time badge. + +The schemas — phase files, the run-line grammar, rulings, batches, the +restricted `projects.yaml` subset — and the `scripts/cortex.py` verbs are in +[REFERENCE.md](REFERENCE.md). Every choice the birth epic did not fix is dated +in [docs/schema_decisions.md](docs/schema_decisions.md); read decisions there, +do not re-derive them. + +## The ruling of record + +**A verdict recorded only outside the Cortex does not exist.** `rulings/` is +canonical. A project's own ledger (`DECISIONS.md`, `state.md`, `RESULTS.md`) +remains as scientific commentary — evidence, reasoning, consequences — and +cites the ruling id. Rulings are **append-only**: a wrong ruling is superseded +by a new one (`Supersedes: R-…`) and never edited; `scripts/ledger_merge.py` +treats any modification or deletion under `rulings/` as code, which is a +human's turn. Full rules in [rulings/AGENTS.md](rulings/AGENTS.md). + +## Layout (operational) + +- **`projects.yaml`** — the science body map. One row per project. **This is + code, not ledger**: `sync_cli` and `local_path` are paths a conductor will + execute under, so a change to it is always a human's turn. It is written in a + restricted YAML subset that `cortex.py` parses with the stdlib (REFERENCE.md + "The restricted YAML subset"). +- **`phases//.md`** — one file per phase, with the Mind's light + header (`Key: value` lines, no YAML frontmatter) and a `## Runs` body in the + SLURM run-line grammar. `State:` is one of `planned | gated | ready | + submitted | running | pulled | awaiting-ruling | accepted | rerun | dropped`. + `legacy` and `legacy_wrong` are states of a **run**, never of a phase. +- **`rulings///R--.md`** — the ledger of record. +- **`batches/`** — the rolling review board: one record per slot, the archived + packet page under `packets/`, the human's verbatim review under `reviews/`. +- **`epics.md`** — the Cortex half of every split epic; `- mind-half:` names + the Mind entry by slug. +- **`scripts/cortex.py`** — the one lifecycle script (stdlib only): + - `check` — every structural rule, hermetic; `cortex check: OK` or `DRIFT` + with one ` - …` line per finding, exit 1. + - `gates [--grade [--write]]` — offline: the gated phases and their refs; + `--grade` polls GitHub and reports which gates cleared; `--write` flips + `gated → ready` (and `ready → gated` on a reopened gate). + - `rule --body …` — assigns the next ruling id, writes + the ruling file(s), updates the phase's `Ruling:` and `State:`. The only + door to `accepted`, `rerun`, `dropped` and `leave-to-finish`. + - `move …` — every other transition, per the table in + REFERENCE.md; refuses the ruling edges. + - `new --phase …` — writes a phase file from the + template. +- **`scripts/ledger_merge.py`** — the default-deny classifier behind + `.github/workflows/ledger_merge.yml`: a `claude/**` push whose whole diff is + ledger (`phases/`, `rulings/`, `batches/`, `epics.md`) lands on `main` by + itself; anything else waits for a human. `python3 scripts/ledger_merge.py + classify --base origin/main` predicts the verdict. + +## The workspace-paths exception + +Every other organ keeps its paths inside the workspace. The Cortex is the one +exception, and it is confined to `projects.yaml`: a project's `local_path` and +`mirror` are absolute laptop paths **outside** the workspace (the Science +folder under `/mnt/c/…`), because that is where the datasets, the `output/` +trees and the pulled results live and the review happens at the laptop. The +exception is stated in the file's header and here, and nowhere else — a phase +file or a ruling points into a project through its `projects.yaml` row, never +with a bare absolute path of its own. + +## The laptop lane — what is out of scope, and why + +Quoted verbatim from `PyAutoMind/draft/research/euclid/batch_science_lane.md` +so nobody re-derives these: + +> ## What is now out of scope, and why +> +> - **RAL as canonical home for the science project.** Refused above. The datasets +> and `output/` stay under `/mnt/c/…/Science/`. +> - **A git-courier cron on the RAL login node.** Its value collapses once the +> laptop is canonical: the laptop has to be on to hold and push the data anyway, +> so a courier saves almost nothing. It was the right answer to a question no +> longer being asked. +> - **Globus Compute endpoint / self-hosted GitHub runner on the login node.** +> Same reasoning, plus both are persistent login-node processes needing an +> operator conversation. Not worth it for a lane the human is happy to drive. +> - Recorded so nobody re-derives them: SSH from a Claude container is a +> non-starter in every variant (HTTPS-only proxied egress, no keys); Open +> OnDemand is an admin-installed inbound portal; Cirun cannot reach someone +> else's SLURM. And phases 4, 6a and 6b of `euclid-dr1-prep` say in their own +> prompts that they are human-driven and supervised with a judged verdict as the +> deliverable — **no transport was ever going to make those unattended.** + +Every Cortex phase is therefore `Lane: local-dev`, always; the field exists so +the batch conductor's lane filter reads one vocabulary across both surfaces. + +## Hard rules + +1. **Never rewrite history on any branch with a remote** (block below). +2. **Pull before edit.** `git fetch && git status` first, every time. +3. **Never edit a ruling.** Supersede it. `ledger_merge.py` refuses to + auto-merge a modified or deleted file under `rulings/`. +4. **Run `python3 scripts/cortex.py check` before you push** a phase, ruling + or batch change; `ledger_merge.yml` runs it on the trial-merge tree and a + failing check leaves the branch for a human. +5. **No hand-written HTML.** Packets are rendered by the batch conductor + (phase 2 of the birth epic); until then a packet is whatever the session + produced, archived unchanged. + + +## Remote sessions (Claude Code on web and mobile) + +Three facts, all measured in a web/mobile container, where this file is loaded +and little else is. They ride in every organ because a session may hold any +subset of them — and the session that needs this most is the one holding +several, which is exactly the session no hook fires in. + +- **Bootstrap in the first turn, unconditionally** — before the first test + command, not as a remedy once something looks wrong: + + ``` + bash PyAutoMind/scripts/session_bootstrap.sh # ~10s cold, ~1s warm + bash PyAutoMind/scripts/session_bootstrap.sh --check # report only + ``` + + A session holding several organs registers no SessionStart hook — Claude Code + reads project hooks from the project directory, which in that layout is the + repos' *parent*, not a repo — so nothing has set this session up. It was once + phrased as a remedy keyed to `No module named pytest` or collection + `ImportError`s naming `yaml`; that symptom stopped appearing when the + container image moved to Python 3.12, while the environment is still wrong in + ways that read like a bad command rather than a stale session (`pytest -n + auto` → `unrecognized arguments: -n`). The bootstrap also **unshallows the + clones**: a remote session clones shallow, and `git merge-base --is-ancestor` + then answers "not an ancestor" for a commit whose ancestry is merely absent — + the answer the ship and close-out procedures act on when proving a branch + merged. + +- **Then run the suite in parallel.** 4 cores, subprocess-heavy suites, no + single slow test: about 3.5x. `python3 -m pytest -q -n auto`, with + `pytest-xdist` supplied by the bootstrap above. + +- **There is no `gh`, and installing one does not help.** A remote session + reaches GitHub through the `mcp__github__*` tools, already scoped to the + session's repos. `gh` installs in two seconds and is a trap: it authenticates, + then 403s every repo-scoped call, because the egress proxy serves neither the + REST repo paths nor GraphQL beyond a pinned set of PR-review operations — a + binary that looks healthy and fails everything that matters. It also defeats + the surface probe, which keys off `gh auth status`. Read + `PyAutoBrain/skills/GITHUB_ACCESS.md` at the top of any run that touches + GitHub; it maps each `gh` operation onto its MCP tool. Spell that path from + the workspace root, as written: a multi-organ session is cwd'd at the repos' + *parent*, so a bare `skills/…` reads as a missing file rather than a missing + repo prefix. + + +## When in doubt + +Read [README.md](README.md) and [REFERENCE.md](REFERENCE.md). The birth epic's +ledger is `PyAutoMind/draft/feature/pyautocortex/cortex_birth_epic.md`. + + +## Never rewrite history + +Never rewrite pushed history on any repo with a remote — no `git init` over a +tracked repo, no force-push to `main`, no fresh-start "Initial commit", no +`filter-repo` / `filter-branch` / `rebase -i` on pushed branches. To get a +clean tree: `git fetch origin && git reset --hard origin/main && git clean -fd`. + diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..81b22d4 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,5 @@ +@AGENTS.md + + diff --git a/README.md b/README.md index 42139a7..152ed1f 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,39 @@ # PyAutoCortex **PyAutoCortex is the Cortex of the PyAutoScientist** — where the organism -learns what is true: the science body map (`projects.yaml`) and the rulings of -record for every science run. The Mind decides what to build, the Brain routes -the work and executes nothing, the Cortex learns what is true. The skeleton -lands in the first PR. +learns what is true. The Mind decides what to build, the Brain routes the work +and executes nothing, the Cortex learns what is true: it holds the **science +body map** (every science project, where it lives, how it syncs) and the +**rulings of record** — every verdict a human has passed on a science run, +append-only, superseded but never edited. + +## How PyAutoCortex works + +The unit is a **phase** of a **project**, which spawns **runs** (SLURM job ids) +and ends in a **ruling**: + +1. **Declare the phase.** A markdown file under `phases//` states the + question, the pre-registered witness, the wall budget and the GitHub issues + it waits on (`Gates:`). It starts `planned` or `gated`. +2. **Run it.** When the gates clear the phase is `ready`; a submission makes + it `submitted` and then `running`, with every job id recorded in the file's + `## Runs` block in SLURM notation. +3. **Pull and review.** When the results are pulled to the laptop the phase + joins the **rolling review board** (`batches/`); a member joins when its + results are in hand, never mid-flight. +4. **Rule.** The human's verdict — `accept`, `rerun`, `drop` or + `leave-to-finish` — is written into `rulings/` with a permanent id. *A + verdict recorded only outside the Cortex does not exist.* + +The Cortex is **not a second PyAutoMind**: development tasks stay in the Mind +and are named here only as gates. Its one script, `scripts/cortex.py`, checks +the tree and moves phases between states; the ledger folders auto-merge from +any surface that can push. + +The schemas — phase files, the run-line grammar, rulings, batches, the +`projects.yaml` subset — are in [REFERENCE.md](REFERENCE.md); how agents should +operate this repo is in [AGENTS.md](AGENTS.md); every design choice the birth +epic did not fix is dated in [docs/schema_decisions.md](docs/schema_decisions.md). +The organism this repo is the Cortex of is described once in +[PyAutoBrain/ORGANISM.md](https://github.com/PyAutoLabs/PyAutoBrain/blob/main/ORGANISM.md) +and documented in full at . diff --git a/REFERENCE.md b/REFERENCE.md new file mode 100644 index 0000000..04a4248 --- /dev/null +++ b/REFERENCE.md @@ -0,0 +1,484 @@ +# PyAutoCortex reference + +The schemas, grammars and conventions for this repo — the file `scripts/cortex.py` +implements and `scripts/cortex.py check` enforces. Agent docs that point at a +schema ("the transition table", "the run-line grammar", "the ruling schema") +resolve here, one link from [AGENTS.md](AGENTS.md). + +--- + +## What a phase looks like + +Here is a phase file — `phases/example/05_running_array.md` from the test +fixture — of a project whose GPU array is most of the way through: + +````markdown +# Example — phase 5: the nine-lens array + +Project: example +Phase: 5 +State: running +Gates: PyAutoArray#431 +Gates-cleared: 2026-08-29 +Witness: nine of ten array tasks write a sane checkpoint.hdf5 within 8:00 wall +Budget: 8:00 +Runs: 342091, 342102 +Ruling: R-20260901-03 +Lane: local-dev +Review-minutes: 8 +Epic: example-programme +Filed: 2026-08-29 + +## Question + +Does the Delaunay pipeline reach the same theta_E basin on all ten lenses? + +## Witness + +`output/phase_05/*/checkpoint.hdf5` present for nine lenses, `.err` clean. + +## Where to look + +- `/mnt/c/Users/Jammy/Science/example/output/phase_05/` + +## Runs + +- 342091_[0-8,10]: done — gpu — submitted 2026-08-30 — wall 6:12 + pulled_to: /mnt/c/Users/Jammy/Science/example/output/phase_05 +- 342091_9: failed — gpu — submitted 2026-08-30 — wall 0:00 — OOM before the first step +- 342102: running — gpu — submitted 2026-09-01 — wall 0:00 — task 9 resubmitted alone + after: 342091_9 + +## Ruling + +R-20260901-03 — leave-to-finish +```` + +The header is the Mind's **light header**: line 1 is `# `; `Key: value` +lines within the first 30 lines — the block starts at the first `Key: value` +line after the title and ends at the next blank line; the first occurrence of a +key wins; a list-valued key is a bare `Key:` line followed by `- item` lines; a +key with no value (`Gates:`, `Runs:`, `Ruling:`) may be left empty or omitted. +No YAML frontmatter. The body +sections are fixed: `## Question`, `## Witness`, `## Where to look`, `## Runs`, +`## Ruling`. + +### Phase header keys + +| Key | Value | Notes | +|---|---|---| +| `Project:` | project key | must equal the directory name **and** a `projects.yaml` key | +| `Phase:` | integer | unique per project; revival of a dropped phase is a new number | +| `State:` | `planned \| gated \| ready \| submitted \| running \| pulled \| awaiting-ruling \| accepted \| rerun \| dropped` | | +| `Gates:` | comma-separated GitHub refs | `Repo#N` (owner `PyAutoLabs`) or an issue/PR URL; **no `owner/Repo#N`** form | +| `Gates-cleared:` | `YYYY-MM-DD` | written by `gates --grade --write` when every ref cleared | +| `Gate-override:` | reason | written by `move --override "<reason>"` | +| `Witness:` | free text | **mandatory before `submitted`** — the pre-registered checkable claim | +| `Budget:` | `H+:MM` | wall budget per run | +| `Runs:` | comma-separated job **stems** | the index of the `## Runs` body; equal to the set of body stems | +| `Ruling:` | ruling id | the chain head (see "Rulings") | +| `Lane:` | `local-dev` | always | +| `Review-minutes:` | integer | a seed, not a measurement | +| `Epic:` | slug | shared with the Mind — the join key across the two dashboards | +| `Filed:` | `YYYY-MM-DD` | | + +Gate refs are matched by `GATE_REF_RE`, copied verbatim from +`PyAutoMind/scripts/lifecycle.py`: + +```python +GATE_REF_RE = re.compile( + r"https://github\.com/([\w.-]+)/([\w.-]+)/(?:issues|pull)/(\d+)" + r"|(?<![\w/])([A-Za-z_][\w.]*)#(\d+)\b" +) +DEFAULT_GATE_OWNER = "PyAutoLabs" +``` + +The lookbehind `(?<![\w/])` is what rejects `owner/Repo#N`: the shorthand is +`Repo#N` with the default owner, and any other owner is spelled as a URL. + +--- + +## How a phase flows + +``` + planned ──(Gates: non-empty)──► gated ──(gates --grade --write | move --override)──► ready + │ ▲ │ + └──────(Gates: empty)────────────┼──────────────────────────────────────────────────┤ + │ (a cleared gate reopened; --write only) │ + └──────────────────────────────────────────────────┤ + ▼ + ready ──(Witness: + --run)──► submitted ──► running ──(no live run)──► pulled ──► awaiting-ruling + ▲ │ │ │ + │ (no live run, ≥1 failed|timeout|void, --reason) │ rule + └──────────────────────────────┴──────────────┘ ▼ + accepted │ rerun ──► ready │ dropped + │ rule --supersedes + └──► rerun | dropped +``` + +`cortex.py move` owns every edge except the ruling edges, which `cortex.py rule` +owns. The full table: + +| from | to | condition | +|---|---|---| +| planned | gated / ready | `Gates:` non-empty / empty | +| gated | ready | `gates --grade --write` (all refs cleared → writes `Gates-cleared:`) or `move --override "<reason>"` (→ writes `Gate-override:`) | +| ready | gated | `gates --grade --write` only, when a cleared gate reopened; refused if `Gate-override:` present; for `submitted`+ phases a reopened gate is *reported*, never enforced | +| ready | submitted | `Witness:` non-empty AND `--run <id>` supplied | +| ready | pulled | legacy-born phase: every run line is `legacy\|legacy_wrong` (`new --legacy-run` / `move pulled`); `Witness:` still mandatory | +| submitted / running | same | `--run <id>` appends a wave, a chained job or a checkpoint resubmit (`resumes:`); state unchanged | +| submitted | running | — | +| submitted / running | ready | no run line in `submitted\|running` AND ≥1 `failed\|timeout\|void`; `--reason` required | +| running | pulled | no run line live; or `--partial` (a partial array), which `check` expects closed by a `leave-to-finish` ruling | +| pulled | awaiting-ruling | — (the phase joins the rolling board) | +| awaiting-ruling | accepted / rerun / dropped | **`rule` only** | +| running / pulled / awaiting-ruling | same | `rule leave-to-finish` (state unchanged) | +| accepted | rerun / dropped | `rule --supersedes <current Ruling:>` only (the REWIND case) | +| rerun | ready | — (the witness may be re-registered; run history is kept) | +| any non-terminal | dropped | `rule drop` only | +| dropped | — | terminal (revival = a new phase number) | + +`accepted` is **not** terminal: a later ruling may supersede the acceptance +(2026-08-31's REWIND superseded accepted gates). `dropped` is. + +Offline invariants `check` enforces on top of the table: + +- `State ∈ ready..accepted` AND `Gates:` non-empty ⇒ `Gates-cleared:` or + `Gate-override:` present (`ready..accepted` = every state from `ready` + through `accepted` in the order listed above, `rerun` included). +- `State ∈ submitted..accepted` (and `rerun`) ⇒ `Witness:` non-empty. +- `State ∈ accepted | rerun | dropped` ⇒ `Ruling:` present (those states are + reachable only through `rule`). +- `State = pulled` AND any run line `submitted | running` ⇒ `Ruling:` present + and its head verb is `leave-to-finish` (the `--partial` case). + +--- + +## Repository layout + +``` +PyAutoCortex/ +├── README.md ← short front page +├── AGENTS.md ← agent guidance (generated blocks: organism map, remote sessions, history) +├── CLAUDE.md ← GENERATED pointer (@AGENTS.md) — repos_sync --write +├── REFERENCE.md ← this file (schemas + grammars) +├── LICENSE .gitignore +│ +├── projects.yaml ← the science body map — CODE, not ledger (restricted YAML subset) +├── epics.md ← the Cortex half of each split epic (`- mind-half:`) +│ +├── phases/<project>/<slug>.md ← one file per phase (LEDGER) +├── rulings/AGENTS.md ← the append-only rule +├── rulings/<YYYY>/<MM>/R-<YYYYMMDD>-<nn>.md ← the ledger of record (LEDGER, append-only) +├── batches/AGENTS.md ← batch-record schema (rolling board) +├── batches/<YYYY-MM-DD>-<slot>.md ← one record per slot (LEDGER) +├── batches/packets/{AGENTS.md,TEMPLATE.md,<slot>.html} +├── batches/reviews/{AGENTS.md,<slot>.md} +│ +├── docs/schema_decisions.md ← every dated choice the epic did not fix +├── policy/never_rewrite_history.md policy/remote_sessions.md ← copies of the Mind's; spliced into AGENTS.md +│ +├── scripts/cortex.py ← check · gates · rule · move · new (stdlib only) +├── scripts/ledger_merge.py ← the default-deny ledger classifier (+ append-only on rulings/) +├── tests/test_cortex.py tests/test_ledger_merge.py +├── tests/fixtures/skeleton/ ← one project, one phase per state, five rulings, one batch — the witness +├── tests/fixtures/empty/ ← an empty map passes `check` +│ +├── .github/workflows/cortex_check.yml ← check + pytest on push/PR +├── .github/workflows/ledger_merge.yml ← lands ledger-only `claude/**` branches +└── .claude/hooks/session-start.sh .claude/settings.json ← GENERATED — repos_sync --write +``` + +No `skills/` — the Cortex exposes no commands of its own in phase 1; the +conductor that drives it lives in the Brain (phase 2). + +--- + +## Run lines (`## Runs`) + +SLURM notation, strict grammar. One line per job or per task set of an array; +structured facts on **indented continuation lines**, never in the note. + +``` +- <stem>[_<task>|_[<a>-<b>,<c>]]: <run-state> — <partition> — submitted <YYYY-MM-DD> — wall <H+:MM>[ — <note>] + pulled_to: <path> | after: <run> | resumes: <run> | where: <path> | ruled: <id> +``` + +- `<stem>` — the SLURM job id, digits. `_<task>` — one array task. + `_[<set>]` — an array task set: comma-separated integers and `a-b` ranges, + ascending, no spaces (`342091_[0-8,10]`). +- `<run-state>` ∈ `submitted | running | done | failed | timeout | void | + legacy | legacy_wrong`. `void` = cancelled or never produced a step. + `legacy` / `legacy_wrong` = quarantine (reusable / not) — **a run state, + never a phase state**. +- `<partition>` — the SLURM partition the job went to, a bare word + (`^[a-z][a-z0-9_-]*$`, e.g. `gpu`). The project's `partition:` row says + which it may use. +- `submitted <YYYY-MM-DD>` — the submission date. `wall <H+:MM>` — wall time + used so far (`0:00` for a job that never ran). +- `<note>` — free text after a fourth ` — `; never carries a structured fact. +- Continuation lines — exactly four spaces then `<key>: <value>`, keys: + `pulled_to:` (laptop path the results were pulled to), `after:` (SLURM + `afterok` dependency), `resumes:` (a checkpoint resubmit of that run), + `where:` (the quarantine path of a `legacy*` run), `ruled:` (a ruling id + naming this run). +- The em dash `—` is canonical; `--` is accepted for it (phone keyboards). + `check` reads both as the same separator; the writing verbs emit `—`. + +The regex, as `check` applies it (after `--` → `—` normalisation): + +```python +RUN_LINE_RE = re.compile( + r"^- (?P<stem>\d+)" + r"(?:_(?P<task>\d+)|_\[(?P<tasks>\d+(?:-\d+)?(?:,\d+(?:-\d+)?)*)\])?" + r": (?P<state>submitted|running|done|failed|timeout|void|legacy|legacy_wrong)" + r" — (?P<partition>[a-z][a-z0-9_-]*)" + r" — submitted (?P<date>\d{4}-\d{2}-\d{2})" + r" — wall (?P<wall>\d+:\d{2})" + r"(?: — (?P<note>.+))?$" +) +RUN_CONT_RE = re.compile( + r"^ (?P<key>pulled_to|after|resumes|where|ruled): (?P<value>\S.*)$" +) +``` + +`check` enforces: + +- every non-blank line under `## Runs` matches `RUN_LINE_RE` or `RUN_CONT_RE`; + a continuation line follows a run line; +- task sets on one stem are **disjoint within a phase** (job ids are unique + per phase, not globally — one array may feed two phases); +- the `Runs:` header equals the set of body stems (both empty when the phase + has no runs); +- `State: pulled` ⇒ at least one `done | legacy` run carries `pulled_to:`; +- a `legacy | legacy_wrong` run carries `where:`; +- an `after:` / `resumes:` target is the identifier (the text before the + colon — `342091`, `342091_9` or `342091_[0-8,10]`) of another run line of + the **same phase**; +- a `ruled:` value resolves to a ruling file. + +--- + +## Rulings + +`rulings/<YYYY>/<MM>/R-<YYYYMMDD>-<nn>.md`. The id is `R-YYYYMMDD-nn` — a +two-digit per-day sequence, global across projects; `rule` assigns it; the +filename equals the id and so does the title line (`# <id>` or +`# <id> — <one-line summary>`). + +````markdown +# R-20260901-02 — re-accept phase 8 with the corrected evidence pointer + +Project: example +Phase: phases/example/08_accepted.md +Runs: 342050 +Ruling: accept +Supersedes: R-20260901-01 +Batch: 2026-09-01-pm +Reviewed-at: 2026-09-01T17:20Z +Review-minutes-actual: 6 +Follow-ups: PyAutoLens#901 + +## Ruling + +The human's words, verbatim. + +## Evidence + +- pointers into the pulled results, the project ledger, figures +```` + +| Key | Value | Notes | +|---|---|---| +| `Project:` | project key | | +| `Phase:` | one phase path | repo-relative, `phases/<project>/<slug>.md` | +| `Runs:` | comma-separated stems, or empty | ⊆ the phase's `Runs:` | +| `Ruling:` | `accept \| rerun \| drop \| leave-to-finish` | the verb | +| `Supersedes:` | one ruling id | optional; see the chain rules | +| `Batch:` | `<YYYY-MM-DD>-<slot>` | optional; the join for rulings filed together | +| `Reviewed-at:` | timestamp | | +| `Review-minutes-actual:` | integer | | +| `Follow-ups:` | comma-separated GitHub refs | `GATE_REF_RE`; **the issue is created before `rule` runs** | + +Body: `## Ruling` — the human's words verbatim; `## Evidence` — pointers. + +**One ruling file per phase.** A multi-phase ruling (a REWIND) is N files with +the same body and one `Batch:`; `rule --also <phase>` fans out. + +**Chain rules** (`check`): + +- the id is unique, equals the filename stem and the title; +- `Supersedes:` resolves to an existing ruling, is not the ruling itself, is + lexically smaller (earlier), and names the same project **and** phase; +- **at most one successor per ruling** — a chain, not a tree: supersede the + head; +- the phase's `Ruling:` is a chain head (no ruling supersedes it) whose + `Phase:` is that phase; +- the head's verb matches the phase's state: `accept ⇒ accepted`, + `drop ⇒ dropped`, `rerun ⇒ rerun | ready | submitted | running | pulled | + awaiting-ruling` (the phase has moved on), `leave-to-finish ⇒ any + non-terminal state`; +- the ruling's `Runs:` ⊆ the phase's runs; +- `Batch:`, when present, names an existing `batches/<slot>.md`. + +A phase may hold rulings that are not chained to each other — a +`leave-to-finish` followed by an `accept` is two chains of one. `Supersedes:` +is for replacing a verdict, not for sequencing. + +The rest of the rule — append-only, supersede-never-edit, *a verdict recorded +only outside the Cortex does not exist* — is [rulings/AGENTS.md](rulings/AGENTS.md). + +--- + +## Batches — the rolling board + +One record per slot, `batches/<YYYY-MM-DD>-<slot>.md`, plus the archived +packet and the human's verbatim review. Full schema and the fields that are +easy to get wrong: [batches/AGENTS.md](batches/AGENTS.md). + +Member line: + +``` + - <slug>: <phase path> — <runs> — <review-minutes> — <state> +``` + +`<slug>` is the phase file's stem; `<runs>` is comma-separated stems or +`none`; `<state>` is the phase state at the last refresh. `check` verifies +the path exists, the slug matches its stem, the runs ⊆ the phase's `Runs:` +and the state is legal. + +**One review grammar** (`batches/reviews/<slot>.md`): + +```markdown +# Batch review <YYYY-MM-DD>-<slot> + +- packet: batches/packets/<YYYY-MM-DD>-<slot>.html +- reviewed-at: <free text> +- review-minutes-actual: <integer or (not given)> + +## <slug> — <HEALTH> +- decision: accept|rerun|drop|leave-to-finish|(none) +- ruled: yes|no + +<note verbatim, or (no note)> +``` + +`<HEALTH>` ∈ `HEALTHY | SUSPECT | FAILED | RUNNING`. `(none)` with +`ruled: no` is a member the human left untouched — listed, never silently +dropped. `ruled: yes` requires a verb; a verb with `ruled: no` is a leaning the +human did not tick. The four verbs are the whole vocabulary; +the Mind's `merge | tweak | reject | defer` is the dev surface's and does not +appear here. + +--- + +## The restricted YAML subset (`projects.yaml`) + +Real YAML, parsed with the stdlib by a ~40-line line loop. The subset: + +``` +<key>: # column 0, ^[a-z][a-z0-9_]*$ + <field>: <scalar> # exactly two spaces; fixed field set, unknown field = error + sync_verbs: [pull, push] # flow list of bare words — the only list +``` + +- Scalars are bare unless they contain `#`, `:` or edge spaces, then + `"..."` with no escapes. +- Comments on their own line or after ` #`. Blank lines anywhere. +- No block lists, no nesting beyond the two levels, no anchors, no `---`, no + quoted keys. +- Fields (all required): `remote` (`owner/repo` | `none`), `local_path` + (absolute laptop path — the Cortex-only exception to the workspace-paths + rule), `ral_root`, `mirror` (path | `none`), `sync_cli`, `sync_verbs`, + `ledger`, `witness_file` (glob), `partition` (`gpu | ral | both`), + `status` (`active | dormant`). +- A file with no rows parses to an empty map (`{}`); PyYAML returns `None` + for a comment-only document, so the `yaml.safe_load(text) == parse(text)` + parity test runs on the fixture, which has a row. + +--- + +## How the ledger lands (`ledger_merge.yml`) + +A push to a `claude/**` branch whose whole diff is **ledger** is merged into +`main` by `.github/workflows/ledger_merge.yml` and the branch deleted — no PR, +no session step. The line is drawn by `scripts/ledger_merge.py`, **default +deny**: + +| Ledger — merged automatically | Code — always a human | +|---|---| +| `phases/**`, `rulings/**`, `batches/**` | `scripts/`, `tests/`, `.github/`, `policy/`, `docs/` | +| `epics.md` | `projects.yaml`, `README.md`, `AGENTS.md`, `REFERENCE.md`, … | +| | anything unclassified — a new root file, a new top-level folder | +| | **any modification or deletion under `rulings/`** (append-only) | + +Two exceptions inside the ledger dirs, as in the Mind: a **dot-path** anywhere, +and a file pytest would **collect** (`conftest.py`, `test_*.py`, `*_test.py`). +The Cortex adds a third: `ledger_merge.py` classifies via `git diff +--name-status`, and an `M`, `D` or `R` entry under `rulings/**` is code. + +The blocking check is `python3 scripts/cortex.py check` run on the **trial-merge +tree** — after `git merge --no-ff`, before `git push` — which is what catches +the ruling-id race (two branches both assign `R-…-03`: each passes alone, the +merge fails). A failing check resets the trial merge and leaves the branch for +a human. Predict the verdict before you push: + +```bash +python3 scripts/ledger_merge.py classify --base origin/main # exit 0 = will auto-merge +python3 scripts/cortex.py check # exit 0 = will not block +``` + +--- + +## `scripts/cortex.py` — the verb reference + +Stdlib only; `main(argv)`; no import-time side effects; every verb takes +`--root <dir>` (default: the repo root the script lives in). Every leg of +`check` takes `root: Path`, so tests run it against a `tmp_path` copy. + +- **`check`** — every rule in this file: phase headers and states, the gates + invariant, the witness invariant, run lines, ruling ids and chains, the + verb↔state agreement, batch member lines and review files, every project + named by a phase path is a `projects.yaml` key, the `projects.yaml` subset + itself. Hermetic (no network, no git). Output in `lifecycle.py`'s shape — + `cortex check: OK` or `cortex check: DRIFT` followed by one ` - …` line per + finding; exit 1 on drift. +- **`gates [--grade [--write]]`** — offline: list every `gated` phase and its + refs. `--grade`: per ref, + `gh api repos/<o>/<r>/issues/<n> --jq '{state, state_reason, merged_at: .pull_request.merged_at, is_pr: (.pull_request != null)}'` + with a stdlib `urllib` fallback (User-Agent `pyautocortex`) when `gh` is + absent. **Cleared** = PR ⇒ `merged_at != null` (closed-unmerged is a dead + gate, reported); issue ⇒ `state == closed` and `state_reason ∈ {completed, + null}` (`not_planned` / `duplicate` are dead gates); anything unreadable + fails closed. `--grade` reports only; `--write` flips `gated → ready` + (writing `Gates-cleared:`) and `ready → gated` when a cleared gate reopened + (refused when `Gate-override:` is present; for `submitted`+ phases a + reopened gate is reported, never enforced). +- **`rule <phase> <verb> --body <file> [--supersedes <id>] [--batch <slot>] + [--minutes n] [--follow-up <ref>]... [--also <phase>]...`** — assigns the + next id for today, writes the ruling file(s) (one per phase; `--also` fans + out with the same body and batch), updates the phase's `Ruling:` and + `State:` per the table; refuses to touch an existing ruling; refuses a verb + the table does not allow from the phase's state. +- **`move <phase> <state> [--run <id>] [--reason ..] [--override ..] + [--partial]`** — the table; refuses every ruling edge with a message naming + `rule`. `--run` on `submitted`/`running` appends a run line and keeps the + state. +- **`new <project> <slug> --phase <n> [--gates ..] [--epic ..] + [--legacy-run <id>]...`** — writes `phases/<project>/<slug>.md` from the + template in `planned` (or `ready` when `--legacy-run` is given and every run + is legacy); refuses a duplicate phase number or an unknown project. + +--- + +## Bootstrap + +```bash +cd ~/Code/PyAutoLabs +git clone https://github.com/PyAutoLabs/PyAutoCortex.git +python3 PyAutoMind/scripts/repos_sync.py --write # CLAUDE.md, .claude/, the AGENTS.md blocks +python3 PyAutoCortex/scripts/cortex.py check # OK on an empty tree +``` + +The checkout directory must be named `PyAutoCortex`; the organism's sync and +drift scripts address it by that name. diff --git a/batches/AGENTS.md b/batches/AGENTS.md new file mode 100644 index 0000000..60aa66e --- /dev/null +++ b/batches/AGENTS.md @@ -0,0 +1,82 @@ +# Batch records — the rolling board + +One file per review slot: `<YYYY-MM-DD>-<slot>.md`. This is the ledger of what +was on the board for one slot and what the human ruled — the same genre as the +Mind's batch records, and the evidence base the review-minute estimate is +calibrated from. It is the Mind's schema with what a science batch changes. + +It auto-merges (`scripts/ledger_merge.py` — `batches/` is a ledger dir), +because an unattended system that cannot record its own history unattended +will not record it. + +## A rolling board, not a dispatch + +A dev batch is dispatched at once and reviewed at once. A Cortex batch is a +**live board**: a phase **joins** the review when its results are `pulled`; +nothing in `submitted` or `running` holds review control — those members are +shown as running, and a `leave-to-finish` is the only thing the human can say +about them; the board is the live view of run progress. Each pull appends a +`refreshed:` line — that list is the record of the board filling in. + +## Schema + +```markdown +# Batch 2026-09-01 pm +- dispatched: 2026-09-01T09:00Z # when the board was opened +- review-at: 2026-09-01T17:00Z # stated by the human AT DISPATCH — the shift is dispatch → review-at +- shift: day # free-text label the human gives it +- lane: local-dev # always — the review happens at the laptop +- review-minutes-planned: 20 +- members: + - <slug>: <phase path> — <runs> — <review-minutes> — <state> +- refreshed: 2026-09-01T11:40Z — <slug> pulled # one line per pull, appended in order +- refreshed: 2026-09-01T15:05Z — <slug> pulled +- collected: 2026-09-01T17:00Z +- reviewed-at: 2026-09-01T17:10Z # when they actually sat down +- delivered: <n>/<n> +- packet: batches/packets/<YYYY-MM-DD>-<slot>.html +- review: batches/reviews/<YYYY-MM-DD>-<slot>.md +- review-minutes-actual: <n> +- notes: | + What actually happened. Anything that stalled, and why. +``` + +The member line is `<slug>: <phase path> — <runs> — <review-minutes> — +<state>`: the phase file's stem, its repo-relative path, the comma-separated +run stems (or `none`), the review-minute seed, and the phase's state at the +last refresh. `cortex.py check` verifies every member line against its phase +file. + +Dropped from the Mind's schema, on purpose: `usage-window-*` (no cloud +sessions are spent by a science batch), `heart-ack:` (the Heart gates +releases, not runs) and `expected-effects:` (the licence for an autonomous +YELLOW acknowledgement; a science batch has no autonomy leg to license). + +## The fields that are easy to get wrong + +**`delivered:` is not `sacct COMPLETED`.** SLURM's verdict says the process +exited; it does not say the run produced science. A member counts as delivered +only when **all four** hold: the `.err` file is clean, wall < `Budget:`, the +output carries the version stamp of the installed stack, and +`checkpoint.hdf5` is sane (opens, holds the expected step count). A run that +`COMPLETED` with an empty checkpoint is **not delivered**, loudly, at the top +of the packet. + +**`review-at:` is the shift, and it is the human's to declare.** There is no +schedule: a slot is whenever they come in, so at dispatch they state when they +expect to be back and the shift is dispatch → `review-at:`. `reviewed-at:` is +the same number measured rather than promised. Per Cortex batch, separately +from any dev batch open the same day: the two surfaces have their own records +and their own `review-at:`. + +**`refreshed:` is the board's history.** A member joins on the pull that +fills its results in; the line says when and which. A packet refreshed without +the line is a board whose history cannot be audited. + +**`review-minutes-actual:` is the only calibration there is.** The planned +figure is a seed; this is what the slot really cost. + +**`packet:` is the page the human actually reviewed, archived** — see +`packets/AGENTS.md`. **`review:` is what the human said, verbatim** — see +`reviews/AGENTS.md`; every ruling `cortex.py rule --batch <slot>` files +traces back to a line in that file. diff --git a/batches/packets/AGENTS.md b/batches/packets/AGENTS.md new file mode 100644 index 0000000..2bd3e4e --- /dev/null +++ b/batches/packets/AGENTS.md @@ -0,0 +1,35 @@ +# Batch packets — the review pages, archived + +One **self-contained** HTML page per slot: `<YYYY-MM-DD>-<slot>.html`. This is +the page the human actually opens to review a board — every member with its +question, witness, health evidence, readout, ruling block, follow-ups, local +"where to look" pointers and figures — and, once the slot is over, the +permanent record of what they were shown. The batch record's `packet:` field +points here. + +Rules: + +- **Self-contained means self-contained.** Figures are embedded as data URIs, + styles and script are inline; the archive must render identically in ten + years with no external fetch. Keep a page under ~5 MB; downsize figures. +- **A rolling board.** The page opens with every phase on the board — the + `submitted` / `running` ones as RUNNING entries with no review control — + and carries `generated:` / `refreshed:` stamps. Each pull regenerates the + member that just joined **in place** (same `id`) and bumps the `refreshed` + stamp; the batch record gets the matching `refreshed:` line. +- **Never rewritten after the review is submitted.** The archived page plus + the `batches/reviews/` file together are the audit pair: what was shown, + what was ruled. A correction gets a new dated page, not an edit. +- **Pointers are local paths** (the laptop mirror each project's sync CLI + fills — the `mirror:` of its `projects.yaml` row), because the review + happens at the laptop. A pointer may stay remote only when the pull cannot + fetch it by design, and must say so. +- **Not hand-written.** The renderer is the batch conductor's (phase 2 of the + birth epic). Until it exists a packet is whatever the session produced, + archived unchanged. + +**Visibility.** Packets and reviews are public in this repository, as the +Mind's are (ruled 2026-08-31): science numbers, figures and run paths in a +packet are thereby on the open internet. A project whose data may not be +shown (Euclid) keeps its figures out of the packet and points at the local +path instead. diff --git a/batches/packets/TEMPLATE.md b/batches/packets/TEMPLATE.md new file mode 100644 index 0000000..850ee0d --- /dev/null +++ b/batches/packets/TEMPLATE.md @@ -0,0 +1,95 @@ +# Review-packet page template + +Spec for emitting a board-review page (the surface the batch conductor's +collect — or the session driving one — hands the human). The Mind's reference +implementation, `PyAutoMind/batches/packets/2026-08-31-am.html`, is the visual +model; the member block and the submit grammar below are the Cortex's. + +## File + +- One self-contained HTML file: `PyAutoCortex/batches/packets/<YYYY-MM-DD>-<slot>.html` + (`<slot>` is the human's free-text label, e.g. `am`, `pm`, `night`). +- A **complete document**: `<!DOCTYPE html>`, `<html lang>`, `<head>` with charset + viewport, + `<body>`; the stylesheet carries its own `[hidden] { display: none !important; }` reset. A + fragment relying on a host wrapper renders in quirks mode when served raw. +- Figures embedded as base64 data URIs, ≤ 720 px wide, JPEG q80; page ≤ 3 MB. +- The archive URL/path must stay **stable across refreshes** — the page's notes and decisions live + in `localStorage` keyed by origin+page, and a moved file orphans them. + +## Page structure, in order + +1. **Header** — eyebrow (date · lane · what is on the board), title, one-sentence lede, + mirror-note (where run outputs were pulled locally), stamp line + (`Generated <ts> · refreshed <ts>`), `Permanent home: <code>path</code>`. +2. **Stat tiles** — members by health + `Est. review-minutes <sum>` with the addition shown. +3. **Inputs bar** — `Reviewed at` (text, auto-filled, editable) and `Review-minutes-actual` + (number). Both persist; both go into the submit markdown. +4. **Most-important-finding callout.** +5. **Rulings needed** — a real `<ol>`, one line per decision, each linking to its member. +6. **Members** (sticky sidenav beside them), ordered: failures → rulings-required → clean → + running. Cross-cutting sections and a closed `<details>` of packet-building notes follow. +7. **Fixed submit bar** — live progress (`Ruled r of N · decisions d of N`) + `Submit review`. + +## Member block order + +Question · Witness · Health evidence (`.err` clean · wall vs budget · version stamp · +`checkpoint.hdf5` sane — the four `delivered:` legs, each shown) · Readout (tables in +`overflow-x:auto` wrappers; figures) · **Ruling** (accent border, the one-line decision) · +**Your review** (see below) · Follow-ups (`[repo]`-tagged) · Where to look yourself (`<dl>`; +local mirror paths; `RAL only` tag **only** when the artefact genuinely cannot be mirrored) · +Est. review-minutes chip. + +**Your review** = radio chips `Accept / Rerun / Drop / Leave to finish` — the four verbs and +nothing else; a RUNNING member shows only `Leave to finish` — + one auto-growing textarea. All +state (chips, notes, Ruled ticks, the two top inputs) persists to `localStorage` under one +versioned key, every access in try/catch, saves debounced. + +## RUNNING members (the rolling board) + +A member whose phase is `submitted` or `running` is emitted as +`<section class="member running" id="…">` with a `chip-running` chip reading +`RUNNING — joins the board when pulled`, showing only Question, Witness and a status line +(the run lines). The pull that fills it in **regenerates that section in place** (same `id`, +full block set), bumps the header's `refreshed` stamp, and republishes to the **same path** — +never a new file, so stored notes survive. Never touch other members' markup during a refresh. + +## Submit markdown schema (parse-stable) + +The submit button assembles exactly: + +```markdown +# Batch review <YYYY-MM-DD>-<slot> + +- packet: batches/packets/<YYYY-MM-DD>-<slot>.html +- reviewed-at: <free text> +- review-minutes-actual: <integer or (not given)> + +## <member-slug> — <HEALTH> +- decision: accept|rerun|drop|leave-to-finish|(none) +- ruled: yes|no + +<note verbatim, or (no note)> +``` + +One `## <member-slug> — <HEALTH>` section per member, in page order; the slug is the phase +file's stem (embed them in the page's `MEMBERS` JS array); `<HEALTH>` ∈ `HEALTHY | SUSPECT | +FAILED | RUNNING`. The orchestrator parses on those exact heading forms and the two `- key:` +lines. `(none)` + `ruled: no` is an untouched member. + +Delivery affordances, in this order: **Copy to clipboard** (always works; clipboard API with +`execCommand` fallback), **Download .md** (inert inside a Claude-artifact sandbox — say so in +small print), **Commit on GitHub** — a link to +`https://github.com/PyAutoLabs/PyAutoCortex/new/main?filename=batches/reviews/<slot>.md&value=<urlencoded>`, +disabled with a use-Copy hint when the URL exceeds 7,500 chars. Close with: "tell the +orchestrator chat 'review submitted'." The committed file lands at +`PyAutoCortex/batches/reviews/<YYYY-MM-DD>-<slot>.md`. + +## Design tokens (brief) + +Light on bare `:root`, dark redefined under `@media (prefers-color-scheme: dark)` guarded +`:root:not([data-theme="light"])` and again under `:root[data-theme="dark"]`; body background +always `var(--ground)`. Ground `#eef1f4` / surface white / ink `#17202b` / accent teal `#0f6e73`; +semantic (separate from accent): failed `#b3261e`, suspect `#a65f00`, healthy `#2c7a4b`, +running `#2f5fc4`, each with a `-soft` tint; dark equivalents lifted for contrast. Type: IBM +Plex Serif (headings) / Sans (body) / Mono (paths, ids, table numerals with `tabular-nums`). +No emoji, no decorative numbering; only real sequences get an `<ol>`. diff --git a/batches/reviews/AGENTS.md b/batches/reviews/AGENTS.md new file mode 100644 index 0000000..dc5b1fc --- /dev/null +++ b/batches/reviews/AGENTS.md @@ -0,0 +1,40 @@ +# Batch reviews — what the human ruled, verbatim + +One markdown file per slot: `<YYYY-MM-DD>-<slot>.md` — the review the human +submitted from the packet page (its "submit" step produces exactly this file, +committed via the page's GitHub button or pasted to the orchestrator), or the +same content dictated in-chat and transcribed. The batch record's `review:` +field points here. + +The orchestrator parses this file at close-out: every ruling it files +(`cortex.py rule <phase> <verb> --batch <slot>`) and every follow-up issue it +opens traces back to a line here — not to a memory of a conversation. +Follow-ups are enacted in the **next** slot; a review never executes its own +follow-ups. + +## Schema — the one review grammar + +```markdown +# Batch review 2026-09-01-pm + +- packet: batches/packets/2026-09-01-pm.html +- reviewed-at: 2026-09-01T17:10Z +- review-minutes-actual: 18 + +## <member slug> — <HEALTH> +- decision: accept | rerun | drop | leave-to-finish | (none) +- ruled: yes | no + +The human's words, verbatim, or (no note). For rerun this paragraph becomes +the rerun phase's seed; for drop it is the ruling body. +``` + +`<HEALTH>` ∈ `HEALTHY | SUSPECT | FAILED | RUNNING`. The four verbs are the +**whole** vocabulary — the Mind's `merge | tweak | reject | defer` belongs to +the dev surface and never appears here. Members the human left untouched are +listed with `- decision: (none)` and `- ruled: no` by the orchestrator, never +silently dropped. `cortex.py check` verifies every section names a member of +the slot's batch record and uses the grammar. + +Like every file in `batches/`, this is ledger-side: it auto-merges, because a +review that cannot land unattended cannot close a board unattended. diff --git a/docs/schema_decisions.md b/docs/schema_decisions.md new file mode 100644 index 0000000..c28a423 --- /dev/null +++ b/docs/schema_decisions.md @@ -0,0 +1,123 @@ +# Schema decisions — 2026-09-01 + +Every choice made in phase 1 of the Cortex birth epic (PyAutoMind#379) that +the epic ledger's "The decision" (2026-09-01) did not already fix. Phase 2's +conductor reads these; nobody re-derives them. Each entry: the decision, then +one line of why. The human's fixed inputs — the organ, the state model, rulings +of record (Option A), gates as GitHub refs, the remotes, the epic split, the +rolling board — are in the epic ledger and are not repeated here. + +1. **Ruling id `R-YYYYMMDD-nn`, a two-digit per-day sequence, global.** + Why: date-first sorts the ledger chronologically on disk and the per-day + counter is small enough to assign by listing one directory; global rather + than per-project so one id space is enough to cite from any ledger. +2. **One ruling file per phase; a multi-phase ruling is N files joined by + `Batch:`.** Why: `check` then has one phase per file to validate against, + and the REWIND case (one verdict over five phases) stays expressible via + `rule --also`. +3. **Chains, not trees: at most one successor per ruling; supersede the + head.** Why: "which ruling stands" must have one answer; a tree would need + a tiebreak. +4. **`accepted` is not terminal — a later ruling may supersede the + acceptance (`rule --supersedes`), moving the phase to `rerun` or + `dropped`.** Why: the 2026-08-31 REWIND superseded accepted gates; + a terminal `accepted` would have needed a new phase number for a verdict + about the same runs. +5. **`submitted` / `running` self-loops: `--run <id>` appends a wave, a + chained job or a checkpoint resubmit without changing state.** Why: real + phases have several waves (`342091_[0-8,10]` then `342102`); one state + change per wave would be noise. +6. **`move pulled --partial` on a running phase, which `check` expects + closed by a `leave-to-finish` ruling.** Why: partial arrays are routine; + the board needs the nine finished tasks now and the ruling records that + the tenth is still owed. +7. **Legacy-born phases: `new --legacy-run` writes a phase whose every run is + `legacy | legacy_wrong`, moving `ready → pulled` directly; the witness is + still mandatory.** Why: phase 4 migrates pre-Cortex runs, which were never + submitted through the Cortex but must still be ruled on. +8. **`void` is a run state: cancelled, or never produced a step.** Why: a + job that was killed at submit time is neither `failed` (it never ran) nor + absent (the id exists in `sacct`). +9. **A gate is cleared when: a PR has `merged_at != null`; an issue is + `closed` with `state_reason ∈ {completed, null}`. Closed-unmerged PRs and + `not_planned` / `duplicate` issues are dead gates, reported. Unreadable + refs fail closed.** Why: "closed" alone would clear a gate on a rejected + PR. +10. **`gates --grade` reports; `--write` flips.** Why: the hand tool must be + safe to run from any session; phase 2 schedules the write. +11. **Reopen demotion `ready → gated` is `--write`-only, refused when + `Gate-override:` is present, and for `submitted`+ phases reported but + never enforced.** Why: a run already on the queue is not un-submitted by + a reopened issue; the human overrode on purpose. +12. **`projects.yaml` is code, not ledger, and is written in a restricted + YAML subset parsed by the stdlib.** Why: `sync_cli` and `local_path` are + paths a conductor executes under (worse than `repos.yaml`, which is + already code in the Mind); the subset keeps `cortex.py` stdlib-only while + a PyYAML parity test keeps the file real YAML. +13. **No `owner/Repo#N` gate form.** Why: `GATE_REF_RE`'s lookbehind rejects + it; another owner is spelled as a URL, one grammar shared with the Mind. +14. **Run lines in strict SLURM notation with structured facts on indented + continuation lines (`pulled_to:`, `after:`, `resumes:`, `where:`, + `ruled:`), never in the note.** Why: a fact in prose cannot be checked; + `--` accepted for `—` because phones cannot type the em dash. +15. **Job ids are unique per phase, not globally.** Why: one array fed two + phases in the real ledgers (`342120_[0-4]` / `_[5-9]` in the fixture). +16. **No `skills/` and no `install.sh` change in phase 1.** Why: the Cortex + exposes no commands yet; `install.sh`'s `ORGAN_REPOS` is a hardcoded list + only Brain's `AGENTS.md` uses. +17. **No Mind PR for the session-hook fan-out.** Why: + `session_hook_propagate.yml` reads every `repos.yaml` row and the Cortex + row already exists (phase 0). +18. **The fixture is the witness: `tests/fixtures/skeleton/` holds one + project, one phase per state, five rulings (one chain of two), one batch + record and one review; `tests/fixtures/empty/` passes `check`.** Why: a + schema nobody can instantiate is prose. +19. **`ledger_merge.yml` runs `cortex.py check` on the trial-merge tree, + after `git merge --no-ff` and before `git push`.** Why: that is the only + place two branches that both assigned `R-…-03` are seen together. + +Choices made while writing the files, beyond the plan: + +20. **A header key with no value may be omitted or left empty (`Gates:`, + `Runs:`, `Ruling:`).** Why: `new` writes the empty key so a human sees + the slot; a migrated file may not carry it; both parse the same. +21. **`State ∈ accepted | rerun | dropped` ⇒ `Ruling:` present.** Why: those + states are reachable only through `rule`, so a file in one of them with + no ruling is a contradiction `check` should name. +22. **The ruling id also lives in the title line (`# <id>` or `# <id> — + <summary>`), and `check` requires it to equal the filename.** Why: the + plan says "id == filename" and the file must state its id somewhere a + human reads. +23. **A run line's `<partition>` is a bare SLURM partition name + (`^[a-z][a-z0-9_-]*$`), not an enumeration.** Why: the project row's + `partition: gpu | ral | both` is a capability; the run records the + actual queue. +24. **An `after:` / `resumes:` target is the verbatim identifier of another + run line in the same phase (`342091_9`, `342091_[0-8,10]`), not a bare + stem.** Why: a checkpoint resubmit resumes one task, not the array. +25. **A `ruled:` continuation must resolve to a ruling file.** Why: it is a + pointer and a dangling pointer is drift. +26. **Rulings on one phase need not chain: `leave-to-finish` followed by + `accept` is two chains of one; `Supersedes:` replaces a verdict, never + sequences.** Why: the plan makes `--supersedes` an explicit flag; forcing + every later ruling to supersede would make it redundant. +27. **`Batch:` on a ruling must name an existing `batches/<slot>.md`.** Why: + it is the join key; a typo would silently orphan the ruling from its + slot. +28. **Review grammar: `<HEALTH>` ∈ `HEALTHY | SUSPECT | FAILED | RUNNING`; + `- decision: (none)` + `- ruled: no` is an untouched member; a verb with + `ruled: no` is a leaning the human did not tick.** Why: the Mind's + packet has the same tick/decision split; `(none)` keeps the four-verb + vocabulary pure while listing every member. +29. **Batch record keys keep the Mind's names (`dispatched:` = when the + board opened) and drop `usage-window-*`, `heart-ack:` and + `expected-effects:`; `- refreshed:` is a repeated line.** Why: one + parser in the Brain can read both record kinds; the dropped fields + license cloud sessions and autonomous YELLOW acks, which a science batch + never spends. The member `<state>` is the state at the last refresh and + `check` requires it legal, not current. +30. **`ledger_merge.yml` keeps a blocking `cortex.py check` on the branch tip + in addition to the trial-merge check.** Why: `workflow_dispatch` in + `audit` mode never merges, so without it audit mode reports nothing. +31. **`cortex_check.yml` triggers on its own path too.** Why: a workflow edit + that does not run itself is invisible until the next ledger push. diff --git a/epics.md b/epics.md new file mode 100644 index 0000000..a48771c --- /dev/null +++ b/epics.md @@ -0,0 +1,27 @@ +# Epics — the Cortex half + +Long-running multi-phase programmes whose **science** phases live here. An +epic is split by slug across the two dashboards: the Mind holds the +development phases (prompts, issues, PRs), the Cortex holds the run-and-ruling +phases (`phases/<project>/`), and each half's card links the other. A Cortex +phase declares its membership in its own header: `Epic: <slug>` (this file's +slug) plus its `Phase: <n>`, the same join key the Mind's prompts use. + +Schema per entry — the Mind's, plus one field: + +```markdown +## <slug> +- title: <the programme in one line> +- ledger: <the file that holds this half's phase/gate state — a phase directory or a project ledger> +- mind-half: <slug> # the Mind's epics.md entry for the development half; `none` if the epic moved whole +- status: <coarse, durable state — never per-phase detail> # optional +- notes: <free text> +``` + +`- mind-half:` names the Mind entry by slug; the Mind's entries gain the +reciprocal `- cortex-half:` in phase 4 of the birth epic, when the split epics +(`euclid-dr1-prep` phases 4, 5, 6a, 6b; `jax-inference-profiling` whole; +`graphical-ep` phases 3/4 science halves; `cluster-strong-lensing` phase 11) +move here. Until then this file holds no entries. + +This file is ledger: it auto-merges (`scripts/ledger_merge.py`). diff --git a/policy/never_rewrite_history.md b/policy/never_rewrite_history.md new file mode 100644 index 0000000..1b8b668 --- /dev/null +++ b/policy/never_rewrite_history.md @@ -0,0 +1,6 @@ +## Never rewrite history + +Never rewrite pushed history on any repo with a remote — no `git init` over a +tracked repo, no force-push to `main`, no fresh-start "Initial commit", no +`filter-repo` / `filter-branch` / `rebase -i` on pushed branches. To get a +clean tree: `git fetch origin && git reset --hard origin/main && git clean -fd`. diff --git a/policy/remote_sessions.md b/policy/remote_sessions.md new file mode 100644 index 0000000..b168198 --- /dev/null +++ b/policy/remote_sessions.md @@ -0,0 +1,44 @@ +## Remote sessions (Claude Code on web and mobile) + +Three facts, all measured in a web/mobile container, where this file is loaded +and little else is. They ride in every organ because a session may hold any +subset of them — and the session that needs this most is the one holding +several, which is exactly the session no hook fires in. + +- **Bootstrap in the first turn, unconditionally** — before the first test + command, not as a remedy once something looks wrong: + + ``` + bash PyAutoMind/scripts/session_bootstrap.sh # ~10s cold, ~1s warm + bash PyAutoMind/scripts/session_bootstrap.sh --check # report only + ``` + + A session holding several organs registers no SessionStart hook — Claude Code + reads project hooks from the project directory, which in that layout is the + repos' *parent*, not a repo — so nothing has set this session up. It was once + phrased as a remedy keyed to `No module named pytest` or collection + `ImportError`s naming `yaml`; that symptom stopped appearing when the + container image moved to Python 3.12, while the environment is still wrong in + ways that read like a bad command rather than a stale session (`pytest -n + auto` → `unrecognized arguments: -n`). The bootstrap also **unshallows the + clones**: a remote session clones shallow, and `git merge-base --is-ancestor` + then answers "not an ancestor" for a commit whose ancestry is merely absent — + the answer the ship and close-out procedures act on when proving a branch + merged. + +- **Then run the suite in parallel.** 4 cores, subprocess-heavy suites, no + single slow test: about 3.5x. `python3 -m pytest -q -n auto`, with + `pytest-xdist` supplied by the bootstrap above. + +- **There is no `gh`, and installing one does not help.** A remote session + reaches GitHub through the `mcp__github__*` tools, already scoped to the + session's repos. `gh` installs in two seconds and is a trap: it authenticates, + then 403s every repo-scoped call, because the egress proxy serves neither the + REST repo paths nor GraphQL beyond a pinned set of PR-review operations — a + binary that looks healthy and fails everything that matters. It also defeats + the surface probe, which keys off `gh auth status`. Read + `PyAutoBrain/skills/GITHUB_ACCESS.md` at the top of any run that touches + GitHub; it maps each `gh` operation onto its MCP tool. Spell that path from + the workspace root, as written: a multi-organ session is cwd'd at the repos' + *parent*, so a bare `skills/…` reads as a missing file rather than a missing + repo prefix. diff --git a/projects.yaml b/projects.yaml new file mode 100644 index 0000000..103e120 --- /dev/null +++ b/projects.yaml @@ -0,0 +1,49 @@ +# projects.yaml — the science body map. +# +# One row per science project. This file is CODE, not ledger: `sync_cli` and +# `local_path` are paths the batch conductor will execute under, so a change +# here is always a human's turn (scripts/ledger_merge.py leaves it for review, +# exactly like PyAutoMind/repos.yaml). +# +# Schema — a restricted YAML subset that scripts/cortex.py parses with the +# stdlib (REFERENCE.md "The restricted YAML subset"). It is also real YAML: +# `yaml.safe_load` reads it identically, and a test keeps the two in step. +# +# <key>: # column 0, ^[a-z][a-z0-9_]*$ — the project key +# remote: <owner/repo | none> # the private PyAutoLabs remote (code, config, wiki, witness files only) +# local_path: <abs path> # the laptop checkout +# ral_root: <abs path> # the project root on RAL (/mnt/ral/jnightin/<project>) +# mirror: <abs path | none> # the laptop pull root the sync CLI fills (holds data, not science) +# sync_cli: <path> # the project's sync CLI, relative to local_path +# sync_verbs: [a, b, c] # the verbs that CLI has — a flow list of bare words, the only list +# ledger: <path> # the project's own commentary ledger, relative to local_path +# witness_file: <glob> # where a run's witness lands, relative to the mirror +# partition: gpu | ral | both # which SLURM partition(s) the project runs on +# status: active | dormant +# +# Scalars are bare unless they contain `#`, `:` or edge spaces (then "..." +# with no escapes); comments on their own line or after ` #`; no block lists, +# no deeper nesting, no anchors, no `---`. Every field is required; an +# unknown field is an error. +# +# THE WORKSPACE-PATHS EXCEPTION. `local_path` and `mirror` are absolute laptop +# paths OUTSIDE the PyAutoLabs workspace — the Science folder under /mnt/c/… +# where the datasets, output trees and pulled results live. This file is the +# one place in the organism that carries such a path; a phase file or a ruling +# reaches a project through its row here, never with a bare path of its own. +# +# Worked example — the JAX inference programme, the project the schema was +# stress-tested against. Real rows are seeded in phase 3 of the birth epic; +# until then the map is empty. +# +# autolens_profiling: +# remote: PyAutoLabs/autolens_profiling +# local_path: /home/jammy/Code/PyAutoLabs/autolens_profiling +# ral_root: /mnt/ral/jnightin/autolens_profiling +# mirror: /mnt/c/Users/Jammy/Science/inference_programme +# sync_cli: hpc/sync +# sync_verbs: [pull, logs, status, submit, jobs, sacct, cancel, tail, du, check] +# ledger: results/notes/inference/PROGRAMME.md +# witness_file: results/searches/**/*.json +# partition: both +# status: active diff --git a/rulings/AGENTS.md b/rulings/AGENTS.md new file mode 100644 index 0000000..863488b --- /dev/null +++ b/rulings/AGENTS.md @@ -0,0 +1,47 @@ +# Rulings — the ledger of record + +`rulings/<YYYY>/<MM>/R-<YYYYMMDD>-<nn>.md`, one file per verdict per phase. +This folder is canonical: **a verdict recorded only outside the Cortex does +not exist.** A project's own ledger (`DECISIONS.md`, `state.md`, +`RESULTS.md`) is scientific commentary — evidence, reasoning, consequences — +and cites the ruling id; it never holds a verdict the Cortex does not. + +Why: on 2026-09-01 one ruling was written in up to three places and one of +them was never written at all. + +## The rules + +1. **Append-only.** A ruling file, once committed, is never modified or + deleted. `scripts/ledger_merge.py` classifies an `M`, `D` or `R` entry + under `rulings/**` as code, which leaves the branch for a human; there is + no auto-merge path for an edited ruling. +2. **Supersede, never edit.** A wrong ruling gets a new ruling with + `Supersedes: <old id>` — same project, same phase, later id. The new one + becomes the phase's `Ruling:`. Chains, not trees: supersede the current + head, never a ruling that already has a successor. +3. **`cortex.py rule` writes them.** It assigns the id, refuses to touch an + existing file, updates the phase's `Ruling:` and `State:` in the same + change. Write one by hand only to migrate history (phase 4), and run + `cortex.py check` before pushing. +4. **The human's words, verbatim.** `## Ruling` is what they said; `## + Evidence` is where to look. Paraphrase belongs in the project ledger. + +## The id + +`R-YYYYMMDD-nn`: the review date and a two-digit per-day sequence, global +across projects (`01`, `02`, … in the order filed that day). The filename is +the id; so is the title line (`# R-20260901-02` or `# R-20260901-02 — <one-line +summary>`). Two branches that both assign `-03` on the same day collide on the +trial-merge check in `ledger_merge.yml`, which is the point of running it +there. + +## One file per phase + +A ruling names exactly one phase (`Phase: phases/<project>/<slug>.md`). A +verdict spanning several phases — a REWIND — is N files with the same +`## Ruling` body, one per phase, joined by one `Batch:` slot; `cortex.py rule +--also <phase>` fans them out. `Batch:` is also how rulings filed from one +review slot are found together. + +The header keys and the chain rules `check` enforces are in +[REFERENCE.md](../REFERENCE.md) "Rulings". diff --git a/tests/fixtures/empty/phases/.gitkeep b/tests/fixtures/empty/phases/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/empty/projects.yaml b/tests/fixtures/empty/projects.yaml new file mode 100644 index 0000000..516b255 --- /dev/null +++ b/tests/fixtures/empty/projects.yaml @@ -0,0 +1 @@ +# Test fixture — an empty science body map. No rows: parses to an empty map. diff --git a/tests/fixtures/empty/rulings/.gitkeep b/tests/fixtures/empty/rulings/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tests/fixtures/skeleton/batches/2026-09-01-pm.md b/tests/fixtures/skeleton/batches/2026-09-01-pm.md new file mode 100644 index 0000000..a45f922 --- /dev/null +++ b/tests/fixtures/skeleton/batches/2026-09-01-pm.md @@ -0,0 +1,26 @@ +# Batch 2026-09-01 pm +- dispatched: 2026-09-01T09:00Z +- review-at: 2026-09-01T17:00Z +- shift: day +- lane: local-dev +- review-minutes-planned: 33 +- members: + - 05_running_array: phases/example/05_running_array.md — 342091, 342102 — 8 — running + - 06_pulled: phases/example/06_pulled.md — 342120 — 5 — pulled + - 07_awaiting_ruling: phases/example/07_awaiting_ruling.md — 342110, 342120 — 5 — awaiting-ruling + - 08_accepted: phases/example/08_accepted.md — 342050 — 6 — awaiting-ruling + - 09_rerun: phases/example/09_rerun.md — 341900 — 5 — awaiting-ruling + - 10_dropped: phases/example/10_dropped.md — 341950, 341960 — 4 — awaiting-ruling +- refreshed: 2026-09-01T11:40Z — 06_pulled pulled +- refreshed: 2026-09-01T14:05Z — 07_awaiting_ruling pulled +- refreshed: 2026-09-01T15:30Z — 05_running_array pulled (partial — nine of ten tasks) +- collected: 2026-09-01T17:00Z +- reviewed-at: 2026-09-01T17:10Z +- delivered: 4/5 +- packet: batches/packets/2026-09-01-pm.html +- review: batches/reviews/2026-09-01-pm.md +- review-minutes-actual: 18 +- notes: | + Phase 5's task 9 OOMed before its first step and was resubmitted alone + (342102); the board showed the other nine. Phase 10 counted as not + delivered — legacy_wrong, nothing to review but the drop. diff --git a/tests/fixtures/skeleton/batches/reviews/2026-09-01-pm.md b/tests/fixtures/skeleton/batches/reviews/2026-09-01-pm.md new file mode 100644 index 0000000..3c8d5f4 --- /dev/null +++ b/tests/fixtures/skeleton/batches/reviews/2026-09-01-pm.md @@ -0,0 +1,44 @@ +# Batch review 2026-09-01-pm + +- packet: batches/packets/2026-09-01-pm.html +- reviewed-at: 2026-09-01T17:10Z +- review-minutes-actual: 18 + +## 10_dropped — FAILED +- decision: drop +- ruled: yes + +Wrong regularisation; phase 8 answers the question. Drop it, revive as a new +phase if mesh-pix comes back. + +## 09_rerun — SUSPECT +- decision: rerun +- ruled: yes + +Mask edge one pixel out and the run predates the fix. Redo on the released +stack after PyAutoArray#510. + +## 08_accepted — HEALTHY +- decision: accept +- ruled: yes + +Same verdict as this morning; the earlier ruling cited the phase 3 summary by +mistake. Supersede it and file the sampler-default follow-up. + +## 07_awaiting_ruling — HEALTHY +- decision: accept +- ruled: no + +Looks right but I want the phase 6 half beside it before I rule. + +## 06_pulled — HEALTHY +- decision: (none) +- ruled: no + +(no note) + +## 05_running_array — RUNNING +- decision: leave-to-finish +- ruled: yes + +Nine in, task 9 resubmitted alone. Next board. diff --git a/tests/fixtures/skeleton/phases/example/01_scope.md b/tests/fixtures/skeleton/phases/example/01_scope.md new file mode 100644 index 0000000..5740345 --- /dev/null +++ b/tests/fixtures/skeleton/phases/example/01_scope.md @@ -0,0 +1,30 @@ +# Example — phase 1: scope the programme + +Project: example +Phase: 1 +State: planned +Gates: +Witness: +Budget: 4:00 +Lane: local-dev +Review-minutes: 3 +Epic: example-programme +Filed: 2026-08-25 + +## Question + +Which three lenses anchor the programme, and at what pixel scale? + +## Witness + +(not yet registered — a planned phase may leave this empty) + +## Where to look + +- `/mnt/c/Users/Jammy/Science/example/wiki/project/state.md` + +## Runs + +## Ruling + +(none) diff --git a/tests/fixtures/skeleton/phases/example/02_gated_on_dev.md b/tests/fixtures/skeleton/phases/example/02_gated_on_dev.md new file mode 100644 index 0000000..14a3ef8 --- /dev/null +++ b/tests/fixtures/skeleton/phases/example/02_gated_on_dev.md @@ -0,0 +1,31 @@ +# Example — phase 2: the pixel-scale sweep, gated on two dev fixes + +Project: example +Phase: 2 +State: gated +Gates: PyAutoArray#431, https://github.com/PyAutoLabs/PyAutoFit/pull/1436 +Witness: three lenses reach log-evidence within 2.0 of the anchor at 0.05 arcsec +Budget: 6:00 +Lane: local-dev +Review-minutes: 6 +Epic: example-programme +Filed: 2026-08-26 + +## Question + +Does the pixel-scale sweep change the preferred model once the mask fix and the +prior-loading fix land? + +## Witness + +`output/phase_02/*/samples_summary.json` — log-evidence within 2.0 of phase 1. + +## Where to look + +- `/mnt/c/Users/Jammy/Science/example/output/phase_02/` + +## Runs + +## Ruling + +(none) diff --git a/tests/fixtures/skeleton/phases/example/03_ready_cleared.md b/tests/fixtures/skeleton/phases/example/03_ready_cleared.md new file mode 100644 index 0000000..cc30787 --- /dev/null +++ b/tests/fixtures/skeleton/phases/example/03_ready_cleared.md @@ -0,0 +1,31 @@ +# Example — phase 3: the anchor refit, gate cleared + +Project: example +Phase: 3 +State: ready +Gates: PyAutoGalaxy#486 +Gates-cleared: 2026-08-30 +Witness: anchor lens theta_E within 0.01 arcsec of the published value +Budget: 5:00 +Lane: local-dev +Review-minutes: 4 +Epic: example-programme +Filed: 2026-08-27 + +## Question + +Does the refit on the released stack reproduce the published anchor? + +## Witness + +`output/phase_03/anchor/samples_summary.json` — `theta_E` within 0.01 arcsec. + +## Where to look + +- `/mnt/c/Users/Jammy/Science/example/output/phase_03/` + +## Runs + +## Ruling + +(none) diff --git a/tests/fixtures/skeleton/phases/example/04_submitted_override.md b/tests/fixtures/skeleton/phases/example/04_submitted_override.md new file mode 100644 index 0000000..6625d73 --- /dev/null +++ b/tests/fixtures/skeleton/phases/example/04_submitted_override.md @@ -0,0 +1,35 @@ +# Example — phase 4: the two-band fit, submitted under a gate override + +Project: example +Phase: 4 +State: submitted +Gates: PyAutoLens#900 +Gate-override: the fix is verified locally on the branch; not waiting for the merge +Witness: both bands converge to the same source centre within 0.02 arcsec +Budget: 6:00 +Runs: 342001, 342010 +Lane: local-dev +Review-minutes: 6 +Epic: example-programme +Filed: 2026-08-28 + +## Question + +Do the two bands agree on the source centre once fitted jointly? + +## Witness + +`output/phase_04/joint/samples_summary.json` — source centres within 0.02 arcsec. + +## Where to look + +- `/mnt/c/Users/Jammy/Science/example/output/phase_04/` + +## Runs + +- 342001: failed — gpu — submitted 2026-08-31 — wall 0:00 — node failure before the first step +- 342010: submitted — gpu — submitted 2026-09-01 — wall 0:00 + +## Ruling + +(none) diff --git a/tests/fixtures/skeleton/phases/example/05_running_array.md b/tests/fixtures/skeleton/phases/example/05_running_array.md new file mode 100644 index 0000000..f385df3 --- /dev/null +++ b/tests/fixtures/skeleton/phases/example/05_running_array.md @@ -0,0 +1,39 @@ +# Example — phase 5: the nine-lens array + +Project: example +Phase: 5 +State: running +Gates: PyAutoArray#431 +Gates-cleared: 2026-08-29 +Witness: nine of ten array tasks write a sane checkpoint.hdf5 within 8:00 wall +Budget: 8:00 +Runs: 342091, 342102 +Ruling: R-20260901-03 +Lane: local-dev +Review-minutes: 8 +Epic: example-programme +Filed: 2026-08-29 + +## Question + +Does the Delaunay pipeline reach the same theta_E basin on all ten lenses? + +## Witness + +`output/phase_05/*/checkpoint.hdf5` present for nine lenses, `.err` clean. + +## Where to look + +- `/mnt/c/Users/Jammy/Science/example/output/phase_05/` + +## Runs + +- 342091_[0-8,10]: done — gpu — submitted 2026-08-30 — wall 6:12 + pulled_to: /mnt/c/Users/Jammy/Science/example/output/phase_05 +- 342091_9: failed — gpu — submitted 2026-08-30 — wall 0:00 — OOM before the first step +- 342102: running — gpu — submitted 2026-09-01 — wall 0:00 — task 9 resubmitted alone + after: 342091_9 + +## Ruling + +R-20260901-03 — leave-to-finish diff --git a/tests/fixtures/skeleton/phases/example/06_pulled.md b/tests/fixtures/skeleton/phases/example/06_pulled.md new file mode 100644 index 0000000..430905c --- /dev/null +++ b/tests/fixtures/skeleton/phases/example/06_pulled.md @@ -0,0 +1,33 @@ +# Example — phase 6: the first half of the shared array, pulled + +Project: example +Phase: 6 +State: pulled +Witness: five lenses' checkpoint.hdf5 sane and .err clean under 4:00 wall +Budget: 4:00 +Runs: 342120 +Lane: local-dev +Review-minutes: 5 +Epic: example-programme +Filed: 2026-08-30 + +## Question + +Do the five bright lenses converge under the fast preset? + +## Witness + +`output/phase_06/*/checkpoint.hdf5` sane for all five. + +## Where to look + +- `/mnt/c/Users/Jammy/Science/example/output/phase_06/` + +## Runs + +- 342120_[0-4]: done — gpu — submitted 2026-08-31 — wall 3:40 — array shared with phase 7 + pulled_to: /mnt/c/Users/Jammy/Science/example/output/phase_06 + +## Ruling + +(none) diff --git a/tests/fixtures/skeleton/phases/example/07_awaiting_ruling.md b/tests/fixtures/skeleton/phases/example/07_awaiting_ruling.md new file mode 100644 index 0000000..219ef9c --- /dev/null +++ b/tests/fixtures/skeleton/phases/example/07_awaiting_ruling.md @@ -0,0 +1,35 @@ +# Example — phase 7: the second half of the shared array, awaiting a ruling + +Project: example +Phase: 7 +State: awaiting-ruling +Witness: five faint lenses' checkpoint.hdf5 sane after the checkpoint resubmit +Budget: 8:00 +Runs: 342110, 342120 +Lane: local-dev +Review-minutes: 5 +Epic: example-programme +Filed: 2026-08-30 + +## Question + +Do the five faint lenses converge once resumed from the timed-out checkpoint? + +## Witness + +`output/phase_07/*/checkpoint.hdf5` sane for all five. + +## Where to look + +- `/mnt/c/Users/Jammy/Science/example/output/phase_07/` + +## Runs + +- 342110: timeout — gpu — submitted 2026-08-30 — wall 8:00 — hit the budget; resubmitted from checkpoint +- 342120_[5-9]: done — gpu — submitted 2026-08-31 — wall 3:55 — array shared with phase 6 + resumes: 342110 + pulled_to: /mnt/c/Users/Jammy/Science/example/output/phase_07 + +## Ruling + +(none) diff --git a/tests/fixtures/skeleton/phases/example/08_accepted.md b/tests/fixtures/skeleton/phases/example/08_accepted.md new file mode 100644 index 0000000..2ac3093 --- /dev/null +++ b/tests/fixtures/skeleton/phases/example/08_accepted.md @@ -0,0 +1,37 @@ +# Example — phase 8: the anchor comparison, accepted + +Project: example +Phase: 8 +State: accepted +Gates: PyAutoFit#1400 +Gates-cleared: 2026-08-25 +Witness: anchor theta_E within 0.01 arcsec of the published value on the new sampler +Budget: 6:00 +Runs: 342050 +Ruling: R-20260901-02 +Lane: local-dev +Review-minutes: 6 +Epic: example-programme +Filed: 2026-08-24 + +## Question + +Does the new sampler reproduce the anchor within the published error? + +## Witness + +`output/phase_08/anchor/samples_summary.json` — `theta_E` within 0.01 arcsec. + +## Where to look + +- `/mnt/c/Users/Jammy/Science/example/output/phase_08/` + +## Runs + +- 342050: done — gpu — submitted 2026-08-28 — wall 5:10 + pulled_to: /mnt/c/Users/Jammy/Science/example/output/phase_08 + ruled: R-20260901-02 + +## Ruling + +R-20260901-02 — accept (supersedes R-20260901-01) diff --git a/tests/fixtures/skeleton/phases/example/09_rerun.md b/tests/fixtures/skeleton/phases/example/09_rerun.md new file mode 100644 index 0000000..9c151ba --- /dev/null +++ b/tests/fixtures/skeleton/phases/example/09_rerun.md @@ -0,0 +1,35 @@ +# Example — phase 9: the legacy-born mask test, ruled rerun + +Project: example +Phase: 9 +State: rerun +Witness: the legacy run's mask edge matches the released mask within one pixel +Budget: 8:00 +Runs: 341900 +Ruling: R-20260901-04 +Lane: local-dev +Review-minutes: 5 +Epic: example-programme +Filed: 2026-08-31 + +## Question + +Is the pre-Cortex mask-edge run reusable, or must it be redone on the released stack? + +## Witness + +`output/legacy/phase_09/mask.fits` edge vs the released mask. + +## Where to look + +- `/mnt/c/Users/Jammy/Science/example/output/legacy/phase_09/` + +## Runs + +- 341900: legacy — gpu — submitted 2026-08-20 — wall 7:50 — pre-Cortex run, quarantined as reusable + where: /mnt/c/Users/Jammy/Science/example/output/legacy/phase_09 + ruled: R-20260901-04 + +## Ruling + +R-20260901-04 — rerun diff --git a/tests/fixtures/skeleton/phases/example/10_dropped.md b/tests/fixtures/skeleton/phases/example/10_dropped.md new file mode 100644 index 0000000..b1f6bba --- /dev/null +++ b/tests/fixtures/skeleton/phases/example/10_dropped.md @@ -0,0 +1,35 @@ +# Example — phase 10: the mesh-pix sweep, dropped + +Project: example +Phase: 10 +State: dropped +Witness: mesh-pix log-evidence beats the Delaunay anchor by 5.0 +Budget: 6:00 +Runs: 341950, 341960 +Ruling: R-20260901-05 +Lane: local-dev +Review-minutes: 4 +Epic: example-programme +Filed: 2026-08-31 + +## Question + +Does the mesh-pix sweep beat the Delaunay anchor? + +## Witness + +`output/legacy_wrong/phase_10/*/samples_summary.json` log-evidence vs the anchor. + +## Where to look + +- `/mnt/c/Users/Jammy/Science/example/output/legacy_wrong/phase_10/` + +## Runs + +- 341950_[0-3]: legacy_wrong — gpu — submitted 2026-08-22 — wall 6:00 — ran on the wrong regularisation + where: /mnt/c/Users/Jammy/Science/example/output/legacy_wrong/phase_10 +- 341960: void — gpu — submitted 2026-08-23 — wall 0:00 — cancelled before the first step + +## Ruling + +R-20260901-05 — drop diff --git a/tests/fixtures/skeleton/projects.yaml b/tests/fixtures/skeleton/projects.yaml new file mode 100644 index 0000000..879255e --- /dev/null +++ b/tests/fixtures/skeleton/projects.yaml @@ -0,0 +1,14 @@ +# Test fixture — the science body map with one fictional project. +# Same restricted subset as the live projects.yaml (REFERENCE.md). + +example: + remote: none + local_path: /mnt/c/Users/Jammy/Science/example + ral_root: /mnt/ral/jnightin/example + mirror: /mnt/c/Users/Jammy/Science/example/mirror # the pull root + sync_cli: hpc/sync + sync_verbs: [pull, submit, jobs, tail] + ledger: wiki/project/state.md + witness_file: output/**/checkpoint.hdf5 + partition: gpu + status: active diff --git a/tests/fixtures/skeleton/rulings/2026/09/R-20260901-01.md b/tests/fixtures/skeleton/rulings/2026/09/R-20260901-01.md new file mode 100644 index 0000000..752bb93 --- /dev/null +++ b/tests/fixtures/skeleton/rulings/2026/09/R-20260901-01.md @@ -0,0 +1,18 @@ +# R-20260901-01 — accept phase 8 + +Project: example +Phase: phases/example/08_accepted.md +Runs: 342050 +Ruling: accept +Reviewed-at: 2026-09-01T10:30Z +Review-minutes-actual: 5 + +## Ruling + +Accept. theta_E is 1.204 against the published 1.20 ± 0.01; the new sampler +reproduces the anchor. + +## Evidence + +- `/mnt/c/Users/Jammy/Science/example/output/phase_08/anchor/samples_summary.json` +- `/mnt/c/Users/Jammy/Science/example/wiki/project/state.md` (phase 8) diff --git a/tests/fixtures/skeleton/rulings/2026/09/R-20260901-02.md b/tests/fixtures/skeleton/rulings/2026/09/R-20260901-02.md new file mode 100644 index 0000000..a5e1164 --- /dev/null +++ b/tests/fixtures/skeleton/rulings/2026/09/R-20260901-02.md @@ -0,0 +1,22 @@ +# R-20260901-02 — re-accept phase 8 with the corrected evidence pointer + +Project: example +Phase: phases/example/08_accepted.md +Runs: 342050 +Ruling: accept +Supersedes: R-20260901-01 +Batch: 2026-09-01-pm +Reviewed-at: 2026-09-01T17:20Z +Review-minutes-actual: 6 +Follow-ups: PyAutoLens#901 + +## Ruling + +Accept, as before. R-20260901-01 cited the wrong summary file (the phase 3 +anchor, not the phase 8 one); the number and the verdict stand. File the +sampler-default follow-up. + +## Evidence + +- `/mnt/c/Users/Jammy/Science/example/output/phase_08/anchor/samples_summary.json` +- `/mnt/c/Users/Jammy/Science/example/wiki/project/state.md` (phase 8) diff --git a/tests/fixtures/skeleton/rulings/2026/09/R-20260901-03.md b/tests/fixtures/skeleton/rulings/2026/09/R-20260901-03.md new file mode 100644 index 0000000..10bbab7 --- /dev/null +++ b/tests/fixtures/skeleton/rulings/2026/09/R-20260901-03.md @@ -0,0 +1,18 @@ +# R-20260901-03 — leave phase 5 to finish + +Project: example +Phase: phases/example/05_running_array.md +Runs: 342091, 342102 +Ruling: leave-to-finish +Batch: 2026-09-01-pm +Reviewed-at: 2026-09-01T17:25Z +Review-minutes-actual: 2 + +## Ruling + +Leave it to finish. Nine of ten are in and look right; task 9 is resubmitted +alone and will be on the next board. + +## Evidence + +- `/mnt/c/Users/Jammy/Science/example/output/phase_05/` (nine checkpoints) diff --git a/tests/fixtures/skeleton/rulings/2026/09/R-20260901-04.md b/tests/fixtures/skeleton/rulings/2026/09/R-20260901-04.md new file mode 100644 index 0000000..0ccb1b7 --- /dev/null +++ b/tests/fixtures/skeleton/rulings/2026/09/R-20260901-04.md @@ -0,0 +1,19 @@ +# R-20260901-04 — rerun phase 9 on the released stack + +Project: example +Phase: phases/example/09_rerun.md +Runs: 341900 +Ruling: rerun +Batch: 2026-09-01-pm +Reviewed-at: 2026-09-01T17:30Z +Review-minutes-actual: 4 +Follow-ups: https://github.com/PyAutoLabs/PyAutoArray/issues/510 + +## Ruling + +Rerun. The legacy mask edge is one pixel out and the run predates the mask +fix; redo it on the released stack once PyAutoArray#510 lands. + +## Evidence + +- `/mnt/c/Users/Jammy/Science/example/output/legacy/phase_09/mask.fits` diff --git a/tests/fixtures/skeleton/rulings/2026/09/R-20260901-05.md b/tests/fixtures/skeleton/rulings/2026/09/R-20260901-05.md new file mode 100644 index 0000000..bf15b7c --- /dev/null +++ b/tests/fixtures/skeleton/rulings/2026/09/R-20260901-05.md @@ -0,0 +1,19 @@ +# R-20260901-05 — drop the mesh-pix sweep + +Project: example +Phase: phases/example/10_dropped.md +Runs: 341950, 341960 +Ruling: drop +Batch: 2026-09-01-pm +Reviewed-at: 2026-09-01T17:35Z +Review-minutes-actual: 3 + +## Ruling + +Drop. The sweep ran on the wrong regularisation and the question is answered +by phase 8; nothing here is reusable. Revive as a new phase if mesh-pix comes +back. + +## Evidence + +- `/mnt/c/Users/Jammy/Science/example/output/legacy_wrong/phase_10/` From 3350ec20fea0b1a3d46a138aed4f530ba3665895 Mon Sep 17 00:00:00 2001 From: Jammy2211 <JNightingale2211@gmail.com> Date: Tue, 1 Sep 2026 18:24:04 -0400 Subject: [PATCH 2/2] =?UTF-8?q?cortex:=20cortex.py,=20ledger=5Fmerge.py=20?= =?UTF-8?q?and=20the=20test=20suite=20=E2=80=94=20the=20skeleton,=20part?= =?UTF-8?q?=20B=20(PyAutoMind#379)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scripts/cortex.py — check · gates · rule · move · new, stdlib only, main(argv), --root and --today on every verb; the light-header parser and in-place header editor; the restricted-YAML loop for projects.yaml; every check rule in REFERENCE.md as a leg over `root: Path`; gates grading with an injectable fetch (gh api first, urllib fallback, User-Agent pyautocortex). scripts/ledger_merge.py — the Mind's classifier re-pointed (phases/, rulings/, batches/, epics.md) plus the append-only leg: a `git diff --name-status` entry under rulings/ that is not `A` is code. tests/test_cortex.py (99) and tests/test_ledger_merge.py (24): both fixtures pass check unchanged; every check rule as a tmp_path mutation; every legal transition and every refused one; rule ids, supersession, --also fan-out; gates --grade/--write with an injected fetch; new; PyYAML parity. No fixture was changed. REFERENCE.md and docs/schema_decisions.md gain the choices the code forced (decisions 32–37): the `Reset:` key, `--pulled-to`, the typo-guard checks, the partition default, `--also` on accepted phases, the demotion removing `Gates-cleared:`. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01SESCzU1FcwktEAdTZfmf3j --- REFERENCE.md | 68 +- docs/schema_decisions.md | 26 + scripts/cortex.py | 1613 ++++++++++++++++++++++++++++++++++++ scripts/ledger_merge.py | 221 +++++ tests/test_cortex.py | 1032 +++++++++++++++++++++++ tests/test_ledger_merge.py | 228 +++++ 6 files changed, 3170 insertions(+), 18 deletions(-) create mode 100644 scripts/cortex.py create mode 100755 scripts/ledger_merge.py create mode 100644 tests/test_cortex.py create mode 100644 tests/test_ledger_merge.py diff --git a/REFERENCE.md b/REFERENCE.md index 04a4248..c0bdca1 100644 --- a/REFERENCE.md +++ b/REFERENCE.md @@ -73,6 +73,7 @@ sections are fixed: `## Question`, `## Witness`, `## Where to look`, `## Runs`, | `Gates:` | comma-separated GitHub refs | `Repo#N` (owner `PyAutoLabs`) or an issue/PR URL; **no `owner/Repo#N`** form | | `Gates-cleared:` | `YYYY-MM-DD` | written by `gates --grade --write` when every ref cleared | | `Gate-override:` | reason | written by `move --override "<reason>"` | +| `Reset:` | reason | written by `move ready --reason "<reason>"` when a `submitted \| running` phase goes back to `ready` | | `Witness:` | free text | **mandatory before `submitted`** — the pre-registered checkable claim | | `Budget:` | `H+:MM` | wall budget per run | | `Runs:` | comma-separated job **stems** | the index of the `## Runs` body; equal to the set of body stems | @@ -125,11 +126,11 @@ owns. The full table: | gated | ready | `gates --grade --write` (all refs cleared → writes `Gates-cleared:`) or `move --override "<reason>"` (→ writes `Gate-override:`) | | ready | gated | `gates --grade --write` only, when a cleared gate reopened; refused if `Gate-override:` present; for `submitted`+ phases a reopened gate is *reported*, never enforced | | ready | submitted | `Witness:` non-empty AND `--run <id>` supplied | -| ready | pulled | legacy-born phase: every run line is `legacy\|legacy_wrong` (`new --legacy-run` / `move pulled`); `Witness:` still mandatory | +| ready | pulled | legacy-born phase: every run line is `legacy\|legacy_wrong` (`new --legacy-run` / `move pulled`); `Witness:` still mandatory; every `legacy` run gets a `pulled_to:` (`--pulled-to`, else its own `where:`); refused when no run is `legacy` (nothing to review — `rule drop`) | | submitted / running | same | `--run <id>` appends a wave, a chained job or a checkpoint resubmit (`resumes:`); state unchanged | | submitted | running | — | -| submitted / running | ready | no run line in `submitted\|running` AND ≥1 `failed\|timeout\|void`; `--reason` required | -| running | pulled | no run line live; or `--partial` (a partial array), which `check` expects closed by a `leave-to-finish` ruling | +| submitted / running | ready | no run line in `submitted\|running` AND ≥1 `failed\|timeout\|void`; `--reason` required (→ writes `Reset:`) | +| running | pulled | no run line live; or `--partial` (a partial array), which `check` expects closed by a `leave-to-finish` ruling; `--pulled-to <path>` writes `pulled_to:` on every `done` run lacking one, and the move is refused when no `done` run would carry one | | pulled | awaiting-ruling | — (the phase joins the rolling board) | | awaiting-ruling | accepted / rerun / dropped | **`rule` only** | | running / pulled / awaiting-ruling | same | `rule leave-to-finish` (state unchanged) | @@ -151,6 +152,13 @@ Offline invariants `check` enforces on top of the table: reachable only through `rule`). - `State = pulled` AND any run line `submitted | running` ⇒ `Ruling:` present and its head verb is `leave-to-finish` (the `--partial` case). +- `State = gated` ⇒ `Gates:` non-empty (a gate that does not exist cannot + clear). +- A header key outside the table is drift (`Gate-cleared:` must not pass as + a silent typo); the five body sections are present; `Lane:`, when present, + is `local-dev`; `Phase:` and `Review-minutes:` are integers, `Budget:` is + `H+:MM`, `Gates-cleared:` and `Filed:` are `YYYY-MM-DD`. A key with an + empty value is read as absent. --- @@ -307,7 +315,9 @@ the same body and one `Batch:`; `rule --also <phase>` fans out. **Chain rules** (`check`): -- the id is unique, equals the filename stem and the title; +- the id is unique, equals the filename stem and the title, and its date is + the file's `<YYYY>/<MM>` directory; the body has `## Ruling` and + `## Evidence`; - `Supersedes:` resolves to an existing ruling, is not the ruling itself, is lexically smaller (earlier), and names the same project **and** phase; - **at most one successor per ruling** — a chain, not a tree: supersede the @@ -344,8 +354,11 @@ Member line: `<slug>` is the phase file's stem; `<runs>` is comma-separated stems or `none`; `<state>` is the phase state at the last refresh. `check` verifies -the path exists, the slug matches its stem, the runs ⊆ the phase's `Runs:` -and the state is legal. +the path exists, the slug matches its stem, the runs ⊆ the phase's `Runs:`, +the review-minutes are an integer and the state is legal; a `review:` field +must resolve. A review file's title is `# Batch review <slot>` for its own +filename, its batch record `batches/<slot>.md` exists, and every section +names a member of it. **One review grammar** (`batches/reviews/<slot>.md`): @@ -450,24 +463,43 @@ Stdlib only; `main(argv)`; no import-time side effects; every verb takes absent. **Cleared** = PR ⇒ `merged_at != null` (closed-unmerged is a dead gate, reported); issue ⇒ `state == closed` and `state_reason ∈ {completed, null}` (`not_planned` / `duplicate` are dead gates); anything unreadable - fails closed. `--grade` reports only; `--write` flips `gated → ready` - (writing `Gates-cleared:`) and `ready → gated` when a cleared gate reopened - (refused when `Gate-override:` is present; for `submitted`+ phases a - reopened gate is reported, never enforced). + fails closed (reported, nothing flipped, exit 1). `--grade` reports only; + `--write` flips `gated → ready` (writing `Gates-cleared:`) and + `ready → gated` when a cleared gate reopened (removing the stale + `Gates-cleared:`; refused when `Gate-override:` is present; for + `submitted`+ phases a reopened gate is reported, never enforced). The + fetch is injectable (`gates_report(root, fetch=…)`) so the grading is + tested offline. - **`rule <phase> <verb> --body <file> [--supersedes <id>] [--batch <slot>] [--minutes n] [--follow-up <ref>]... [--also <phase>]...`** — assigns the next id for today, writes the ruling file(s) (one per phase; `--also` fans out with the same body and batch), updates the phase's `Ruling:` and - `State:` per the table; refuses to touch an existing ruling; refuses a verb - the table does not allow from the phase's state. + `State:` per the table (and appends `<id> — <verb>` to the phase's + `## Ruling`); the ruling's `Runs:` is the phase's, its `## Evidence` is the + phase's `## Where to look`; refuses to touch an existing ruling; refuses a + verb the table does not allow from the phase's state; validates everything + for every phase before writing anything. An `--also` phase in `accepted` + supersedes its own `Ruling:` (the REWIND is N accepted phases). - **`move <phase> <state> [--run <id>] [--reason ..] [--override ..] - [--partial]`** — the table; refuses every ruling edge with a message naming - `rule`. `--run` on `submitted`/`running` appends a run line and keeps the - state. + [--partial] [--pulled-to <path>] [--partition ..] [--after <run>] + [--resumes <run>] [--note ..]`** — the table; refuses every ruling edge + with a message naming `rule`. `--run` on `submitted`/`running` appends a + run line and keeps the state; the appended line is + `- <id>: submitted — <partition> — submitted <today> — wall 0:00[ — <note>]` + with `after:` / `resumes:` continuations from the flags, and its + partition is the project's `partition:` row unless that is `both`, when + `--partition` is required. Every edit is an in-place header edit or an + appended run line; every other byte of the file is preserved. - **`new <project> <slug> --phase <n> [--gates ..] [--epic ..] - [--legacy-run <id>]...`** — writes `phases/<project>/<slug>.md` from the - template in `planned` (or `ready` when `--legacy-run` is given and every run - is legacy); refuses a duplicate phase number or an unknown project. + [--legacy-run <id>]... [--legacy-wrong <id>]... [--where <path>] + [--partition ..] [--witness ..] [--budget ..] [--minutes n] [--title ..]`** + — writes `phases/<project>/<slug>.md` from the template in `planned` (or + `ready` when `--legacy-run` / `--legacy-wrong` is given and every run is + legacy); each legacy run line is written with today's date, `wall 0:00` + and the note `pre-Cortex run, migrated` — the human corrects the date and + wall by hand — and `--where` (required) as its `where:`; a legacy-born + phase refuses `--gates`. Refuses a duplicate phase number, an existing + file or an unknown project. --- diff --git a/docs/schema_decisions.md b/docs/schema_decisions.md index c28a423..82ddc81 100644 --- a/docs/schema_decisions.md +++ b/docs/schema_decisions.md @@ -121,3 +121,29 @@ Choices made while writing the files, beyond the plan: `audit` mode never merges, so without it audit mode reports nothing. 31. **`cortex_check.yml` triggers on its own path too.** Why: a workflow edit that does not run itself is invisible until the next ledger push. + +Choices made while writing `cortex.py` (slice B), beyond the plan: + +32. **`Reset:` is a phase header key, written by `move ready --reason`.** + Why: the table demands a reason for `submitted | running → ready` and the + key table gave it no home; a reason in prose cannot be found again. +33. **`move pulled` writes `pulled_to:` — `--pulled-to <path>`, or for a + `legacy` run its own `where:` — and is refused when no `done | legacy` + run would carry one.** Why: `check` requires it on a pulled phase, and a + verb that writes a state its own check rejects is a trap. +34. **An unknown header key, a missing body section, a `gated` phase with an + empty `Gates:`, or a `Lane:` other than `local-dev` is drift.** Why: + `Gate-cleared:` is one letter from `Gates-cleared:` and would otherwise + pass silently while the invariant it carries fails. +35. **The run-line partition on `move --run` and `new --legacy-run` is the + project's `partition:` row when that is `gpu | ral`; `both` requires + `--partition`.** Why: the row is a capability, the run records the + actual queue (decision 23), and the verb should not guess between two. +36. **`rule --also` on an `accepted` phase supersedes that phase's own + `Ruling:`; the primary phase takes `--supersedes` explicitly.** Why: the + REWIND is N accepted phases with N different heads, and spelling N ids by + hand is the error-prone form of the same instruction. +37. **`gates --grade --write` removes `Gates-cleared:` when it demotes + `ready → gated`; `--grade` exits 1 when any ref is unreadable.** Why: a + stale cleared-date on a gated phase misreads as cleared; an unreadable + ref fails closed and a scheduled run must notice. diff --git a/scripts/cortex.py b/scripts/cortex.py new file mode 100644 index 0000000..9b4b302 --- /dev/null +++ b/scripts/cortex.py @@ -0,0 +1,1613 @@ +#!/usr/bin/env python3 +"""cortex.py — the Cortex's lifecycle script: check · gates · rule · move · new. + +The run-and-ruling registry (project → phase → runs → rulings) is a tree of +markdown files with light headers, and this script is the one thing that +writes their state and the one thing that checks it. Every rule it enforces is +written down in REFERENCE.md; the transition table there is `move`'s and the +chain rules are `rule`'s. Nothing here is not in that file. + +Stdlib only, `main(argv)`, no import-time side effects, `--root` on every verb +(default: the repo this script lives in), so every leg runs against a +`tmp_path` copy of the fixture in tests. The date is injectable (`--today`) +for the same reason. + +Usage: + python3 scripts/cortex.py check # OK or DRIFT (exit 1) + python3 scripts/cortex.py gates [--grade [--write]] # gated phases and their refs + python3 scripts/cortex.py rule <phase> <verb> --body <file> [...] + python3 scripts/cortex.py move <phase> <state> [...] + python3 scripts/cortex.py new <project> <slug> --phase <n> [...] + +Exit codes: 0 = done · 1 = drift, a refused edit, or an unreadable gate · 2 = +bad arguments. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from datetime import date, datetime, timezone +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] + +# --------------------------------------------------------------------------- # +# vocabulary (REFERENCE.md) +# --------------------------------------------------------------------------- # +PHASE_STATES = ( + "planned", "gated", "ready", "submitted", "running", "pulled", + "awaiting-ruling", "accepted", "rerun", "dropped", +) +TERMINAL_STATES = {"dropped"} +NON_TERMINAL_STATES = set(PHASE_STATES) - TERMINAL_STATES +#: `ready..accepted` in the order listed, `rerun` included — the states in +#: which a gated phase must carry `Gates-cleared:` or `Gate-override:`. +GATES_SETTLED_STATES = {"ready", "submitted", "running", "pulled", + "awaiting-ruling", "accepted", "rerun"} +#: `submitted..accepted` (and `rerun`) — the states that need a `Witness:`. +WITNESS_STATES = {"submitted", "running", "pulled", "awaiting-ruling", + "accepted", "rerun"} +#: reachable only through `rule`, so they need a `Ruling:`. +RULED_STATES = {"accepted", "rerun", "dropped"} +#: the states a `gates --grade` run looks at (a reopened gate is reported on +#: every phase that has left `gated`; only `gated`/`ready` are ever flipped). +GRADED_STATES = {"gated", "ready", "submitted", "running", "pulled", + "awaiting-ruling", "rerun"} + +RUN_STATES = ("submitted", "running", "done", "failed", "timeout", "void", + "legacy", "legacy_wrong") +LIVE_RUN_STATES = {"submitted", "running"} +LEGACY_RUN_STATES = {"legacy", "legacy_wrong"} +RESET_RUN_STATES = {"failed", "timeout", "void"} + +RULING_VERBS = ("accept", "rerun", "drop", "leave-to-finish") +#: the head verb ↔ phase state agreement `check` enforces. +VERB_STATES = { + "accept": {"accepted"}, + "drop": {"dropped"}, + "rerun": {"rerun", "ready", "submitted", "running", "pulled", "awaiting-ruling"}, + "leave-to-finish": NON_TERMINAL_STATES, +} +#: `rule` writes the phase into this state (None = unchanged). +VERB_TARGET = {"accept": "accepted", "rerun": "rerun", "drop": "dropped", + "leave-to-finish": None} + +HEALTHS = ("HEALTHY", "SUSPECT", "FAILED", "RUNNING") +LANE = "local-dev" + +# Canonical header key order — `new` writes it and the edit helpers insert a +# missing key at its slot so a hand-edited file keeps reading the same. +PHASE_KEYS = ( + "Project", "Phase", "State", "Gates", "Gates-cleared", "Gate-override", + "Reset", "Witness", "Budget", "Runs", "Ruling", "Lane", "Review-minutes", + "Epic", "Filed", +) +RULING_KEYS = ( + "Project", "Phase", "Runs", "Ruling", "Supersedes", "Batch", "Reviewed-at", + "Review-minutes-actual", "Follow-ups", +) +PHASE_SECTIONS = ("Question", "Witness", "Where to look", "Runs", "Ruling") +RULING_SECTIONS = ("Ruling", "Evidence") + +PROJECT_FIELDS = ("remote", "local_path", "ral_root", "mirror", "sync_cli", + "sync_verbs", "ledger", "witness_file", "partition", "status") +PARTITIONS = ("gpu", "ral", "both") +PROJECT_STATUSES = ("active", "dormant") + +# --------------------------------------------------------------------------- # +# grammars (REFERENCE.md) +# --------------------------------------------------------------------------- # +# Copied verbatim from PyAutoMind/scripts/lifecycle.py: the lookbehind is what +# rejects `owner/Repo#N` — another owner is spelled as a URL. +GATE_REF_RE = re.compile( + r"https://github\.com/([\w.-]+)/([\w.-]+)/(?:issues|pull)/(\d+)" + r"|(?<![\w/])([A-Za-z_][\w.]*)#(\d+)\b" +) +DEFAULT_GATE_OWNER = "PyAutoLabs" + +RUN_LINE_RE = re.compile( + r"^- (?P<stem>\d+)" + r"(?:_(?P<task>\d+)|_\[(?P<tasks>\d+(?:-\d+)?(?:,\d+(?:-\d+)?)*)\])?" + r": (?P<state>submitted|running|done|failed|timeout|void|legacy|legacy_wrong)" + r" — (?P<partition>[a-z][a-z0-9_-]*)" + r" — submitted (?P<date>\d{4}-\d{2}-\d{2})" + r" — wall (?P<wall>\d+:\d{2})" + r"(?: — (?P<note>.+))?$" +) +RUN_CONT_RE = re.compile( + r"^ (?P<key>pulled_to|after|resumes|where|ruled): (?P<value>\S.*)$" +) +RUN_IDENT_RE = re.compile(r"^\d+(?:_\d+|_\[\d+(?:-\d+)?(?:,\d+(?:-\d+)?)*\])?$") +PARTITION_RE = re.compile(r"^[a-z][a-z0-9_-]*$") + +RULING_ID_RE = re.compile(r"^R-(\d{4})(\d{2})(\d{2})-(\d{2})$") +RULING_FILE_RE = re.compile(r"^rulings/(\d{4})/(\d{2})/(R-\d{8}-\d{2})\.md$") +PHASE_FILE_RE = re.compile(r"^phases/([a-z][a-z0-9_]*)/([^/]+)\.md$") +SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9_-]*$") +DATE_RE = re.compile(r"^\d{4}-\d{2}-\d{2}$") +WALL_RE = re.compile(r"^\d+:\d{2}$") +INT_RE = re.compile(r"^\d+$") +SLOT_RE = re.compile(r"^\d{4}-\d{2}-\d{2}-[A-Za-z0-9_-]+$") +PROJECT_KEY_RE = re.compile(r"^([a-z][a-z0-9_]*):\s*$") +PROJECT_FIELD_RE = re.compile(r"^ ([a-z_]+):(?:\s+(.*?))?\s*$") + +#: a light-header line: `Key: value` or a bare `Key:` (empty, or a list follows). +HEADER_KEY_RE = re.compile(r"^([A-Z][A-Za-z0-9-]*):(?:[ \t]+(.*?))?[ \t]*$") +HEADER_LINES = 30 + +#: a batch record's `- key: value` line and its ` - <slug>: …` member line. +RECORD_KEY_RE = re.compile(r"^- ([a-z][a-z0-9-]*):(?:\s+(.*?))?\s*$") +MEMBER_RE = re.compile( + r"^ - (?P<slug>[^:]+): (?P<path>\S+) — (?P<runs>.+?) — (?P<minutes>\S+) — (?P<state>\S+)$" +) +REVIEW_SECTION_RE = re.compile(r"^## (?P<slug>.+?) — (?P<health>[A-Z]+)$") +REVIEW_KEY_RE = re.compile(r"^- (decision|ruled): (.+?)\s*$") + + +class CortexError(Exception): + """A refused edit or an unusable input — printed, exit 1, nothing written.""" + + +def _dash(line: str) -> str: + """`--` is accepted for the em dash (phone keyboards); read both as one.""" + return line.replace(" -- ", " — ") + + +# --------------------------------------------------------------------------- # +# the light header +# --------------------------------------------------------------------------- # +def header_span(lines: "list[str]") -> "tuple[int, int]": + """(start, end) line indexes of the header block, end exclusive. + + The block starts at the first `Key:` line after the title (within the + first 30 lines) and ends at the next blank line; (1, 1) when there is + none.""" + start = None + for i in range(1, min(len(lines), HEADER_LINES)): + if HEADER_KEY_RE.match(lines[i]): + start = i + break + if start is None: + return 1, 1 + end = start + while end < min(len(lines), HEADER_LINES) and lines[end].strip(): + end += 1 + return start, end + + +def parse_header(text: str) -> "tuple[str | None, dict[str, str]]": + """(title, fields) — first occurrence of a key wins; a bare `Key:` followed + by `- item` lines is a list, joined with commas.""" + lines = text.split("\n") + title = lines[0][2:].strip() if lines and lines[0].startswith("# ") else None + fields: "dict[str, str]" = {} + start, end = header_span(lines) + last_key = None + for line in lines[start:end]: + m = HEADER_KEY_RE.match(line) + if m: + key, value = m.group(1), (m.group(2) or "").strip() + if key not in fields: + fields[key] = value + last_key = key + else: + last_key = None + continue + if line.startswith("- ") and last_key is not None: + item = line[2:].strip() + fields[last_key] = f"{fields[last_key]}, {item}" if fields[last_key] else item + return title, fields + + +def edit_header(text: str, updates: "dict[str, str | None]", order=PHASE_KEYS) -> str: + """Return `text` with header keys set (value) or removed (None), in place. + + Every other byte is preserved: a present key has only its value replaced; + a missing key is inserted at its canonical slot; the block, the body and + the line endings are untouched.""" + lines = text.split("\n") + start, end = header_span(lines) + if start == end and any(v is not None for v in updates.values()): + # No header block yet: open one after the title line. + insert_at = 1 + if len(lines) > 1 and lines[1].strip(): + lines.insert(1, "") + lines.insert(insert_at + 1, "") + start = end = insert_at + 1 + for key, value in updates.items(): + idx = next((i for i in range(start, end) if HEADER_KEY_RE.match(lines[i]) + and HEADER_KEY_RE.match(lines[i]).group(1) == key), None) + if value is None: + if idx is not None: + del lines[idx] + end -= 1 + continue + line = f"{key}: {value}" if value else f"{key}:" + if idx is not None: + lines[idx] = line + continue + rank = {k: i for i, k in enumerate(order)} + after = start - 1 + for i in range(start, end): + m = HEADER_KEY_RE.match(lines[i]) + if m and rank.get(m.group(1), -1) < rank.get(key, len(order)): + after = i + lines.insert(after + 1, line) + end += 1 + return "\n".join(lines) + + +def sections(text: str) -> "dict[str, tuple[int, int]]": + """{name: (first body line index, end index exclusive)} for `## name`.""" + lines = text.split("\n") + heads = [(i, ln[3:].strip()) for i, ln in enumerate(lines) if ln.startswith("## ")] + out = {} + for n, (i, name) in enumerate(heads): + stop = heads[n + 1][0] if n + 1 < len(heads) else len(lines) + out.setdefault(name, (i + 1, stop)) + return out + + +def append_to_section(text: str, name: str, new_lines: "list[str]", + replace_placeholder: "str | None" = None) -> str: + """Append lines at the end of `## name` (before the next heading), + keeping one blank line between the content and the next heading.""" + lines = text.split("\n") + span = sections(text).get(name) + if span is None: + raise CortexError(f"no `## {name}` section to write into") + body_start, stop = span + # last content line of the section + last = stop - 1 + while last >= body_start and not lines[last].strip(): + last -= 1 + if replace_placeholder is not None and last >= body_start \ + and lines[last].strip() == replace_placeholder \ + and all(not ln.strip() for ln in lines[body_start:last]): + lines[last:last + 1] = new_lines + return "\n".join(lines) + if last < body_start: + # An empty section: replace whatever blank lines it held with + # `blank, content, blank` so the next heading keeps its gap. + lines[body_start:stop] = [""] + new_lines + [""] + return "\n".join(lines) + lines[last + 1:last + 1] = new_lines + return "\n".join(lines) + + +# --------------------------------------------------------------------------- # +# projects.yaml — the restricted subset +# --------------------------------------------------------------------------- # +def _strip_comment(line: str) -> str: + """Drop a ` #…` comment; a `#` inside a quoted scalar is not a comment.""" + if line.lstrip().startswith("#"): + return "" + if '"' in line: + first = line.index('"') + close = line.find('"', first + 1) + if close != -1: + tail = line[close + 1:] + cut = tail.find(" #") + return line[:close + 1] + (tail if cut == -1 else tail[:cut]) + cut = line.find(" #") + return line if cut == -1 else line[:cut] + + +def _scalar(raw: str, where: str, problems: "list[str]") -> str: + raw = raw.strip() + if raw.startswith('"'): + if not (raw.endswith('"') and len(raw) >= 2) or '"' in raw[1:-1]: + problems.append(f"{where}: unterminated or nested quotes: {raw}") + return raw + return raw[1:-1] + if any(ch in raw for ch in "#:") or raw != raw.strip() or raw.startswith(("[", "{", "'", "&", "*", "!", "|", ">")): + problems.append(f"{where}: scalar needs quoting (contains `#`, `:` or a " + f"YAML indicator): {raw}") + return raw + + +def parse_projects(text: str) -> "tuple[dict[str, dict], list[str]]": + """Parse the restricted YAML subset into {key: {field: value}}. + + Unknown field, missing field, bad indentation, anything outside the + subset → a problem line (the map is still returned). A comment-only file + parses to an empty map.""" + rows: "dict[str, dict]" = {} + problems: "list[str]" = [] + current = None + for lineno, raw in enumerate(text.split("\n"), 1): + line = _strip_comment(raw).rstrip() + if not line.strip(): + continue + where = f"projects.yaml:{lineno}" + m = PROJECT_KEY_RE.match(line) + if m: + if current is not None: + _finish_row(current, rows[current], problems) + current = m.group(1) + if current in rows: + problems.append(f"{where}: duplicate project key {current}") + rows[current] = {} + continue + m = PROJECT_FIELD_RE.match(line) + if m and current is not None: + field, value = m.group(1), (m.group(2) or "") + if field not in PROJECT_FIELDS: + problems.append(f"{where}: unknown field `{field}` on {current}") + continue + if field in rows[current]: + problems.append(f"{where}: duplicate field `{field}` on {current}") + continue + if field == "sync_verbs": + v = value.strip() + if not (v.startswith("[") and v.endswith("]")): + problems.append(f"{where}: sync_verbs must be a flow list `[a, b]`") + continue + items = [x.strip() for x in v[1:-1].split(",") if x.strip()] + bad = [x for x in items if not re.match(r"^[a-z][a-z0-9_-]*$", x)] + if bad or (v[1:-1].strip() and not items): + problems.append(f"{where}: sync_verbs holds a non-bare word: {bad}") + rows[current][field] = items + else: + if not value.strip(): + problems.append(f"{where}: `{field}` has no value") + rows[current][field] = _scalar(value, where, problems) + continue + problems.append(f"{where}: outside the subset (a key is `^[a-z][a-z0-9_]*:$` " + f"at column 0, a field is two-space indented): {raw.strip()}") + if current is not None: + _finish_row(current, rows[current], problems) + return rows, problems + + +def _finish_row(key: str, row: dict, problems: "list[str]") -> None: + missing = [f for f in PROJECT_FIELDS if f not in row] + if missing: + problems.append(f"projects.yaml: {key} is missing {', '.join(missing)}") + for f, v in row.items(): + if isinstance(v, str) and not v.strip(): + problems.append(f"projects.yaml: {key}.{f} is empty") + if row.get("partition") and row["partition"] not in PARTITIONS: + problems.append(f"projects.yaml: {key}.partition must be gpu | ral | both, " + f"not {row['partition']}") + if row.get("status") and row["status"] not in PROJECT_STATUSES: + problems.append(f"projects.yaml: {key}.status must be active | dormant, " + f"not {row['status']}") + remote = row.get("remote") + if remote and remote != "none" and not re.match(r"^[\w.-]+/[\w.-]+$", remote): + problems.append(f"projects.yaml: {key}.remote must be owner/repo or none") + for f in ("local_path", "ral_root"): + if row.get(f) and not row[f].startswith("/"): + problems.append(f"projects.yaml: {key}.{f} must be an absolute path") + if row.get("mirror") and row["mirror"] != "none" and not row["mirror"].startswith("/"): + problems.append(f"projects.yaml: {key}.mirror must be an absolute path or none") + + +def load_projects(root: Path) -> "tuple[dict[str, dict], list[str]]": + path = root / "projects.yaml" + if not path.is_file(): + return {}, ["projects.yaml: missing"] + return parse_projects(path.read_text(encoding="utf-8")) + + +# --------------------------------------------------------------------------- # +# run lines +# --------------------------------------------------------------------------- # +class Run: + __slots__ = ("ident", "stem", "task", "tasks", "state", "partition", "date", + "wall", "note", "cont", "lineno") + + def __init__(self, m: "re.Match", lineno: int): + self.stem = m.group("stem") + self.task = m.group("task") + self.tasks = m.group("tasks") + self.ident = self.stem + (f"_{self.task}" if self.task else + f"_[{self.tasks}]" if self.tasks else "") + self.state = m.group("state") + self.partition = m.group("partition") + self.date = m.group("date") + self.wall = m.group("wall") + self.note = m.group("note") or "" + self.cont: "dict[str, str]" = {} + self.lineno = lineno + + def task_set(self) -> "set[int] | None": + """The tasks this line covers; None = the whole job (a bare stem).""" + if self.task: + return {int(self.task)} + if self.tasks is None: + return None + out: "set[int]" = set() + for part in self.tasks.split(","): + if "-" in part: + a, b = (int(x) for x in part.split("-")) + out.update(range(a, b + 1)) + else: + out.add(int(part)) + return out + + def ascending(self) -> bool: + if self.tasks is None: + return True + last = -1 + for part in self.tasks.split(","): + a, b = (int(x) for x in part.split("-")) if "-" in part else (int(part),) * 2 + if a <= last or b < a: + return False + last = b + return True + + +def parse_runs(text: str) -> "tuple[list[Run], list[str]]": + """Run lines of `## Runs`; a missing section = no runs (reported by the + section check, not here).""" + lines = text.split("\n") + span = sections(text).get("Runs") + runs: "list[Run]" = [] + problems: "list[str]" = [] + if span is None: + return runs, problems + for i in range(*span): + raw = lines[i] + if not raw.strip(): + continue + line = _dash(raw) + m = RUN_LINE_RE.match(line) + if m: + run = Run(m, i + 1) + if not run.ascending(): + problems.append(f"line {i + 1}: task set not ascending: {run.ident}") + runs.append(run) + continue + m = RUN_CONT_RE.match(line) + if m: + if not runs: + problems.append(f"line {i + 1}: continuation line without a run line") + else: + runs[-1].cont[m.group("key")] = m.group("value").strip() + continue + problems.append(f"line {i + 1}: run line does not parse: {raw.strip()}") + return runs, problems + + +def _overlaps(a: "set[int] | None", b: "set[int] | None") -> bool: + return a is None or b is None or bool(a & b) + + +def run_problems(runs: "list[Run]", header_runs: str, state: str, + ruling_ids: "set[str]") -> "list[str]": + problems: "list[str]" = [] + idents = {r.ident for r in runs} + for i, a in enumerate(runs): + for b in runs[i + 1:]: + if a.stem == b.stem and _overlaps(a.task_set(), b.task_set()): + problems.append(f"run {a.ident} overlaps {b.ident} on stem {a.stem}") + body_stems = {r.stem for r in runs} + listed = [s.strip() for s in header_runs.split(",") if s.strip()] + for s in listed: + if not INT_RE.match(s): + problems.append(f"Runs: '{s}' is not a job stem") + if set(listed) != body_stems: + problems.append(f"Runs: header {{{', '.join(sorted(listed))}}} != body stems " + f"{{{', '.join(sorted(body_stems))}}}") + if state == "pulled" and not any(r.state in ("done", "legacy") and "pulled_to" in r.cont + for r in runs): + problems.append("State: pulled needs at least one done | legacy run with pulled_to:") + for r in runs: + if r.state in LEGACY_RUN_STATES and "where" not in r.cont: + problems.append(f"run {r.ident} is {r.state} without where:") + for key in ("after", "resumes"): + target = r.cont.get(key) + if target is not None and (target not in idents or target == r.ident): + problems.append(f"run {r.ident} {key}: {target} names no other run of this phase") + ruled = r.cont.get("ruled") + if ruled is not None and ruled not in ruling_ids: + problems.append(f"run {r.ident} ruled: {ruled} does not resolve to a ruling file") + return problems + + +def gate_refs(value: str) -> "tuple[list[str], list[str]]": + """(refs, bad) from a comma-separated `Gates:` / `Follow-ups:` value.""" + refs, bad = [], [] + for token in (t.strip() for t in value.split(",")): + if not token: + continue + if GATE_REF_RE.fullmatch(token): + refs.append(token) + else: + bad.append(token) + return refs, bad + + +def gate_url(ref: str) -> str: + """Canonical issues URL for either GATE_REF_RE form (a PR *is* an issue).""" + m = GATE_REF_RE.fullmatch(ref) + owner, repo, num, short_repo, short_num = m.groups() + if owner: + return f"https://github.com/{owner}/{repo}/issues/{num}" + return f"https://github.com/{DEFAULT_GATE_OWNER}/{short_repo}/issues/{short_num}" + + +# --------------------------------------------------------------------------- # +# the tree +# --------------------------------------------------------------------------- # +class Phase: + def __init__(self, root: Path, path: Path): + self.path = path + self.rel = path.relative_to(root).as_posix() + self.text = path.read_text(encoding="utf-8") + self.title, self.fields = parse_header(self.text) + self.project_dir = path.parent.name + self.slug = path.stem + self.runs, self.run_parse_problems = parse_runs(self.text) + + def get(self, key: str) -> str: + return self.fields.get(key, "") + + @property + def state(self) -> str: + return self.get("State") + + +class Ruling: + def __init__(self, root: Path, path: Path): + self.path = path + self.rel = path.relative_to(root).as_posix() + self.text = path.read_text(encoding="utf-8") + self.title, self.fields = parse_header(self.text) + self.id = path.stem + + def get(self, key: str) -> str: + return self.fields.get(key, "") + + +def _md_files(d: Path): + return sorted(p for p in d.rglob("*.md")) if d.is_dir() else [] + + +def load_phases(root: Path) -> "tuple[list[Phase], list[str]]": + phases, problems = [], [] + for p in _md_files(root / "phases"): + rel = p.relative_to(root).as_posix() + if not PHASE_FILE_RE.match(rel): + problems.append(f"{rel}: not a phase path (phases/<project>/<slug>.md)") + continue + phases.append(Phase(root, p)) + return phases, problems + + +def load_rulings(root: Path) -> "tuple[list[Ruling], list[str]]": + rulings, problems = [], [] + for p in _md_files(root / "rulings"): + rel = p.relative_to(root).as_posix() + if rel == "rulings/AGENTS.md": + continue + m = RULING_FILE_RE.match(rel) + if not m: + problems.append(f"{rel}: not a ruling path (rulings/<YYYY>/<MM>/R-<YYYYMMDD>-<nn>.md)") + continue + yyyy, mm, rid = m.groups() + r = Ruling(root, p) + if rid[2:6] != yyyy or rid[6:8] != mm: + problems.append(f"{rel}: filed under {yyyy}/{mm} but the id is dated {rid[2:6]}-{rid[6:8]}") + rulings.append(r) + return rulings, problems + + +def ruling_files(root: Path) -> "dict[str, Path]": + """{id: path} of every ruling on disk (no validation).""" + out = {} + for p in _md_files(root / "rulings"): + if RULING_ID_RE.match(p.stem): + out.setdefault(p.stem, p) + return out + + +def batch_records(root: Path) -> "list[Path]": + d = root / "batches" + return sorted(p for p in d.glob("*.md") if p.name != "AGENTS.md") if d.is_dir() else [] + + +def batch_reviews(root: Path) -> "list[Path]": + d = root / "batches" / "reviews" + return sorted(p for p in d.glob("*.md") if p.name != "AGENTS.md") if d.is_dir() else [] + + +# --------------------------------------------------------------------------- # +# check +# --------------------------------------------------------------------------- # +def phase_problems(root: Path, phases: "list[Phase]", projects: "dict[str, dict]", + rulings: "dict[str, Ruling]", successors: "dict[str, list[str]]") -> "list[str]": + problems: "list[str]" = [] + numbers: "dict[tuple[str, str], str]" = {} + for ph in phases: + f = ph.fields + p = [f"{ph.rel}: {msg}" for msg in ph.run_parse_problems] + if ph.title is None: + p.append(f"{ph.rel}: line 1 is not `# <title>`") + for key in ("Project", "Phase", "State"): + if not f.get(key): + p.append(f"{ph.rel}: missing header key {key}:") + for key in f: + if key not in PHASE_KEYS: + p.append(f"{ph.rel}: unknown header key {key}:") + for name in PHASE_SECTIONS: + if name not in sections(ph.text): + p.append(f"{ph.rel}: missing section ## {name}") + project = f.get("Project", "") + if project and project != ph.project_dir: + p.append(f"{ph.rel}: Project: {project} is not the directory name {ph.project_dir}") + if project and project not in projects: + p.append(f"{ph.rel}: Project: {project} is not a projects.yaml key") + number = f.get("Phase", "") + if number and not INT_RE.match(number): + p.append(f"{ph.rel}: Phase: '{number}' is not an integer") + elif number: + prior = numbers.setdefault((ph.project_dir, number), ph.rel) + if prior != ph.rel: + p.append(f"{ph.rel}: duplicate Phase: {number} (also {prior})") + state = f.get("State", "") + if state and state not in PHASE_STATES: + p.append(f"{ph.rel}: State: '{state}' is not a phase state") + refs, bad = gate_refs(f.get("Gates", "")) + for token in bad: + p.append(f"{ph.rel}: Gates: unrecognised ref '{token}' — Repo#N or an " + f"issue/PR URL; no owner/Repo#N form") + for key in ("Gates-cleared", "Filed"): + if f.get(key) and not DATE_RE.match(f[key]): + p.append(f"{ph.rel}: {key}: '{f[key]}' is not YYYY-MM-DD") + if f.get("Budget") and not WALL_RE.match(f["Budget"]): + p.append(f"{ph.rel}: Budget: '{f['Budget']}' is not H+:MM") + if f.get("Review-minutes") and not INT_RE.match(f["Review-minutes"]): + p.append(f"{ph.rel}: Review-minutes: '{f['Review-minutes']}' is not an integer") + if f.get("Lane") and f["Lane"] != LANE: + p.append(f"{ph.rel}: Lane: '{f['Lane']}' is not {LANE}") + # the offline invariants on top of the table + if state == "gated" and not refs: + p.append(f"{ph.rel}: State: gated with an empty Gates:") + if state in GATES_SETTLED_STATES and refs and not (f.get("Gates-cleared") or f.get("Gate-override")): + p.append(f"{ph.rel}: State: {state} with Gates: needs Gates-cleared: or Gate-override:") + if state in WITNESS_STATES and not f.get("Witness"): + p.append(f"{ph.rel}: State: {state} needs a Witness:") + ruling_id = f.get("Ruling", "") + if state in RULED_STATES and not ruling_id: + p.append(f"{ph.rel}: State: {state} needs a Ruling: (reachable only through rule)") + head = None + if ruling_id: + if not RULING_ID_RE.match(ruling_id): + p.append(f"{ph.rel}: Ruling: '{ruling_id}' is not a ruling id") + elif ruling_id not in rulings: + p.append(f"{ph.rel}: Ruling: {ruling_id} does not resolve to a ruling file") + else: + head = rulings[ruling_id] + if successors.get(ruling_id): + p.append(f"{ph.rel}: Ruling: {ruling_id} is not a chain head " + f"(superseded by {', '.join(successors[ruling_id])})") + if head.get("Phase") != ph.rel: + p.append(f"{ph.rel}: Ruling: {ruling_id} names {head.get('Phase') or '(nothing)'}, not this phase") + verb = head.get("Ruling") + if state and verb in VERB_STATES and state not in VERB_STATES[verb]: + p.append(f"{ph.rel}: Ruling: {ruling_id} verb '{verb}' does not fit State: {state}") + if state == "pulled" and any(r.state in LIVE_RUN_STATES for r in ph.runs): + if head is None or head.get("Ruling") != "leave-to-finish": + p.append(f"{ph.rel}: State: pulled with a live run needs a leave-to-finish ruling head (--partial)") + p.extend(f"{ph.rel}: {msg}" for msg in + run_problems(ph.runs, f.get("Runs", ""), state, set(rulings))) + problems.extend(p) + return problems + + +def ruling_problems(root: Path, rulings: "list[Ruling]", phases_by_rel: "dict[str, Phase]", + by_id: "dict[str, Ruling]", successors: "dict[str, list[str]]") -> "list[str]": + problems: "list[str]" = [] + seen: "dict[str, str]" = {} + for r in rulings: + p: "list[str]" = [] + prior = seen.setdefault(r.id, r.rel) + if prior != r.rel: + p.append(f"{r.rel}: duplicate ruling id {r.id} (also {prior})") + if r.title is None or not (r.title == r.id or r.title.startswith(f"{r.id} — ")): + p.append(f"{r.rel}: title must be `# {r.id}` or `# {r.id} — <summary>`") + for key in ("Project", "Phase", "Ruling"): + if not r.get(key): + p.append(f"{r.rel}: missing header key {key}:") + for key in r.fields: + if key not in RULING_KEYS: + p.append(f"{r.rel}: unknown header key {key}:") + for name in RULING_SECTIONS: + if name not in sections(r.text): + p.append(f"{r.rel}: missing section ## {name}") + verb = r.get("Ruling") + if verb and verb not in RULING_VERBS: + p.append(f"{r.rel}: Ruling: '{verb}' is not a ruling verb") + phase_rel = r.get("Phase") + ph = phases_by_rel.get(phase_rel) + if phase_rel and ph is None: + p.append(f"{r.rel}: Phase: {phase_rel} does not resolve to a phase file") + if ph is not None and r.get("Project") and ph.get("Project") != r.get("Project"): + p.append(f"{r.rel}: Phase: {phase_rel} is not a phase of Project: {r.get('Project')}") + if ph is not None: + stems = {s.strip() for s in r.get("Runs").split(",") if s.strip()} + phase_stems = {x.stem for x in ph.runs} + if not stems <= phase_stems: + p.append(f"{r.rel}: Runs: {{{', '.join(sorted(stems - phase_stems))}}} not in the phase's runs") + sup = r.get("Supersedes") + if sup: + if sup == r.id: + p.append(f"{r.rel}: Supersedes: itself") + elif sup not in by_id: + p.append(f"{r.rel}: Supersedes: {sup} does not resolve to a ruling file") + else: + old = by_id[sup] + if not sup < r.id: + p.append(f"{r.rel}: Supersedes: {sup} is not earlier than {r.id}") + if (old.get("Project"), old.get("Phase")) != (r.get("Project"), r.get("Phase")): + p.append(f"{r.rel}: Supersedes: {sup} names a different project/phase") + if len(successors.get(r.id, [])) > 1: + p.append(f"{r.rel}: has {len(successors[r.id])} successors " + f"({', '.join(successors[r.id])}) — a chain, not a tree") + batch = r.get("Batch") + if batch and not (root / "batches" / f"{batch}.md").is_file(): + p.append(f"{r.rel}: Batch: {batch} names no batches/{batch}.md") + for key in ("Reviewed-at",): + pass + if r.get("Review-minutes-actual") and not INT_RE.match(r.get("Review-minutes-actual")): + p.append(f"{r.rel}: Review-minutes-actual: not an integer") + _, bad = gate_refs(r.get("Follow-ups")) + for token in bad: + p.append(f"{r.rel}: Follow-ups: unrecognised ref '{token}'") + problems.extend(p) + return problems + + +def _record_members(text: str) -> "tuple[list[tuple[int, re.Match | None, str]], dict[str, str]]": + """([(lineno, member match or None, raw)], {key: value}) of a batch record.""" + members, keys = [], {} + in_members = False + for i, raw in enumerate(text.split("\n"), 1): + m = RECORD_KEY_RE.match(raw) + if m: + in_members = m.group(1) == "members" + keys.setdefault(m.group(1), (m.group(2) or "").strip()) + continue + if in_members and raw.startswith(" - "): + members.append((i, MEMBER_RE.match(_dash(raw)), raw)) + elif raw.strip() and not raw.startswith(" ") and in_members: + in_members = False + return members, keys + + +def batch_problems(root: Path, phases_by_rel: "dict[str, Phase]") -> "list[str]": + problems: "list[str]" = [] + members_by_slot: "dict[str, set[str]]" = {} + for path in batch_records(root): + rel = path.relative_to(root).as_posix() + slot = path.stem + if not SLOT_RE.match(slot): + problems.append(f"{rel}: not a slot name (<YYYY-MM-DD>-<slot>.md)") + text = path.read_text(encoding="utf-8") + members, keys = _record_members(text) + slugs: "set[str]" = set() + for lineno, m, raw in members: + if m is None: + problems.append(f"{rel}: line {lineno}: member line does not parse: {raw.strip()}") + continue + slug, prel = m.group("slug"), m.group("path") + slugs.add(slug) + ph = phases_by_rel.get(prel) + if ph is None: + problems.append(f"{rel}: member {slug}: {prel} does not exist") + else: + if ph.slug != slug: + problems.append(f"{rel}: member slug '{slug}' != phase stem '{ph.slug}'") + runs = m.group("runs").strip() + stems = set() if runs == "none" else {s.strip() for s in runs.split(",") if s.strip()} + extra = stems - {x.stem for x in ph.runs} + if extra: + problems.append(f"{rel}: member {slug}: runs {{{', '.join(sorted(extra))}}} not in the phase's runs") + if not INT_RE.match(m.group("minutes")): + problems.append(f"{rel}: member {slug}: review-minutes '{m.group('minutes')}' is not an integer") + if m.group("state") not in PHASE_STATES: + problems.append(f"{rel}: member {slug}: state '{m.group('state')}' is not a phase state") + members_by_slot[slot] = slugs + review = keys.get("review") + if review and not (root / review).is_file(): + problems.append(f"{rel}: review: {review} does not exist") + for path in batch_reviews(root): + rel = path.relative_to(root).as_posix() + slot = path.stem + lines = path.read_text(encoding="utf-8").split("\n") + title = lines[0] if lines else "" + if title != f"# Batch review {slot}": + problems.append(f"{rel}: title must be `# Batch review {slot}`") + if slot not in members_by_slot: + problems.append(f"{rel}: no batch record batches/{slot}.md") + section = None + seen_keys: "dict[str, str]" = {} + for i, raw in enumerate(lines, 1): + line = _dash(raw) + if line.startswith("## "): + if section is not None: + _review_section_problems(rel, section, seen_keys, problems) + m = REVIEW_SECTION_RE.match(line) + if not m: + problems.append(f"{rel}: line {i}: section heading is not `## <slug> — <HEALTH>`: {raw.strip()}") + section, seen_keys = None, {} + continue + section, seen_keys = m.group("slug"), {} + if m.group("health") not in HEALTHS: + problems.append(f"{rel}: {section}: health '{m.group('health')}' not in {' | '.join(HEALTHS)}") + if slot in members_by_slot and section not in members_by_slot[slot]: + problems.append(f"{rel}: section '{section}' names no member of batches/{slot}.md") + continue + m = REVIEW_KEY_RE.match(line) + if m and section is not None: + seen_keys.setdefault(m.group(1), m.group(2)) + if section is not None: + _review_section_problems(rel, section, seen_keys, problems) + return problems + + +def _review_section_problems(rel: str, section: str, keys: "dict[str, str]", problems: "list[str]") -> None: + decision, ruled = keys.get("decision"), keys.get("ruled") + if decision is None or ruled is None: + problems.append(f"{rel}: {section}: needs `- decision:` and `- ruled:` lines") + return + if decision not in RULING_VERBS and decision != "(none)": + problems.append(f"{rel}: {section}: decision '{decision}' is not a ruling verb or (none)") + if ruled not in ("yes", "no"): + problems.append(f"{rel}: {section}: ruled '{ruled}' is not yes | no") + if ruled == "yes" and decision == "(none)": + problems.append(f"{rel}: {section}: ruled: yes needs a verb, not (none)") + + +def check_problems(root: Path) -> "list[str]": + """Every `check` rule, over the tree at `root`. Hermetic.""" + projects, problems = load_projects(root) + phases, stray = load_phases(root) + problems.extend(stray) + rulings, stray = load_rulings(root) + problems.extend(stray) + by_id: "dict[str, Ruling]" = {} + for r in rulings: + by_id.setdefault(r.id, r) + successors: "dict[str, list[str]]" = {} + for r in rulings: + if r.get("Supersedes"): + successors.setdefault(r.get("Supersedes"), []).append(r.id) + phases_by_rel = {ph.rel: ph for ph in phases} + problems.extend(phase_problems(root, phases, projects, by_id, successors)) + problems.extend(ruling_problems(root, rulings, phases_by_rel, by_id, successors)) + problems.extend(batch_problems(root, phases_by_rel)) + return problems + + +def cmd_check(args) -> int: + problems = check_problems(args.root) + if problems: + print("cortex check: DRIFT") + for p in problems: + print(f" - {p}") + return 1 + print("cortex check: OK") + return 0 + + +# --------------------------------------------------------------------------- # +# gates +# --------------------------------------------------------------------------- # +GATE_JQ = '{state, state_reason, merged_at: .pull_request.merged_at, is_pr: (.pull_request != null)}' + + +def _http_gate(owner: str, repo: str, num: str): + """One GitHub REST read via stdlib urllib — the gh-free fallback path. + Returns the same dict `GATE_JQ` shapes, or an `unreadable: …` string.""" + import os + import urllib.error + import urllib.request + + req = urllib.request.Request( + f"https://api.github.com/repos/{owner}/{repo}/issues/{num}", + headers={"Accept": "application/vnd.github+json", "User-Agent": "pyautocortex"}, + ) + token = os.environ.get("GITHUB_TOKEN") or os.environ.get("GH_TOKEN") + if token: + req.add_header("Authorization", f"Bearer {token}") + try: + with urllib.request.urlopen(req, timeout=30) as resp: + data = json.load(resp) + except urllib.error.HTTPError as e: + return f"unreadable: HTTP {e.code}" + except Exception as e: # URLError, timeout, bad JSON — report, don't crash + return f"unreadable: {e.__class__.__name__}" + pr = data.get("pull_request") + return {"state": data.get("state"), "state_reason": data.get("state_reason"), + "merged_at": pr.get("merged_at") if isinstance(pr, dict) else None, + "is_pr": pr is not None} + + +def fetch_gate_states(urls: "list[str]") -> "dict[str, dict | str]": + """{url: {state, state_reason, merged_at, is_pr} | 'unreadable: …'} via + `gh api`, falling back to urllib when gh is absent.""" + import subprocess + + out: "dict[str, dict | str]" = {} + gh_missing = False + for url in urls: + m = re.match(r"https://github\.com/([\w.-]+)/([\w.-]+)/issues/(\d+)$", url) + if not m: + out[url] = "unreadable: not a canonical issues URL" + continue + owner, repo, num = m.groups() + if not gh_missing: + try: + r = subprocess.run( + ["gh", "api", f"repos/{owner}/{repo}/issues/{num}", "--jq", GATE_JQ], + capture_output=True, text=True, + ) + except FileNotFoundError: + gh_missing = True + else: + if r.returncode != 0: + tail = (r.stderr.strip().splitlines() or ["error"])[-1] + out[url] = f"unreadable: {tail}" + continue + try: + out[url] = json.loads(r.stdout) + except ValueError: + out[url] = "unreadable: gh returned no JSON" + continue + out[url] = _http_gate(owner, repo, num) + return out + + +def grade_gate(info) -> "tuple[str, str]": + """('cleared' | 'dead' | 'open' | 'unreadable', detail) for one ref.""" + if not isinstance(info, dict): + return "unreadable", str(info) + state = info.get("state") + if info.get("is_pr"): + if info.get("merged_at"): + return "cleared", "PR merged" + if state == "closed": + return "dead", "PR closed without merging" + if state == "open": + return "open", "PR open" + return "unreadable", f"PR state {state!r}" + if state == "closed": + reason = info.get("state_reason") + if reason in (None, "completed"): + return "cleared", "issue closed" + (f" ({reason})" if reason else "") + return "dead", f"issue closed as {reason}" + if state == "open": + return "open", "issue open" + return "unreadable", f"issue state {state!r}" + + +def gates_report(root: Path, *, grade: bool = False, write: bool = False, + fetch=None, today: "date | None" = None) -> "tuple[list[str], int]": + """(report lines, exit code). Offline: every `gated` phase and its refs. + `grade`: fetch each ref and give a verdict per phase; `write` flips + gated → ready (writing Gates-cleared:) and ready → gated on a reopen.""" + phases, _ = load_phases(root) + today = today or date.today() + lines: "list[str]" = [] + rc = 0 + wanted = [ph for ph in phases if (ph.state in GRADED_STATES if grade else ph.state == "gated") + and gate_refs(ph.get("Gates"))[0]] + if not wanted: + lines.append("gates: no gated phase" if not grade else "gates: no phase with gates to grade") + return lines, 0 + states: "dict[str, dict | str]" = {} + if grade: + fetch = fetch or fetch_gate_states + urls = sorted({gate_url(ref) for ph in wanted for ref in gate_refs(ph.get("Gates"))[0]}) + states = fetch(urls) + lines.append(f"gates: {len(wanted)} phase(s)") + for ph in wanted: + refs = gate_refs(ph.get("Gates"))[0] + if not grade: + lines.append(f" {ph.rel}: {ph.state} — {', '.join(refs)}") + continue + lines.append(f" {ph.rel}: {ph.state}") + verdicts = [] + for ref in refs: + v, detail = grade_gate(states.get(gate_url(ref), "unreadable: not fetched")) + verdicts.append(v) + lines.append(f" {ref} → {v} ({detail})") + if "unreadable" in verdicts: + rc = 1 + lines.append(" verdict: unreadable ref — fails closed, nothing flipped") + continue + if ph.state == "gated": + if all(v == "cleared" for v in verdicts): + if write: + ph.path.write_text(edit_header(ph.text, {"State": "ready", + "Gates-cleared": today.isoformat()}), + encoding="utf-8") + lines.append(f" verdict: cleared — gated → ready, Gates-cleared: {today.isoformat()}") + else: + lines.append(" verdict: cleared (gates --grade --write flips it to ready)") + elif "dead" in verdicts: + lines.append(" verdict: dead gate — will never clear; re-gate or move --override") + else: + lines.append(" verdict: waiting") + continue + reopened = [ref for ref, v in zip(refs, verdicts) if v == "open"] + if not reopened: + if "dead" in verdicts: + lines.append(" verdict: dead gate (reported; the phase has left gated)") + else: + lines.append(" verdict: still cleared") + continue + if ph.state != "ready": + lines.append(f" verdict: reopened {', '.join(reopened)} — reported, never enforced past ready") + elif ph.get("Gate-override"): + lines.append(f" verdict: reopened {', '.join(reopened)} — Gate-override: present, kept ready") + elif write: + ph.path.write_text(edit_header(ph.text, {"State": "gated", "Gates-cleared": None}), + encoding="utf-8") + lines.append(f" verdict: reopened {', '.join(reopened)} — ready → gated") + else: + lines.append(f" verdict: reopened {', '.join(reopened)} (gates --grade --write demotes it to gated)") + return lines, rc + + +def cmd_gates(args) -> int: + lines, rc = gates_report(args.root, grade=args.grade or args.write, write=args.write, + today=_today(args)) + print("\n".join(lines)) + return rc + + +# --------------------------------------------------------------------------- # +# shared helpers for the writing verbs +# --------------------------------------------------------------------------- # +def _today(args) -> date: + if getattr(args, "today", None): + try: + return date.fromisoformat(args.today) + except ValueError: + raise CortexError(f"--today must be YYYY-MM-DD, not {args.today}") + return date.today() + + +def _phase_at(root: Path, ref: str) -> Phase: + """The phase file `ref` names — repo-relative, or a path under root.""" + p = Path(ref) + path = p if p.is_absolute() else root / p + if not path.is_file(): + raise CortexError(f"no phase file at {ref}") + try: + path = path.resolve() + path.relative_to(root.resolve()) + except ValueError: + raise CortexError(f"{ref} is outside {root}") + rel_path = root.resolve() / path.relative_to(root.resolve()) + if not PHASE_FILE_RE.match(path.relative_to(root.resolve()).as_posix()): + raise CortexError(f"{ref} is not phases/<project>/<slug>.md") + return Phase(root.resolve(), rel_path) + + +def _partition_for(root: Path, project: str, given: "str | None") -> str: + if given: + if not PARTITION_RE.match(given): + raise CortexError(f"--partition '{given}' is not a bare partition name") + return given + projects, problems = load_projects(root) + row = projects.get(project) + if row is None: + raise CortexError(f"project {project} is not a projects.yaml key") + part = row.get("partition") + if part in ("gpu", "ral"): + return part + raise CortexError(f"project {project} may use either partition (`both`) — pass --partition") + + +def _run_line(ident: str, state: str, partition: str, day: date, wall: str = "0:00", + note: str = "") -> str: + line = f"- {ident}: {state} — {partition} — submitted {day.isoformat()} — wall {wall}" + return f"{line} — {note}" if note else line + + +def _append_run(ph: Phase, ident: str, state: str, partition: str, day: date, *, + note: str = "", cont: "dict[str, str] | None" = None) -> str: + """Phase text with one run line (and its continuations) appended to + `## Runs` and the `Runs:` header re-derived from the body.""" + if not RUN_IDENT_RE.match(ident): + raise CortexError(f"--run '{ident}' is not <stem>[_<task>|_[<set>]]") + probe = Run(RUN_LINE_RE.match(_run_line(ident, state, partition, day)), 0) + for r in ph.runs: + if r.stem == probe.stem and _overlaps(r.task_set(), probe.task_set()): + raise CortexError(f"run {ident} overlaps {r.ident} already on this phase") + idents = {r.ident for r in ph.runs} + for key, value in (cont or {}).items(): + if key in ("after", "resumes") and value not in idents: + raise CortexError(f"{key}: {value} names no run of this phase") + new_lines = [_run_line(ident, state, partition, day, note=note)] + new_lines += [f" {k}: {v}" for k, v in (cont or {}).items()] + text = append_to_section(ph.text, "Runs", new_lines) + stems = [] + for r in ph.runs + [probe]: + if r.stem not in stems: + stems.append(r.stem) + return edit_header(text, {"Runs": ", ".join(stems)}) + + +def _add_pulled_to(ph: Phase, text: str, pulled_to: "str | None", states: "set[str]") -> str: + """Give every run in `states` a `pulled_to:` it lacks — `pulled_to` if + given, else the run's own `where:` (a legacy run's quarantine path is + where its results already are). Refuse when no run would carry one, so + `move pulled` never writes a state `check` rejects.""" + targets = [r for r in ph.runs if r.state in states] + if not targets: + raise CortexError(f"no {' | '.join(sorted(states))} run to pull — nothing to review " + "(rule drop, or fix the run lines)") + lacking = [r for r in targets if "pulled_to" not in r.cont] + if not lacking or (not pulled_to and all("where" not in r.cont for r in lacking)): + if any("pulled_to" in r.cont for r in targets): + return text + raise CortexError("no run carries pulled_to: — pass --pulled-to <laptop path>") + lines = text.split("\n") + for r in sorted(lacking, key=lambda x: x.lineno, reverse=True): + path = pulled_to or r.cont.get("where") + if not path: + continue + i = r.lineno # 1-based run line → index of the line after it + while i < len(lines) and RUN_CONT_RE.match(_dash(lines[i])): + i += 1 + lines.insert(i, f" pulled_to: {path}") + return "\n".join(lines) + + +# --------------------------------------------------------------------------- # +# move +# --------------------------------------------------------------------------- # +def move_phase(root: Path, ref: str, to: str, *, run: "str | None" = None, + reason: "str | None" = None, override: "str | None" = None, + partial: bool = False, partition: "str | None" = None, + after: "str | None" = None, resumes: "str | None" = None, + note: str = "", pulled_to: "str | None" = None, + today: "date | None" = None) -> str: + """Apply one edge of the transition table; return a one-line summary.""" + today = today or date.today() + ph = _phase_at(root, ref) + cur = ph.state + if cur not in PHASE_STATES: + raise CortexError(f"{ph.rel} has State: '{cur}', which is not a phase state") + if to not in PHASE_STATES: + raise CortexError(f"'{to}' is not a phase state ({' | '.join(PHASE_STATES)})") + if cur == "dropped": + raise CortexError(f"{ph.rel} is dropped — terminal; revival is a new phase number") + if to == "dropped": + raise CortexError("dropped is a ruling edge: `cortex.py rule <phase> drop`") + if to in ("accepted", "rerun"): + raise CortexError(f"{to} is a ruling edge: `cortex.py rule <phase> {VERB_FOR_STATE[to]}`" + + (" --supersedes <Ruling:>" if cur == "accepted" else "")) + refs, bad = gate_refs(ph.get("Gates")) + if bad: + raise CortexError(f"{ph.rel}: Gates: unrecognised ref '{bad[0]}' — fix it first") + updates: "dict[str, str | None]" = {} + text = ph.text + cont = {} + if after: + cont["after"] = after + if resumes: + cont["resumes"] = resumes + + if cur == to: + if cur in ("submitted", "running") and run: + part = _partition_for(root, ph.project_dir, partition) + text = _append_run(ph, run, "submitted", part, today, note=note, cont=cont) + ph.path.write_text(text, encoding="utf-8") + return f"{ph.rel}: {cur} — appended run {run}" + raise CortexError(f"{ph.rel} is already {cur}" + + ("" if cur in ("submitted", "running") else + "; --run is for submitted | running phases")) + edge = (cur, to) + if run and edge not in (("ready", "submitted"), ("submitted", "running")): + raise CortexError("--run applies to ready → submitted and to a submitted | running phase") + if edge == ("planned", "gated"): + if not refs: + raise CortexError(f"{ph.rel}: Gates: is empty — planned → ready") + elif edge == ("planned", "ready"): + if refs: + raise CortexError(f"{ph.rel}: Gates: is non-empty — planned → gated") + elif edge == ("gated", "ready"): + if not override: + raise CortexError("gated → ready is `gates --grade --write` when the refs clear, " + "or `move ready --override \"<reason>\"`") + updates["Gate-override"] = override + elif edge == ("ready", "gated"): + raise CortexError("ready → gated is written by `gates --grade --write` only, " + "when a cleared gate reopened") + elif edge == ("ready", "submitted"): + if not ph.get("Witness"): + raise CortexError(f"{ph.rel}: Witness: is empty — register the witness before submitting") + if not run: + raise CortexError("ready → submitted needs --run <id>") + part = _partition_for(root, ph.project_dir, partition) + text = _append_run(ph, run, "submitted", part, today, note=note, cont=cont) + elif edge == ("ready", "pulled"): + if not ph.runs or not all(r.state in LEGACY_RUN_STATES for r in ph.runs): + raise CortexError("ready → pulled is for a legacy-born phase: every run line " + "legacy | legacy_wrong") + if not ph.get("Witness"): + raise CortexError(f"{ph.rel}: Witness: is empty — still mandatory for a legacy-born phase") + text = _add_pulled_to(ph, text, pulled_to, {"legacy"}) + elif edge == ("submitted", "running"): + if run: + part = _partition_for(root, ph.project_dir, partition) + text = _append_run(ph, run, "submitted", part, today, note=note, cont=cont) + elif edge in (("submitted", "ready"), ("running", "ready")): + if any(r.state in LIVE_RUN_STATES for r in ph.runs): + raise CortexError(f"{ph.rel}: a run is still submitted | running") + if not any(r.state in RESET_RUN_STATES for r in ph.runs): + raise CortexError(f"{ph.rel}: no failed | timeout | void run to reset from") + if not reason: + raise CortexError(f"{cur} → ready needs --reason") + updates["Reset"] = reason + elif edge == ("running", "pulled"): + if any(r.state in LIVE_RUN_STATES for r in ph.runs) and not partial: + raise CortexError(f"{ph.rel}: a run is still submitted | running — " + "--partial for a partial array (then rule leave-to-finish)") + text = _add_pulled_to(ph, text, pulled_to, {"done"}) + elif edge in (("pulled", "awaiting-ruling"), ("rerun", "ready")): + pass + else: + raise CortexError(f"no edge {cur} → {to} in the transition table") + updates["State"] = to + ph.path.write_text(edit_header(text, updates), encoding="utf-8") + return f"{ph.rel}: {cur} → {to}" + + +VERB_FOR_STATE = {"accepted": "accept", "rerun": "rerun", "dropped": "drop"} + + +def cmd_move(args) -> int: + print(move_phase(args.root, args.phase, args.state, run=args.run, reason=args.reason, + override=args.override, partial=args.partial, partition=args.partition, + after=args.after, resumes=args.resumes, note=args.note or "", + pulled_to=args.pulled_to, today=_today(args))) + return 0 + + +# --------------------------------------------------------------------------- # +# rule +# --------------------------------------------------------------------------- # +def next_ruling_id(root: Path, day: date, taken: "set[str] | None" = None) -> str: + stamp = day.strftime("%Y%m%d") + existing = set(ruling_files(root)) | (taken or set()) + for n in range(1, 100): + rid = f"R-{stamp}-{n:02d}" + if rid not in existing: + return rid + raise CortexError(f"99 rulings already filed on {day.isoformat()}") + + +def _ruling_target(ph: Phase, verb: str, supersedes: "str | None", by_id: "dict[str, Ruling]", + successors: "dict[str, list[str]]") -> "str | None": + """The phase state `verb` writes, or raise if the table forbids it.""" + state = ph.state + if verb not in RULING_VERBS: + raise CortexError(f"'{verb}' is not a ruling verb ({' | '.join(RULING_VERBS)})") + if state == "dropped": + raise CortexError(f"{ph.rel} is dropped — terminal") + if supersedes: + if supersedes not in by_id: + raise CortexError(f"--supersedes {supersedes} does not resolve to a ruling file") + old = by_id[supersedes] + if old.get("Phase") != ph.rel: + raise CortexError(f"--supersedes {supersedes} rules on {old.get('Phase')}, not {ph.rel}") + if successors.get(supersedes): + raise CortexError(f"{supersedes} already has a successor " + f"({successors[supersedes][0]}) — supersede the head") + if state == "accepted": + if verb not in ("rerun", "drop"): + raise CortexError(f"accepted takes only rerun | drop, with --supersedes {ph.get('Ruling')}") + if supersedes != ph.get("Ruling"): + raise CortexError(f"accepted → {VERB_TARGET[verb]} needs --supersedes {ph.get('Ruling')} " + "(the REWIND case)") + return VERB_TARGET[verb] + if verb == "leave-to-finish": + if state not in ("running", "pulled", "awaiting-ruling"): + raise CortexError(f"leave-to-finish applies to running | pulled | awaiting-ruling, " + f"not {state}") + return None + if state == "awaiting-ruling": + return VERB_TARGET[verb] + if verb == "drop": + return "dropped" + raise CortexError(f"'{verb}' is not allowed from State: {state} " + f"(the table: awaiting-ruling → {VERB_TARGET[verb]})") + + +def rule_phase(root: Path, ref: str, verb: str, body: str, *, supersedes: "str | None" = None, + batch: "str | None" = None, minutes: "int | None" = None, + follow_ups: "tuple[str, ...]" = (), also: "tuple[str, ...]" = (), + today: "date | None" = None, now: "datetime | None" = None) -> "list[str]": + """File one ruling per phase (the primary plus every --also), update each + phase's `Ruling:` and `State:`; return the ruling paths written. + Everything is validated before anything is written.""" + today = today or date.today() + now = now or datetime.now(timezone.utc) + root = root.resolve() + rulings, _ = load_rulings(root) + by_id = {r.id: r for r in rulings} + successors: "dict[str, list[str]]" = {} + for r in rulings: + if r.get("Supersedes"): + successors.setdefault(r.get("Supersedes"), []).append(r.id) + if batch and not (root / "batches" / f"{batch}.md").is_file(): + raise CortexError(f"--batch {batch} names no batches/{batch}.md") + _, bad = gate_refs(", ".join(follow_ups)) + if bad: + raise CortexError(f"--follow-up '{bad[0]}' is not Repo#N or an issue/PR URL " + "(create the issue first)") + body = body.strip("\n") + if not body.strip(): + raise CortexError("--body is empty — the human's words, verbatim") + + plan = [] + seen = set() + for i, target_ref in enumerate((ref,) + tuple(also)): + ph = _phase_at(root, target_ref) + if ph.rel in seen: + raise CortexError(f"{ph.rel} named twice") + seen.add(ph.rel) + sup = supersedes if i == 0 else (ph.get("Ruling") if ph.state == "accepted" else None) + new_state = _ruling_target(ph, verb, sup, by_id, successors) + plan.append((ph, sup, new_state)) + + taken: "set[str]" = set() + written = [] + for ph, sup, new_state in plan: + rid = next_ruling_id(root, today, taken) + taken.add(rid) + if sup and not sup < rid: + raise CortexError(f"{sup} is not earlier than the new id {rid} (--today?)") + path = root / "rulings" / rid[2:6] / rid[6:8] / f"{rid}.md" + if path.exists(): + raise CortexError(f"refusing to touch an existing ruling: {path.relative_to(root)}") + plan_item = (ph, sup, new_state, rid, path) + written.append(plan_item) + + stamp = f"{today.isoformat()}T{now.strftime('%H:%M')}Z" + out = [] + for ph, sup, new_state, rid, path in written: + header = [f"Project: {ph.get('Project')}", f"Phase: {ph.rel}", + f"Runs: {ph.get('Runs')}" if ph.get("Runs") else "Runs:", + f"Ruling: {verb}"] + if sup: + header.append(f"Supersedes: {sup}") + if batch: + header.append(f"Batch: {batch}") + header.append(f"Reviewed-at: {stamp}") + if minutes is not None: + header.append(f"Review-minutes-actual: {minutes}") + if follow_ups: + header.append(f"Follow-ups: {', '.join(follow_ups)}") + evidence = _where_to_look(ph) or ["- (none given)"] + text = "\n".join([f"# {rid} — {verb} {ph.get('Project')} phase {ph.get('Phase')}", ""] + + header + ["", "## Ruling", "", body, "", "## Evidence", ""] + + evidence) + "\n" + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text, encoding="utf-8") + updates: "dict[str, str | None]" = {"Ruling": rid} + if new_state: + updates["State"] = new_state + line = f"{rid} — {verb}" + (f" (supersedes {sup})" if sup else "") + ptext = append_to_section(ph.text, "Ruling", [line], replace_placeholder="(none)") + ph.path.write_text(edit_header(ptext, updates), encoding="utf-8") + out.append(path.relative_to(root).as_posix()) + return out + + +def _where_to_look(ph: Phase) -> "list[str]": + span = sections(ph.text).get("Where to look") + if span is None: + return [] + lines = ph.text.split("\n")[span[0]:span[1]] + return [ln for ln in lines if ln.startswith("- ") and ln.strip() != "-"] + + +def cmd_rule(args) -> int: + body_path = Path(args.body) + if not body_path.is_file(): + raise CortexError(f"--body {args.body} is not a file") + paths = rule_phase(args.root, args.phase, args.verb, body_path.read_text(encoding="utf-8"), + supersedes=args.supersedes, batch=args.batch, minutes=args.minutes, + follow_ups=tuple(args.follow_up or ()), also=tuple(args.also or ()), + today=_today(args)) + for p in paths: + print(f"wrote {p}") + return 0 + + +# --------------------------------------------------------------------------- # +# new +# --------------------------------------------------------------------------- # +def new_phase(root: Path, project: str, slug: str, number: int, *, gates: str = "", + epic: str = "", legacy_runs: "tuple[str, ...]" = (), + legacy_wrong: "tuple[str, ...]" = (), where: "str | None" = None, + partition: "str | None" = None, witness: str = "", budget: str = "", + minutes: "int | None" = None, title: "str | None" = None, + today: "date | None" = None) -> str: + """Write phases/<project>/<slug>.md from the template; return its rel path.""" + today = today or date.today() + projects, problems = load_projects(root) + if project not in projects: + raise CortexError(f"project {project} is not a projects.yaml key" + + (f" ({problems[0]})" if problems else "")) + if not SLUG_RE.match(slug): + raise CortexError(f"slug '{slug}' must match {SLUG_RE.pattern}") + if number < 1: + raise CortexError("--phase must be a positive integer") + path = root / "phases" / project / f"{slug}.md" + if path.exists(): + raise CortexError(f"{path.relative_to(root).as_posix()} already exists") + for existing in sorted((root / "phases" / project).glob("*.md")) if (root / "phases" / project).is_dir() else []: + _, fields = parse_header(existing.read_text(encoding="utf-8")) + if fields.get("Phase") == str(number): + raise CortexError(f"phase {number} already exists: {existing.relative_to(root).as_posix()}") + _, bad = gate_refs(gates) + if bad: + raise CortexError(f"--gates '{bad[0]}' is not Repo#N or an issue/PR URL (no owner/Repo#N form)") + legacy = [(r, "legacy") for r in legacy_runs] + [(r, "legacy_wrong") for r in legacy_wrong] + if legacy: + if gates.strip(): + raise CortexError("a legacy-born phase cannot be gated — its runs already happened") + if not where: + raise CortexError("--legacy-run needs --where <quarantine path> (check requires where:)") + for ident, _ in legacy: + if not RUN_IDENT_RE.match(ident): + raise CortexError(f"legacy run '{ident}' is not <stem>[_<task>|_[<set>]]") + part = _partition_for(root, project, partition) + if budget and not WALL_RE.match(budget): + raise CortexError(f"--budget '{budget}' is not H+:MM") + state = "ready" if legacy else "planned" + stems = [] + for ident, _ in legacy: + stem = ident.split("_")[0] + if stem not in stems: + stems.append(stem) + words = slug.replace("_", " ").replace("-", " ") + title = title or words + header = [ + f"Project: {project}", f"Phase: {number}", f"State: {state}", + f"Gates: {gates.strip()}" if gates.strip() else "Gates:", + f"Witness: {witness}" if witness else "Witness:", + f"Budget: {budget}" if budget else "Budget:", + f"Runs: {', '.join(stems)}" if stems else "Runs:", + "Ruling:", f"Lane: {LANE}", + f"Review-minutes: {minutes}" if minutes is not None else "Review-minutes:", + f"Epic: {epic}" if epic else "Epic:", f"Filed: {today.isoformat()}", + ] + run_lines = [] + for ident, rstate in legacy: + run_lines.append(_run_line(ident, rstate, part, today, note="pre-Cortex run, migrated")) + run_lines.append(f" where: {where}") + body = [ + f"# {project.capitalize()} — phase {number}: {title}", "", + *header, "", + "## Question", "", "(the question this phase answers)", "", + "## Witness", "", witness or "(not yet registered — a planned phase may leave this empty)", "", + "## Where to look", "", "- (the output path, once there is one)", "", + "## Runs", "", + ] + if run_lines: + body += run_lines + [""] + body += ["## Ruling", "", "(none)", ""] + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("\n".join(body), encoding="utf-8") + return path.relative_to(root).as_posix() + + +def cmd_new(args) -> int: + rel = new_phase(args.root, args.project, args.slug, args.phase, gates=args.gates or "", + epic=args.epic or "", legacy_runs=tuple(args.legacy_run or ()), + legacy_wrong=tuple(args.legacy_wrong or ()), where=args.where, + partition=args.partition, witness=args.witness or "", + budget=args.budget or "", minutes=args.minutes, title=args.title, + today=_today(args)) + print(f"wrote {rel}") + return 0 + + +# --------------------------------------------------------------------------- # +# main +# --------------------------------------------------------------------------- # +def _common(p: argparse.ArgumentParser, dated: bool = False) -> None: + p.add_argument("--root", type=Path, default=ROOT, + help="the Cortex tree to operate on (default: this checkout)") + if dated: + p.add_argument("--today", help="the date to write (YYYY-MM-DD; default: today)") + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="cortex.py", description=__doc__.split("\n\n")[0], + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = parser.add_subparsers(dest="command", required=True) + + c = sub.add_parser("check", help="every rule in REFERENCE.md; exit 1 on drift") + _common(c) + c.set_defaults(func=cmd_check) + + g = sub.add_parser("gates", help="list gated phases and their refs; --grade asks GitHub") + _common(g, dated=True) + g.add_argument("--grade", action="store_true", help="fetch each ref and give a verdict") + g.add_argument("--write", action="store_true", + help="flip gated → ready (Gates-cleared:) and ready → gated on a reopen") + g.set_defaults(func=cmd_gates) + + r = sub.add_parser("rule", help="file a ruling and move the phase per the table") + _common(r, dated=True) + r.add_argument("phase", help="phases/<project>/<slug>.md") + r.add_argument("verb", choices=RULING_VERBS) + r.add_argument("--body", required=True, help="file holding the human's words, verbatim") + r.add_argument("--supersedes", help="the ruling id this one replaces (the chain head)") + r.add_argument("--batch", help="the <YYYY-MM-DD>-<slot> the ruling was filed from") + r.add_argument("--minutes", type=int, help="Review-minutes-actual") + r.add_argument("--follow-up", action="append", metavar="REF", + help="Repo#N or an issue/PR URL; repeatable; the issue exists already") + r.add_argument("--also", action="append", metavar="PHASE", + help="fan the same body and batch out to another phase (a REWIND); " + "an accepted --also phase supersedes its own Ruling:") + r.set_defaults(func=cmd_rule) + + m = sub.add_parser("move", help="one edge of the transition table") + _common(m, dated=True) + m.add_argument("phase", help="phases/<project>/<slug>.md") + m.add_argument("state", help="the state to move to") + m.add_argument("--run", metavar="ID", help="SLURM job id (<stem>[_<task>|_[<set>]]) to append") + m.add_argument("--reason", help="why a submitted | running phase goes back to ready (Reset:)") + m.add_argument("--override", metavar="REASON", help="gated → ready without the gates (Gate-override:)") + m.add_argument("--partial", action="store_true", + help="running → pulled with a run still live (needs a leave-to-finish ruling)") + m.add_argument("--partition", help="the run line's partition (default: the project's row)") + m.add_argument("--after", metavar="RUN", help="afterok dependency of the appended run") + m.add_argument("--resumes", metavar="RUN", help="the run the appended run resumes from") + m.add_argument("--note", help="free text after the appended run line's fourth dash") + m.add_argument("--pulled-to", metavar="PATH", + help="→ pulled: the laptop path written as pulled_to: on each done | legacy " + "run lacking one (default for a legacy run: its where:)") + m.set_defaults(func=cmd_move) + + n = sub.add_parser("new", help="write phases/<project>/<slug>.md from the template") + _common(n, dated=True) + n.add_argument("project") + n.add_argument("slug") + n.add_argument("--phase", type=int, required=True, help="the phase number (unique per project)") + n.add_argument("--gates", help="comma-separated Repo#N or issue/PR URLs") + n.add_argument("--epic", help="the epic slug shared with the Mind") + n.add_argument("--legacy-run", action="append", metavar="ID", help="a pre-Cortex run, reusable") + n.add_argument("--legacy-wrong", action="append", metavar="ID", help="a pre-Cortex run, not reusable") + n.add_argument("--where", help="quarantine path written to each legacy run's where:") + n.add_argument("--partition", help="the legacy runs' partition (default: the project's row)") + n.add_argument("--witness", help="the pre-registered checkable claim") + n.add_argument("--budget", help="wall budget per run, H+:MM") + n.add_argument("--minutes", type=int, help="Review-minutes seed") + n.add_argument("--title", help="the title after `phase <n>:` (default: the slug's words)") + n.set_defaults(func=cmd_new) + return parser + + +def main(argv=None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + if not args.root.is_dir(): + print(f"cortex {args.command}: --root {args.root} is not a directory", file=sys.stderr) + return 2 + try: + return args.func(args) + except CortexError as e: + print(f"cortex {args.command}: {e}", file=sys.stderr) + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ledger_merge.py b/scripts/ledger_merge.py new file mode 100755 index 0000000..e71a8d1 --- /dev/null +++ b/scripts/ledger_merge.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python3 +"""Classify a PyAutoCortex branch diff as *ledger* or *code*. + +WHY THIS EXISTS. The Cortex's own work strands exactly as the Mind's does. A +branch-scoped session (the phone, claude.ai/code, any `claude/**` flow) pushes +a phase move, a ruling, a batch record to a feature branch and nothing moves +it: no workflow looks at a `claude/**` push. The branch sits there until a +human writes an explicit "merge this" prompt. + +Almost all of what strands is *ledger*: a phase file under `phases/`, a ruling +under `rulings/`, a batch record or review under `batches/`, an epic entry. +It is the organism's own bookkeeping, it is template-shaped, its drift check +(`cortex.py check`) is already automated, and a human reviewing it adds +nothing. The minority that is *code* — `scripts/`, `tests/`, `.github/`, +`policy/`, `docs/`, `projects.yaml`, the prose pages — is exactly what review +is for. + +So this script draws that line, and `ledger_merge.yml` merges only what falls +on the ledger side of it. The gate is a script, not workflow YAML, so it is +testable and so a session can predict the verdict before it pushes. + +DEFAULT DENY. A path is ledger only by matching a rule below; an unrecognised +one — a new root file, a new top-level folder — is code. Getting that backwards +would auto-merge the next thing nobody thought about. + +APPEND-ONLY. `rulings/` is the ledger of record and a ruling, once committed, +is never modified or deleted (rulings/AGENTS.md). When the diff comes from git +(`--base`), the classifier reads `git diff --name-status` and any entry under +`rulings/**` whose status is not `A` (added) — `M`, `D`, `R*`, or anything +else — is code, which leaves the branch for a human. Explicit-path and stdin +inputs carry no status and stay path-only, as in the Mind. + +Usage: + python3 scripts/ledger_merge.py classify --base origin/main # diff HEAD vs base + python3 scripts/ledger_merge.py classify path/one path/two # explicit paths + ... < paths-on-stdin + +Exit codes: 0 = ledger-only (safe to auto-merge) · 1 = holds code (a human's +call) · 2 = the script could not run. The caller must distinguish 1 from 2: +"a human should look" and "the gate is broken" are not the same answer. +""" + +from __future__ import annotations + +import argparse +import fnmatch +import subprocess +import sys +from pathlib import Path + +# Directories holding nothing but the run-and-ruling ledger: phase files, +# rulings of record, batch records and reviews. Their whole contents are +# ledger (subject to the EXCLUDED_NAMES guard below, and to the append-only +# leg for rulings/ in classify_entries). +LEDGER_DIRS = ("phases/", "rulings/", "batches/") + +# Root files that are ledger state. Deliberately NOT here: README.md, +# AGENTS.md, CLAUDE.md, REFERENCE.md — prose a human reads, changed rarely +# and on purpose — and projects.yaml, which is CODE: `sync_cli` and +# `local_path` are paths the batch conductor executes under, worse than the +# Mind's repos.yaml, which is already on the human side of its line. +LEDGER_FILES = ("epics.md",) + +# Names that are ledger by location but executable by collection. A file +# pytest would *collect* runs in CI from anywhere in the tree, so it is code +# wherever it sits — a `test_*.py` dropped beside a batch packet included. +EXCLUDED_NAMES = ("conftest.py", "test_*.py", "*_test.py") + + +def is_ledger_path(path: str) -> bool: + """True if `path` (repo-relative, POSIX separators) is ledger material.""" + # Normalise away "./" and any "..", so a traversal cannot smuggle a code + # path in behind a ledger prefix. + parts = [p for p in path.replace("\\", "/").split("/") if p not in ("", ".")] + if not parts or ".." in parts: + return False + # A dot-directory or dot-file anywhere is never ledger: `.github/`, + # `.claude/`, `.codex/`, `.gitignore` all carry behaviour. + if any(p.startswith(".") for p in parts): + return False + name = parts[-1] + if any(fnmatch.fnmatch(name, pattern) for pattern in EXCLUDED_NAMES): + return False + normalised = "/".join(parts) + if len(parts) == 1: + return normalised in LEDGER_FILES + return any(normalised.startswith(d) for d in LEDGER_DIRS) + + +def classify(paths): + """Split `paths` into (ledger, blocked), preserving order and dropping dupes.""" + ledger, blocked, seen = [], [], set() + for path in paths: + path = path.strip() + if not path or path in seen: + continue + seen.add(path) + (ledger if is_ledger_path(path) else blocked).append(path) + return ledger, blocked + + +# The one dir where a change's KIND matters, not only its path: the ledger of +# record is append-only, so only an added file under it is ledger. +APPEND_ONLY_DIRS = ("rulings/",) + + +def is_append_only_violation(status: str, path: str) -> bool: + """True if a `git diff --name-status` entry edits, deletes, renames (or + does anything but add) a file under an append-only dir.""" + parts = [p for p in path.replace("\\", "/").split("/") if p not in ("", ".")] + normalised = "/".join(parts) + return status[:1] != "A" and any(normalised.startswith(d) for d in APPEND_ONLY_DIRS) + + +def classify_entries(entries): + """Split `(status, path)` entries into (ledger, blocked) paths. + + A path is blocked if it is not ledger material, or if it is a non-`A` + entry under `rulings/` (an edited, deleted or renamed ruling). A rename + entry carries both paths; the old one counts as a deletion.""" + ledger, blocked, seen = [], [], set() + for status, path in entries: + path = path.strip() + if not path or path in seen: + continue + seen.add(path) + if is_ledger_path(path) and not is_append_only_violation(status, path): + ledger.append(path) + else: + blocked.append(path) + return ledger, blocked + + +def changed_entries(base: str, head: str = "HEAD", cwd=None): + """`(status, path)` entries changed by `head` relative to its merge base + with `base` — `git diff --name-status`. A rename yields two entries, the + old path as `D` and the new as `R`, so both sides are judged. + + Three-dot: a branch that merged main into itself must be judged on what it + *adds*, not on everything main moved underneath it, or every long-running + branch reads as code the moment someone else touches `scripts/`. + """ + proc = subprocess.run( + ["git", "diff", "--name-status", f"{base}...{head}"], + capture_output=True, + text=True, + cwd=cwd or Path(__file__).resolve().parents[1], + ) + if proc.returncode != 0: + sys.stderr.write(proc.stderr) + raise SystemExit(2) + entries = [] + for line in proc.stdout.splitlines(): + if not line.strip(): + continue + fields = line.split("\t") + status = fields[0].strip() + if status[:1] in ("R", "C") and len(fields) >= 3: + if status[:1] == "R": + entries.append(("D", fields[1])) + entries.append((status, fields[2])) + elif len(fields) >= 2: + entries.append((status, fields[1])) + return entries + + +def changed_paths(base: str, head: str = "HEAD", cwd=None): + """Paths changed by `head` relative to its merge base with `base`.""" + return [path for _, path in changed_entries(base, head, cwd)] + + +def main(argv=None) -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + sub = parser.add_subparsers(dest="command", required=True) + cls = sub.add_parser("classify", help="ledger-only, or does it hold code?") + cls.add_argument("paths", nargs="*", help="repo-relative paths (else stdin, else --base)") + cls.add_argument("--base", help="diff HEAD against the merge base with this ref") + cls.add_argument("--head", default="HEAD", help="the branch tip to judge (default HEAD)") + args = parser.parse_args(argv) + + if args.paths: + paths = args.paths + elif not sys.stdin.isatty(): + paths = sys.stdin.read().splitlines() + elif args.base: + paths = [] + else: + parser.error("give paths, pipe them in, or pass --base") + return 2 + # Strip blanks BEFORE the emptiness check, not inside classify(): a stdin + # of "\n" is one empty string, which is a truthy list, and an unfiltered + # check would call that "0 ledger paths" and exit 0 — fail-open, on the one + # question this gate exists to answer. + paths = [p.strip() for p in paths if p.strip()] + entries = None + if args.base and not paths: + entries = changed_entries(args.base, args.head) + paths = [path for _, path in entries] + + if not paths: + # Nothing changed is not "safe to merge" — there is nothing to merge, + # and the caller must not read exit 0 as "go". Say so and block. + print("no changed paths — nothing to merge") + return 1 + + # Explicit paths and stdin carry no status, so they are judged on the path + # alone; a git diff also carries the append-only leg for rulings/. + ledger, blocked = classify_entries(entries) if entries is not None else classify(paths) + if blocked: + print(f"code: {len(blocked)} of {len(ledger) + len(blocked)} path(s) need a human") + for path in blocked: + print(f" {path}") + return 1 + print(f"ledger-only: {len(ledger)} path(s)") + for path in ledger: + print(f" {path}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_cortex.py b/tests/test_cortex.py new file mode 100644 index 0000000..e565107 --- /dev/null +++ b/tests/test_cortex.py @@ -0,0 +1,1032 @@ +"""Contract tests for scripts/cortex.py — the Cortex's check, gates, rule, move, new. + +Two things these tests deliberately do, matching the Mind's `test_lifecycle_check.py`: + +1. **The fixture is the witness.** `tests/fixtures/skeleton/` holds one project, + one phase per state, five rulings (one chain of two), one batch record and + one review; `tests/fixtures/empty/` is an empty map. Both must pass `check` + unchanged, and every writing verb runs on a `copytree` of the skeleton. +2. **Prove each leg FAILS.** Every `check` rule in REFERENCE.md is driven with + a `tmp_path` mutation that must trip it, asserting the ` - …` line; every + transition the table allows is taken and every one it refuses is refused. +""" + +import shutil +import subprocess +import sys +from datetime import date +from pathlib import Path + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import cortex # noqa: E402 + +REPO = Path(__file__).resolve().parents[1] +SCRIPT = REPO / "scripts" / "cortex.py" +SKELETON = REPO / "tests" / "fixtures" / "skeleton" +EMPTY = REPO / "tests" / "fixtures" / "empty" +TODAY = "2026-09-02" + +P = {n: f"phases/example/{n}.md" for n in ( + "01_scope", "02_gated_on_dev", "03_ready_cleared", "04_submitted_override", + "05_running_array", "06_pulled", "07_awaiting_ruling", "08_accepted", + "09_rerun", "10_dropped")} + + +# --------------------------------------------------------------------------- # +# helpers +# --------------------------------------------------------------------------- # +def _copy(tmp_path: Path, src: Path = SKELETON) -> Path: + root = tmp_path / "tree" + shutil.copytree(src, root) + return root + + +def _run(*args, root=None, stdin=""): + cmd = [sys.executable, str(SCRIPT), *args] + if root is not None: + cmd += ["--root", str(root)] + return subprocess.run(cmd, input=stdin, capture_output=True, text=True) + + +def _problems(root: Path): + return cortex.check_problems(root) + + +def _edit(root: Path, rel: str, old: str, new: str, count: int = 1): + p = root / rel + text = p.read_text() + assert old in text, f"{old!r} not in {rel}" + p.write_text(text.replace(old, new, count)) + + +def _fields(root: Path, rel: str): + return cortex.parse_header((root / rel).read_text())[1] + + +def _assert_drift(root: Path, *needles: str): + problems = _problems(root) + joined = "\n".join(problems) + for needle in needles: + assert needle in joined, f"expected {needle!r} in:\n{joined}" + return problems + + +def _move(root, rel, state, *extra, today=TODAY): + return _run("move", rel, state, "--today", today, *extra, root=root) + + +def _body(tmp_path: Path, text="Accept. The human's words, verbatim.\n") -> Path: + p = tmp_path / "body.txt" + p.write_text(text) + return p + + +# --------------------------------------------------------------------------- # +# the witness — both fixtures pass check unchanged +# --------------------------------------------------------------------------- # +def test_skeleton_fixture_passes_check(): + result = _run("check", root=SKELETON) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "cortex check: OK" + + +def test_empty_fixture_passes_check(): + result = _run("check", root=EMPTY) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout.strip() == "cortex check: OK" + + +def test_check_problems_is_quiet_on_the_witness(): + assert _problems(SKELETON) == [] + assert _problems(EMPTY) == [] + + +def test_check_reports_drift_in_lifecycle_shape(tmp_path): + root = _copy(tmp_path) + _edit(root, P["03_ready_cleared"], "Gates-cleared: 2026-08-30\n", "") + result = _run("check", root=root) + assert result.returncode == 1 + lines = result.stdout.splitlines() + assert lines[0] == "cortex check: DRIFT" + assert lines[1].startswith(" - phases/example/03_ready_cleared.md: ") + + +def test_no_import_time_side_effects_and_root_on_every_verb(): + help_text = _run("--help").stdout + for verb in ("check", "gates", "rule", "move", "new"): + assert verb in help_text + assert "--root" in _run(verb, "--help").stdout + + +# --------------------------------------------------------------------------- # +# the light header and the grammars +# --------------------------------------------------------------------------- # +def test_header_first_occurrence_wins_block_ends_at_blank_line_list_keys(): + text = ("# Title\n\nProject: one\nProject: two\nGates:\n- A#1\n- B#2\nState: ready\n\n" + "Phase: 99\n") + title, fields = cortex.parse_header(text) + assert title == "Title" + assert fields["Project"] == "one" + assert fields["Gates"] == "A#1, B#2" + assert "Phase" not in fields # after the blank line — body, not header + + +def test_header_beyond_30_lines_is_body(): + text = "# T\n\n" + "\n".join(f"K{i}: v" for i in range(40)) + "\n" + _, fields = cortex.parse_header(text) + assert "K27" in fields and "K28" not in fields + + +def test_edit_header_preserves_every_other_byte(tmp_path): + root = _copy(tmp_path) + before = (root / P["03_ready_cleared"]).read_text() + after = cortex.edit_header(before, {"State": "submitted"}) + assert after.replace("State: submitted", "State: ready") == before + # a removed key drops exactly one line; an inserted key lands at its slot + removed = cortex.edit_header(before, {"Gates-cleared": None}) + assert removed == before.replace("Gates-cleared: 2026-08-30\n", "") + inserted = cortex.edit_header(removed, {"Gate-override": "why"}) + lines = inserted.splitlines() + assert lines[lines.index("Gates: PyAutoGalaxy#486") + 1] == "Gate-override: why" + + +def test_run_line_regex_matches_reference_examples(): + for line in ( + "- 342091_[0-8,10]: done — gpu — submitted 2026-08-30 — wall 6:12", + "- 342091_9: failed — gpu — submitted 2026-08-30 — wall 0:00 — OOM before the first step", + "- 342102: running — gpu — submitted 2026-09-01 — wall 0:00 — task 9 resubmitted alone", + ): + assert cortex.RUN_LINE_RE.match(line), line + assert cortex.RUN_CONT_RE.match(" after: 342091_9") + assert not cortex.RUN_CONT_RE.match(" after: 342091_9") + assert not cortex.RUN_LINE_RE.match("- 342102: running — GPU — submitted 2026-09-01 — wall 0:00") + + +def test_double_dash_is_read_as_the_em_dash(tmp_path): + root = _copy(tmp_path) + _edit(root, P["06_pulled"], " — gpu — submitted 2026-08-31 — wall 3:40 — ", + " -- gpu -- submitted 2026-08-31 -- wall 3:40 -- ") + assert _problems(root) == [] + + +def test_gate_ref_regex_rejects_owner_repo_form(): + assert cortex.GATE_REF_RE.fullmatch("PyAutoArray#431") + assert cortex.GATE_REF_RE.fullmatch("https://github.com/PyAutoLabs/PyAutoFit/pull/1436") + assert not cortex.GATE_REF_RE.fullmatch("PyAutoLabs/PyAutoArray#431") + assert cortex.gate_url("PyAutoArray#431") == "https://github.com/PyAutoLabs/PyAutoArray/issues/431" + assert cortex.gate_url("https://github.com/o/r/pull/7") == "https://github.com/o/r/issues/7" + + +# --------------------------------------------------------------------------- # +# projects.yaml — the restricted subset +# --------------------------------------------------------------------------- # +def test_comment_only_projects_yaml_parses_to_an_empty_map(): + rows, problems = cortex.parse_projects((REPO / "projects.yaml").read_text()) + assert rows == {} and problems == [] + rows, problems = cortex.parse_projects((EMPTY / "projects.yaml").read_text()) + assert rows == {} and problems == [] + + +def test_projects_parse_matches_pyyaml_on_the_fixture(): + yaml = pytest.importorskip("yaml") + text = (SKELETON / "projects.yaml").read_text() + rows, problems = cortex.parse_projects(text) + assert problems == [] + assert yaml.safe_load(text) == rows + + +def test_projects_quoted_scalars_and_comments(): + text = ('p:\n remote: none\n local_path: "/a: b" # trailing\n ral_root: /r\n' + ' mirror: none\n sync_cli: "x # y"\n sync_verbs: [pull]\n ledger: l\n' + ' witness_file: "**/*.json"\n partition: gpu\n status: active\n') + rows, problems = cortex.parse_projects(text) + assert problems == [] + assert rows["p"]["local_path"] == "/a: b" + assert rows["p"]["sync_cli"] == "x # y" + assert rows["p"]["sync_verbs"] == ["pull"] + yaml = pytest.importorskip("yaml") + assert yaml.safe_load(text) == rows + + +def test_unknown_projects_field_is_an_error(tmp_path): + root = _copy(tmp_path) + _edit(root, "projects.yaml", " status: active\n", " status: active\n colour: blue\n") + _assert_drift(root, "projects.yaml:15: unknown field `colour` on example") + + +def test_missing_projects_field_and_bad_enums(tmp_path): + root = _copy(tmp_path) + _edit(root, "projects.yaml", " partition: gpu\n", " partition: cpu\n") + _edit(root, "projects.yaml", " ledger: wiki/project/state.md\n", "") + _assert_drift(root, "example is missing ledger", "partition must be gpu | ral | both") + + +def test_projects_outside_the_subset(tmp_path): + root = _copy(tmp_path) + _edit(root, "projects.yaml", " sync_verbs: [pull, submit, jobs, tail]\n", + " sync_verbs:\n - pull\n") + _assert_drift(root, "sync_verbs must be a flow list", "outside the subset") + + +# --------------------------------------------------------------------------- # +# check — phases +# --------------------------------------------------------------------------- # +def test_duplicate_phase_number(tmp_path): + root = _copy(tmp_path) + _edit(root, P["02_gated_on_dev"], "Phase: 2\n", "Phase: 1\n") + _assert_drift(root, "02_gated_on_dev.md: duplicate Phase: 1 (also phases/example/01_scope.md)") + + +def test_unknown_project(tmp_path): + root = _copy(tmp_path) + shutil.move(root / "phases" / "example", root / "phases" / "other") + for p in (root / "phases" / "other").glob("*.md"): + p.write_text(p.read_text().replace("Project: example", "Project: other")) + _assert_drift(root, "phases/other/01_scope.md: Project: other is not a projects.yaml key") + + +def test_directory_must_equal_project(tmp_path): + root = _copy(tmp_path) + _edit(root, P["01_scope"], "Project: example\n", "Project: sample\n") + _assert_drift(root, "01_scope.md: Project: sample is not the directory name example", + "01_scope.md: Project: sample is not a projects.yaml key") + + +def test_illegal_state_unknown_key_and_missing_section(tmp_path): + root = _copy(tmp_path) + _edit(root, P["01_scope"], "State: planned\n", "State: queued\nGate-cleared: x\n") + _edit(root, P["01_scope"], "## Where to look", "## Where") + _assert_drift(root, "State: 'queued' is not a phase state", "unknown header key Gate-cleared:", + "missing section ## Where to look") + + +def test_owner_repo_gate_form_is_refused(tmp_path): + root = _copy(tmp_path) + _edit(root, P["02_gated_on_dev"], "Gates: PyAutoArray#431,", "Gates: PyAutoLabs/PyAutoArray#431,") + _assert_drift(root, "Gates: unrecognised ref 'PyAutoLabs/PyAutoArray#431'") + + +def test_gated_with_empty_gates(tmp_path): + root = _copy(tmp_path) + _edit(root, P["02_gated_on_dev"], "Gates: PyAutoArray#431, https://github.com/PyAutoLabs/PyAutoFit/pull/1436\n", + "Gates:\n") + _assert_drift(root, "02_gated_on_dev.md: State: gated with an empty Gates:") + + +@pytest.mark.parametrize("rel,line", [ + (P["03_ready_cleared"], "Gates-cleared: 2026-08-30\n"), + (P["04_submitted_override"], "Gate-override: the fix is verified locally on the branch; not waiting for the merge\n"), + (P["08_accepted"], "Gates-cleared: 2026-08-25\n"), +]) +def test_gates_invariant_needs_cleared_or_override(tmp_path, rel, line): + root = _copy(tmp_path) + _edit(root, rel, line, "") + _assert_drift(root, f"{rel}: State: {_fields(root, rel)['State']} with Gates: needs " + "Gates-cleared: or Gate-override:") + + +def test_rerun_is_inside_the_gates_invariant(tmp_path): + root = _copy(tmp_path) + _edit(root, P["09_rerun"], "Budget: 8:00\n", "Budget: 8:00\nGates: PyAutoLens#1\n") + _assert_drift(root, "09_rerun.md: State: rerun with Gates: needs Gates-cleared: or Gate-override:") + + +@pytest.mark.parametrize("rel", [P["04_submitted_override"], P["05_running_array"], P["06_pulled"], + P["07_awaiting_ruling"], P["08_accepted"], P["09_rerun"]]) +def test_witness_invariant(tmp_path, rel): + root = _copy(tmp_path) + witness = _fields(root, rel)["Witness"] + _edit(root, rel, f"Witness: {witness}\n", "Witness:\n") + _assert_drift(root, f"{rel}: State: {_fields(root, rel)['State']} needs a Witness:") + + +def test_planned_phase_may_leave_the_witness_empty(): + assert _fields(SKELETON, P["01_scope"])["Witness"] == "" + assert _problems(SKELETON) == [] + + +@pytest.mark.parametrize("rel,rid", [(P["08_accepted"], "R-20260901-02"), + (P["09_rerun"], "R-20260901-04"), + (P["10_dropped"], "R-20260901-05")]) +def test_ruled_states_need_a_ruling(tmp_path, rel, rid): + root = _copy(tmp_path) + _edit(root, rel, f"Ruling: {rid}\n", "Ruling:\n") + _assert_drift(root, f"{rel}: State: {_fields(root, rel)['State']} needs a Ruling:") + + +def test_header_body_run_mismatch(tmp_path): + root = _copy(tmp_path) + _edit(root, P["04_submitted_override"], "Runs: 342001, 342010\n", "Runs: 342001\n") + _assert_drift(root, "04_submitted_override.md: Runs: header {342001} != body stems {342001, 342010}") + + +def test_overlapping_task_sets_on_one_stem(tmp_path): + root = _copy(tmp_path) + _edit(root, P["05_running_array"], "- 342091_9: failed", "- 342091_[7-9]: failed") + _assert_drift(root, "05_running_array.md: run 342091_[0-8,10] overlaps 342091_[7-9] on stem 342091") + + +def test_bare_stem_overlaps_every_task_of_that_stem(tmp_path): + root = _copy(tmp_path) + _edit(root, P["05_running_array"], "- 342091_9: failed", "- 342091: failed") + _assert_drift(root, "run 342091_[0-8,10] overlaps 342091 on stem 342091") + + +def test_one_array_may_feed_two_phases(): + """Job ids are unique per phase, not globally: 342120 sits in phases 6 and 7.""" + assert _fields(SKELETON, P["06_pulled"])["Runs"] == "342120" + assert "342120" in _fields(SKELETON, P["07_awaiting_ruling"])["Runs"] + assert _problems(SKELETON) == [] + + +def test_run_line_that_does_not_parse_and_stray_continuation(tmp_path): + root = _copy(tmp_path) + _edit(root, P["06_pulled"], "- 342120_[0-4]: done — gpu — submitted 2026-08-31 — wall 3:40", + "- 342120_[0-4]: done — gpu — 2026-08-31 — wall 3:40") + _edit(root, P["04_submitted_override"], "## Runs\n\n", "## Runs\n\n after: 1\n") + _assert_drift(root, "06_pulled.md: line 28: run line does not parse", + "04_submitted_override.md: line 30: continuation line without a run line") + + +def test_pulled_needs_a_pulled_to(tmp_path): + root = _copy(tmp_path) + _edit(root, P["06_pulled"], " pulled_to: /mnt/c/Users/Jammy/Science/example/output/phase_06\n", "") + _assert_drift(root, "06_pulled.md: State: pulled needs at least one done | legacy run with pulled_to:") + + +def test_legacy_needs_where(tmp_path): + root = _copy(tmp_path) + _edit(root, P["10_dropped"], " where: /mnt/c/Users/Jammy/Science/example/output/legacy_wrong/phase_10\n", "") + _assert_drift(root, "10_dropped.md: run 341950_[0-3] is legacy_wrong without where:") + + +def test_after_and_resumes_name_a_run_of_the_same_phase(tmp_path): + root = _copy(tmp_path) + _edit(root, P["05_running_array"], " after: 342091_9", " after: 342091") + _edit(root, P["07_awaiting_ruling"], " resumes: 342110", " resumes: 342120_[5-9]") + _assert_drift(root, "05_running_array.md: run 342102 after: 342091 names no other run of this phase", + "07_awaiting_ruling.md: run 342120_[5-9] resumes: 342120_[5-9] names no other run") + + +def test_ruled_continuation_must_resolve(tmp_path): + root = _copy(tmp_path) + _edit(root, P["08_accepted"], " ruled: R-20260901-02", " ruled: R-20260901-09") + _assert_drift(root, "08_accepted.md: run 342050 ruled: R-20260901-09 does not resolve to a ruling file") + + +def test_pulled_with_a_live_run_needs_leave_to_finish(tmp_path): + root = _copy(tmp_path) + _edit(root, P["05_running_array"], "State: running\n", "State: pulled\n") + assert _problems(root) == [] # R-20260901-03 is a leave-to-finish head + _edit(root, P["05_running_array"], "Ruling: R-20260901-03\n", "Ruling:\n") + _assert_drift(root, "05_running_array.md: State: pulled with a live run needs a leave-to-finish ruling head") + + +def test_stray_file_under_phases(tmp_path): + root = _copy(tmp_path) + (root / "phases" / "notes.md").write_text("# stray\n") + _assert_drift(root, "phases/notes.md: not a phase path") + + +# --------------------------------------------------------------------------- # +# check — rulings and chains +# --------------------------------------------------------------------------- # +R = "rulings/2026/09/" + + +def test_dangling_supersedes(tmp_path): + root = _copy(tmp_path) + _edit(root, R + "R-20260901-02.md", "Supersedes: R-20260901-01\n", "Supersedes: R-20260831-01\n") + _assert_drift(root, "R-20260901-02.md: Supersedes: R-20260831-01 does not resolve to a ruling file") + + +def test_self_supersedes(tmp_path): + root = _copy(tmp_path) + _edit(root, R + "R-20260901-02.md", "Supersedes: R-20260901-01\n", "Supersedes: R-20260901-02\n") + _assert_drift(root, "R-20260901-02.md: Supersedes: itself") + + +def test_supersedes_must_be_earlier_and_same_phase(tmp_path): + root = _copy(tmp_path) + _edit(root, R + "R-20260901-01.md", "Ruling: accept\n", "Ruling: accept\nSupersedes: R-20260901-03\n") + _assert_drift(root, "R-20260901-01.md: Supersedes: R-20260901-03 is not earlier than R-20260901-01", + "R-20260901-01.md: Supersedes: R-20260901-03 names a different project/phase") + + +def test_duplicate_successor_is_a_tree_not_a_chain(tmp_path): + root = _copy(tmp_path) + _edit(root, R + "R-20260901-03.md", "Ruling: leave-to-finish\n", + "Ruling: leave-to-finish\nSupersedes: R-20260901-01\n") + _assert_drift(root, "R-20260901-01.md: has 2 successors (R-20260901-02, R-20260901-03) — a chain, not a tree") + + +def test_superseded_ruling_as_phase_head(tmp_path): + root = _copy(tmp_path) + _edit(root, P["08_accepted"], "Ruling: R-20260901-02\n", "Ruling: R-20260901-01\n") + _assert_drift(root, "08_accepted.md: Ruling: R-20260901-01 is not a chain head (superseded by R-20260901-02)") + + +def test_phase_ruling_must_name_that_phase(tmp_path): + root = _copy(tmp_path) + _edit(root, P["09_rerun"], "Ruling: R-20260901-04\n", "Ruling: R-20260901-05\n") + _assert_drift(root, "09_rerun.md: Ruling: R-20260901-05 names phases/example/10_dropped.md, not this phase", + "09_rerun.md: Ruling: R-20260901-05 verb 'drop' does not fit State: rerun") + + +def test_verb_state_mismatch(tmp_path): + root = _copy(tmp_path) + _edit(root, P["08_accepted"], "State: accepted\n", "State: awaiting-ruling\n") + _assert_drift(root, "08_accepted.md: Ruling: R-20260901-02 verb 'accept' does not fit State: awaiting-ruling") + + +def test_ruling_id_filename_title_and_directory(tmp_path): + root = _copy(tmp_path) + _edit(root, R + "R-20260901-05.md", "# R-20260901-05 — drop", "# R-20260901-06 — drop") + (root / "rulings" / "2026" / "08").mkdir() + shutil.copy(root / R / "R-20260901-04.md", root / "rulings" / "2026" / "08" / "R-20260901-04.md") + (root / "rulings" / "2026" / "stray.md").write_text("# stray\n") + _assert_drift(root, "R-20260901-05.md: title must be `# R-20260901-05`", + "rulings/2026/08/R-20260901-04.md: filed under 2026/08 but the id is dated 2026-09", + "duplicate ruling id R-20260901-04", + "rulings/2026/stray.md: not a ruling path") + + +def test_ruling_runs_subset_phase_path_verb_and_batch(tmp_path): + root = _copy(tmp_path) + _edit(root, R + "R-20260901-03.md", "Runs: 342091, 342102\n", "Runs: 342091, 342999\n") + _edit(root, R + "R-20260901-03.md", "Batch: 2026-09-01-pm\n", "Batch: 2026-09-01-am\n") + _edit(root, R + "R-20260901-04.md", "Ruling: rerun\n", "Ruling: redo\n") + _edit(root, R + "R-20260901-05.md", "Phase: phases/example/10_dropped.md\n", "Phase: phases/example/11.md\n") + _assert_drift(root, "R-20260901-03.md: Runs: {342999} not in the phase's runs", + "R-20260901-03.md: Batch: 2026-09-01-am names no batches/2026-09-01-am.md", + "R-20260901-04.md: Ruling: 'redo' is not a ruling verb", + "R-20260901-05.md: Phase: phases/example/11.md does not resolve to a phase file") + + +def test_ruling_follow_up_uses_the_gate_grammar(tmp_path): + root = _copy(tmp_path) + _edit(root, R + "R-20260901-02.md", "Follow-ups: PyAutoLens#901\n", "Follow-ups: PyAutoLabs/PyAutoLens#901\n") + _assert_drift(root, "R-20260901-02.md: Follow-ups: unrecognised ref 'PyAutoLabs/PyAutoLens#901'") + + +def test_two_chains_of_one_on_a_phase_are_fine(tmp_path): + """leave-to-finish then accept need not chain; Supersedes: replaces, never sequences.""" + root = _copy(tmp_path) + _move(root, P["05_running_array"], "pulled", "--partial") + _move(root, P["05_running_array"], "awaiting-ruling") + r = _run("rule", P["05_running_array"], "accept", "--body", str(_body(tmp_path)), + "--today", TODAY, root=root) + assert r.returncode == 0, r.stderr + assert _problems(root) == [] + assert _fields(root, P["05_running_array"])["Ruling"] == "R-20260902-01" + assert "Supersedes" not in _fields(root, "rulings/2026/09/R-20260902-01.md") + + +# --------------------------------------------------------------------------- # +# check — batches and reviews +# --------------------------------------------------------------------------- # +B = "batches/2026-09-01-pm.md" +V = "batches/reviews/2026-09-01-pm.md" + + +def test_batch_member_lines(tmp_path): + root = _copy(tmp_path) + _edit(root, B, " - 06_pulled: phases/example/06_pulled.md — 342120 — 5 — pulled", + " - 06_pull: phases/example/06_pulled.md — 342120, 1 — five — parked") + _edit(root, B, " - 09_rerun: phases/example/09_rerun.md — 341900 — 5 — awaiting-ruling", + " - 09_rerun: phases/example/09_rerun.md — 341900 — 5") + _edit(root, B, " - 10_dropped: phases/example/10_dropped.md", " - 10_dropped: phases/example/10_drop.md") + _assert_drift(root, "2026-09-01-pm.md: member slug '06_pull' != phase stem '06_pulled'", + "member 06_pull: runs {1} not in the phase's runs", + "member 06_pull: review-minutes 'five' is not an integer", + "member 06_pull: state 'parked' is not a phase state", + "line 12: member line does not parse", + "member 10_dropped: phases/example/10_drop.md does not exist") + + +def test_review_grammar(tmp_path): + root = _copy(tmp_path) + _edit(root, V, "## 10_dropped — FAILED", "## 10_dropped — BROKEN") + _edit(root, V, "## 09_rerun — SUSPECT\n- decision: rerun", "## 09_rerun — SUSPECT\n- decision: tweak") + _edit(root, V, "## 08_accepted — HEALTHY\n- decision: accept\n- ruled: yes", + "## 08_accepted — HEALTHY\n- decision: accept\n- ruled: maybe") + _edit(root, V, "## 06_pulled — HEALTHY\n- decision: (none)\n- ruled: no", + "## 06_pulled — HEALTHY\n- decision: (none)\n- ruled: yes") + _edit(root, V, "## 05_running_array — RUNNING", "## 05_running — RUNNING") + _assert_drift(root, "10_dropped: health 'BROKEN' not in HEALTHY | SUSPECT | FAILED | RUNNING", + "09_rerun: decision 'tweak' is not a ruling verb or (none)", + "08_accepted: ruled 'maybe' is not yes | no", + "06_pulled: ruled: yes needs a verb, not (none)", + "section '05_running' names no member of batches/2026-09-01-pm.md") + + +def test_review_title_and_record_must_exist(tmp_path): + root = _copy(tmp_path) + shutil.move(root / V, root / "batches" / "reviews" / "2026-09-01-am.md") + _assert_drift(root, "2026-09-01-am.md: title must be `# Batch review 2026-09-01-am`", + "2026-09-01-am.md: no batch record batches/2026-09-01-am.md", + "2026-09-01-pm.md: review: batches/reviews/2026-09-01-pm.md does not exist") + + +# --------------------------------------------------------------------------- # +# move — every legal transition, every refused one +# --------------------------------------------------------------------------- # +def test_planned_to_gated_or_ready_by_gates(tmp_path): + root = _copy(tmp_path) + r = _move(root, P["01_scope"], "gated") + assert r.returncode == 1 and "Gates: is empty — planned → ready" in r.stderr + r = _move(root, P["01_scope"], "ready") + assert r.returncode == 0 and r.stdout.strip() == "phases/example/01_scope.md: planned → ready" + assert _fields(root, P["01_scope"])["State"] == "ready" + _edit(root, P["01_scope"], "State: ready\nGates:\n", "State: planned\nGates: PyAutoLens#1\n") + r = _move(root, P["01_scope"], "ready") + assert r.returncode == 1 and "Gates: is non-empty — planned → gated" in r.stderr + assert _move(root, P["01_scope"], "gated").returncode == 0 + assert _problems(root) == [] + + +def test_gated_to_ready_needs_override_or_gates_write(tmp_path): + root = _copy(tmp_path) + r = _move(root, P["02_gated_on_dev"], "ready") + assert r.returncode == 1 and "gates --grade --write" in r.stderr and "--override" in r.stderr + r = _move(root, P["02_gated_on_dev"], "ready", "--override", "verified on the branch") + assert r.returncode == 0 + f = _fields(root, P["02_gated_on_dev"]) + assert f["State"] == "ready" and f["Gate-override"] == "verified on the branch" + assert _problems(root) == [] + + +def test_ready_to_gated_is_gates_write_only(tmp_path): + root = _copy(tmp_path) + r = _move(root, P["03_ready_cleared"], "gated") + assert r.returncode == 1 and "gates --grade --write" in r.stderr + + +def test_ready_to_submitted_needs_witness_and_run(tmp_path): + root = _copy(tmp_path) + r = _move(root, P["03_ready_cleared"], "submitted") + assert r.returncode == 1 and "--run" in r.stderr + _edit(root, P["03_ready_cleared"], "Witness: anchor lens theta_E within 0.01 arcsec of the published value\n", + "Witness:\n") + r = _move(root, P["03_ready_cleared"], "submitted", "--run", "350000") + assert r.returncode == 1 and "Witness: is empty" in r.stderr + _edit(root, P["03_ready_cleared"], "Witness:\n", "Witness: theta_E within 0.01\n") + before = (root / P["03_ready_cleared"]).read_text() + r = _move(root, P["03_ready_cleared"], "submitted", "--run", "350000", "--note", "first wave") + assert r.returncode == 0, r.stderr + after = (root / P["03_ready_cleared"]).read_text() + assert "- 350000: submitted — gpu — submitted 2026-09-02 — wall 0:00 — first wave\n" in after + f = _fields(root, P["03_ready_cleared"]) + assert f["State"] == "submitted" and f["Runs"] == "350000" + # the body outside `## Runs` and the rest of the header are untouched + assert after.split("## Runs")[0].replace("State: submitted", "State: ready") \ + .replace("Runs: 350000\n", "") == before.split("## Runs")[0] + assert after.split("## Ruling")[1] == before.split("## Ruling")[1] + assert _problems(root) == [] + + +def test_run_self_loop_appends_a_wave_and_keeps_state(tmp_path): + root = _copy(tmp_path) + r = _move(root, P["04_submitted_override"], "submitted", "--run", "342020", "--after", "342010") + assert r.returncode == 0 and "appended run 342020" in r.stdout + text = (root / P["04_submitted_override"]).read_text() + assert "- 342020: submitted — gpu — submitted 2026-09-02 — wall 0:00\n after: 342010\n" in text + f = _fields(root, P["04_submitted_override"]) + assert f["State"] == "submitted" and f["Runs"] == "342001, 342010, 342020" + r = _move(root, P["05_running_array"], "running", "--run", "342103", "--resumes", "342091_9") + assert r.returncode == 0 + assert _fields(root, P["05_running_array"])["State"] == "running" + assert _problems(root) == [] + + +def test_run_self_loop_refuses_overlap_and_dangling_after(tmp_path): + root = _copy(tmp_path) + r = _move(root, P["05_running_array"], "running", "--run", "342091_[9-10]") + assert r.returncode == 1 and "overlaps" in r.stderr + r = _move(root, P["05_running_array"], "running", "--run", "342200", "--after", "999") + assert r.returncode == 1 and "names no run of this phase" in r.stderr + r = _move(root, P["03_ready_cleared"], "ready", "--run", "1") + assert r.returncode == 1 and "already ready" in r.stderr + + +def test_submitted_to_running_and_back_to_ready(tmp_path): + root = _copy(tmp_path) + assert _move(root, P["04_submitted_override"], "running").returncode == 0 + r = _move(root, P["04_submitted_override"], "ready", "--reason", "node failure") + assert r.returncode == 1 and "still submitted | running" in r.stderr + _edit(root, P["04_submitted_override"], "- 342010: submitted", "- 342010: failed") + r = _move(root, P["04_submitted_override"], "ready") + assert r.returncode == 1 and "--reason" in r.stderr + r = _move(root, P["04_submitted_override"], "ready", "--reason", "both waves died on node failure") + assert r.returncode == 0, r.stderr + f = _fields(root, P["04_submitted_override"]) + assert f["State"] == "ready" and f["Reset"] == "both waves died on node failure" + assert _problems(root) == [] + # a phase whose runs all succeeded has nothing to reset from + _edit(root, P["03_ready_cleared"], "State: ready\n", "State: running\n") + _edit(root, P["03_ready_cleared"], "## Runs\n\n", "## Runs\n\n- 1: done — gpu — submitted 2026-09-01 — wall 1:00\n") + _edit(root, P["03_ready_cleared"], "Runs:\n", "Runs: 1\n") if "Runs:" in (root / P["03_ready_cleared"]).read_text() \ + else _edit(root, P["03_ready_cleared"], "Budget: 5:00\n", "Budget: 5:00\nRuns: 1\n") + r = _move(root, P["03_ready_cleared"], "ready", "--reason", "x") + assert r.returncode == 1 and "no failed | timeout | void run" in r.stderr + + +def test_running_to_pulled_and_partial(tmp_path): + root = _copy(tmp_path) + r = _move(root, P["05_running_array"], "pulled") + assert r.returncode == 1 and "--partial" in r.stderr + r = _move(root, P["05_running_array"], "pulled", "--partial") + assert r.returncode == 0 + assert _problems(root) == [] # the leave-to-finish head closes the partial pull + _edit(root, P["05_running_array"], "- 342102: running", "- 342102: done") + _edit(root, P["05_running_array"], "State: pulled\n", "State: running\n") + assert _move(root, P["05_running_array"], "pulled").returncode == 0 + + +def test_running_to_pulled_writes_pulled_to(tmp_path): + root = _copy(tmp_path) + _edit(root, P["04_submitted_override"], "State: submitted\n", "State: running\n") + _edit(root, P["04_submitted_override"], "- 342010: submitted", "- 342010: done") + r = _move(root, P["04_submitted_override"], "pulled") + assert r.returncode == 1 and "--pulled-to" in r.stderr + r = _move(root, P["04_submitted_override"], "pulled", "--pulled-to", "/mnt/c/out/phase_04") + assert r.returncode == 0, r.stderr + text = (root / P["04_submitted_override"]).read_text() + assert "- 342010: done — gpu — submitted 2026-09-01 — wall 0:00\n pulled_to: /mnt/c/out/phase_04\n" in text + assert "- 342001: failed — gpu — submitted 2026-08-31 — wall 0:00 — node failure before the first step\n- 342010" in text + assert _problems(root) == [] + + +def test_pulled_to_awaiting_ruling_and_rerun_to_ready(tmp_path): + root = _copy(tmp_path) + assert _move(root, P["06_pulled"], "awaiting-ruling").returncode == 0 + assert _move(root, P["09_rerun"], "ready").returncode == 0 + assert _problems(root) == [] + + +def test_legacy_born_ready_to_pulled(tmp_path): + root = _copy(tmp_path) + _edit(root, P["09_rerun"], "State: rerun\nWitness: the legacy run's mask edge matches the released mask within one pixel\n", + "State: ready\nWitness:\n") + r = _move(root, P["09_rerun"], "pulled") + assert r.returncode == 1 and "Witness: is empty" in r.stderr + _edit(root, P["09_rerun"], "Witness:\n", "Witness: mask edge within one pixel\n") + assert _move(root, P["09_rerun"], "pulled").returncode == 0 + _edit(root, P["09_rerun"], "State: pulled\n", "State: ready\n") + _edit(root, P["09_rerun"], "- 341900: legacy", "- 341900: done") + r = _move(root, P["09_rerun"], "pulled") + assert r.returncode == 1 and "legacy-born" in r.stderr + + +@pytest.mark.parametrize("rel,to,needle", [ + (P["07_awaiting_ruling"], "accepted", "rule <phase> accept"), + (P["07_awaiting_ruling"], "rerun", "rule <phase> rerun"), + (P["07_awaiting_ruling"], "dropped", "rule <phase> drop"), + (P["01_scope"], "dropped", "rule <phase> drop"), + (P["08_accepted"], "rerun", "--supersedes"), + (P["10_dropped"], "ready", "terminal"), + (P["04_submitted_override"], "pulled", "no edge submitted → pulled"), + (P["03_ready_cleared"], "running", "no edge ready → running"), + (P["06_pulled"], "ready", "no edge pulled → ready"), + (P["01_scope"], "flying", "not a phase state"), +]) +def test_refused_edges(tmp_path, rel, to, needle): + root = _copy(tmp_path) + before = (root / rel).read_text() + r = _move(root, rel, to) + assert r.returncode == 1, r.stdout + assert needle in r.stderr, r.stderr + assert (root / rel).read_text() == before + + +# --------------------------------------------------------------------------- # +# rule +# --------------------------------------------------------------------------- # +def _rule(root, rel, verb, body, *extra, today=TODAY): + return _run("rule", rel, verb, "--body", str(body), "--today", today, *extra, root=root) + + +def test_rule_assigns_ids_in_sequence_and_moves_the_phase(tmp_path): + root = _copy(tmp_path) + body = _body(tmp_path) + r = _rule(root, P["07_awaiting_ruling"], "accept", body, "--batch", "2026-09-01-pm", + "--minutes", "4", "--follow-up", "PyAutoLens#902") + assert r.returncode == 0, r.stderr + assert r.stdout.strip() == "wrote rulings/2026/09/R-20260902-01.md" + rf = _fields(root, "rulings/2026/09/R-20260902-01.md") + assert rf["Phase"] == P["07_awaiting_ruling"] and rf["Ruling"] == "accept" + assert rf["Runs"] == "342110, 342120" and rf["Batch"] == "2026-09-01-pm" + assert rf["Review-minutes-actual"] == "4" and rf["Follow-ups"] == "PyAutoLens#902" + assert rf["Reviewed-at"].startswith("2026-09-02T") + text = (root / "rulings/2026/09/R-20260902-01.md").read_text() + assert text.startswith("# R-20260902-01 — accept example phase 7\n") + assert "## Ruling\n\nAccept. The human's words, verbatim.\n\n## Evidence\n\n- " in text + pf = _fields(root, P["07_awaiting_ruling"]) + assert pf["State"] == "accepted" and pf["Ruling"] == "R-20260902-01" + assert "R-20260902-01 — accept" in (root / P["07_awaiting_ruling"]).read_text().split("## Ruling")[1] + assert _problems(root) == [] + # the second ruling of the day is -02 + _move(root, P["06_pulled"], "awaiting-ruling") + r = _rule(root, P["06_pulled"], "rerun", body) + assert r.stdout.strip() == "wrote rulings/2026/09/R-20260902-02.md" + assert _fields(root, P["06_pulled"])["State"] == "rerun" + assert _problems(root) == [] + + +def test_rule_supersedes_an_accepted_ruling(tmp_path): + root = _copy(tmp_path) + body = _body(tmp_path, "REWIND: the gates were wrong.\n") + r = _rule(root, P["08_accepted"], "rerun", body) + assert r.returncode == 1 and "--supersedes R-20260901-02" in r.stderr + r = _rule(root, P["08_accepted"], "rerun", body, "--supersedes", "R-20260901-01") + assert r.returncode == 1 and "already has a successor" in r.stderr + r = _rule(root, P["08_accepted"], "accept", body, "--supersedes", "R-20260901-02") + assert r.returncode == 1 and "only rerun | drop" in r.stderr + r = _rule(root, P["08_accepted"], "rerun", body, "--supersedes", "R-20260901-02") + assert r.returncode == 0, r.stderr + rf = _fields(root, "rulings/2026/09/R-20260902-01.md") + assert rf["Supersedes"] == "R-20260901-02" + pf = _fields(root, P["08_accepted"]) + assert pf["State"] == "rerun" and pf["Ruling"] == "R-20260902-01" + assert _problems(root) == [] + + +def test_rule_also_fans_out_with_one_batch(tmp_path): + root = _copy(tmp_path) + body = _body(tmp_path, "Drop the lot.\n") + _move(root, P["06_pulled"], "awaiting-ruling") + r = _rule(root, P["07_awaiting_ruling"], "drop", body, "--batch", "2026-09-01-pm", + "--also", P["06_pulled"], "--also", P["08_accepted"], "--also", P["01_scope"]) + assert r.returncode == 0, r.stderr + assert r.stdout.splitlines() == [f"wrote rulings/2026/09/R-20260902-0{n}.md" for n in (1, 2, 3, 4)] + for n, rel in ((1, P["07_awaiting_ruling"]), (2, P["06_pulled"]), (3, P["08_accepted"]), (4, P["01_scope"])): + rf = _fields(root, f"rulings/2026/09/R-20260902-0{n}.md") + assert rf["Batch"] == "2026-09-01-pm" and rf["Phase"] == rel and rf["Ruling"] == "drop" + assert "Drop the lot." in (root / f"rulings/2026/09/R-20260902-0{n}.md").read_text() + assert _fields(root, rel)["State"] == "dropped" + # the accepted --also phase superseded its own head + assert _fields(root, "rulings/2026/09/R-20260902-03.md")["Supersedes"] == "R-20260901-02" + assert _problems(root) == [] + + +def test_rule_refuses_a_verb_the_table_forbids(tmp_path): + root = _copy(tmp_path) + body = _body(tmp_path) + for rel, verb, needle in ( + (P["05_running_array"], "accept", "not allowed from State: running"), + (P["06_pulled"], "rerun", "not allowed from State: pulled"), + (P["10_dropped"], "drop", "terminal"), + (P["01_scope"], "leave-to-finish", "running | pulled | awaiting-ruling"), + (P["09_rerun"], "accept", "not allowed from State: rerun"), + ): + before = (root / rel).read_text() + r = _rule(root, rel, verb, body) + assert r.returncode == 1, (rel, verb, r.stdout) + assert needle in r.stderr, r.stderr + assert (root / rel).read_text() == before + assert not list((root / "rulings" / "2026" / "09").glob("R-20260902-*")) + + +def test_rule_leave_to_finish_keeps_state(tmp_path): + root = _copy(tmp_path) + r = _rule(root, P["06_pulled"], "leave-to-finish", _body(tmp_path, "Wait.\n")) + assert r.returncode == 0, r.stderr + f = _fields(root, P["06_pulled"]) + assert f["State"] == "pulled" and f["Ruling"] == "R-20260902-01" + assert _problems(root) == [] + + +def test_rule_refuses_to_touch_an_existing_ruling(tmp_path): + root = _copy(tmp_path) + r = _rule(root, P["07_awaiting_ruling"], "accept", _body(tmp_path), today="2026-09-01") + assert r.returncode == 0 # -06 is free on 2026-09-01 + assert r.stdout.strip() == "wrote rulings/2026/09/R-20260901-06.md" + existing = root / "rulings/2026/09/R-20260901-06.md" + before = existing.read_text() + # an id that exists somewhere unexpected is still taken + (root / "rulings" / "2026" / "10").mkdir() + for n in range(7, 100): + (root / "rulings" / "2026" / "09" / f"R-20260901-{n:02d}.md").write_text("# x\n") + _move(root, P["06_pulled"], "awaiting-ruling") + r = _rule(root, P["06_pulled"], "accept", _body(tmp_path), today="2026-09-01") + assert r.returncode == 1 and "99 rulings already filed" in r.stderr + assert existing.read_text() == before + + +def test_rule_validates_batch_follow_up_and_body(tmp_path): + root = _copy(tmp_path) + body = _body(tmp_path) + r = _rule(root, P["07_awaiting_ruling"], "accept", body, "--batch", "2026-09-09-x") + assert r.returncode == 1 and "names no batches/2026-09-09-x.md" in r.stderr + r = _rule(root, P["07_awaiting_ruling"], "accept", body, "--follow-up", "PyAutoLabs/PyAutoLens#1") + assert r.returncode == 1 and "--follow-up" in r.stderr + r = _rule(root, P["07_awaiting_ruling"], "accept", _body(tmp_path, "\n\n")) + assert r.returncode == 1 and "--body is empty" in r.stderr + assert _fields(root, P["07_awaiting_ruling"])["State"] == "awaiting-ruling" + + +# --------------------------------------------------------------------------- # +# gates +# --------------------------------------------------------------------------- # +def test_gates_offline_lists_gated_phases(): + r = _run("gates", root=SKELETON) + assert r.returncode == 0 + assert r.stdout.splitlines() == [ + "gates: 1 phase(s)", + " phases/example/02_gated_on_dev.md: gated — PyAutoArray#431, " + "https://github.com/PyAutoLabs/PyAutoFit/pull/1436", + ] + r = _run("gates", root=EMPTY) + assert r.returncode == 0 and r.stdout.strip() == "gates: no gated phase" + + +def _pr(merged: bool, state: str = "closed"): + return {"state": state, "state_reason": None, "merged_at": "2026-09-01T00:00:00Z" if merged else None, + "is_pr": True} + + +def _issue(state: str, reason=None): + return {"state": state, "state_reason": reason, "merged_at": None, "is_pr": False} + + +ARRAY = "https://github.com/PyAutoLabs/PyAutoArray/issues/431" +FIT_PR = "https://github.com/PyAutoLabs/PyAutoFit/issues/1436" +GALAXY = "https://github.com/PyAutoLabs/PyAutoGalaxy/issues/486" +LENS = "https://github.com/PyAutoLabs/PyAutoLens/issues/900" +FIT_ISSUE = "https://github.com/PyAutoLabs/PyAutoFit/issues/1400" + + +def _grade(root, states, write=False, today=date(2026, 9, 2)): + lines, rc = cortex.gates_report(root, grade=True, write=write, fetch=lambda urls: states, today=today) + return "\n".join(lines), rc + + +def test_grade_gate_verdicts(): + assert cortex.grade_gate(_pr(True))[0] == "cleared" + assert cortex.grade_gate(_pr(False))[0] == "dead" + assert cortex.grade_gate(_pr(False, "open"))[0] == "open" + assert cortex.grade_gate(_issue("closed", "completed"))[0] == "cleared" + assert cortex.grade_gate(_issue("closed"))[0] == "cleared" + assert cortex.grade_gate(_issue("closed", "not_planned"))[0] == "dead" + assert cortex.grade_gate(_issue("closed", "duplicate"))[0] == "dead" + assert cortex.grade_gate(_issue("open"))[0] == "open" + assert cortex.grade_gate("unreadable: HTTP 404")[0] == "unreadable" + + +def test_grade_reports_without_writing(tmp_path): + root = _copy(tmp_path) + before = (root / P["02_gated_on_dev"]).read_text() + out, rc = _grade(root, {ARRAY: _issue("closed", "completed"), FIT_PR: _pr(True), + GALAXY: _issue("closed"), LENS: _issue("closed"), FIT_ISSUE: _issue("closed")}) + assert rc == 0 + assert "02_gated_on_dev.md: gated\n PyAutoArray#431 → cleared (issue closed (completed))\n" in out + assert "→ cleared (PR merged)" in out + assert "verdict: cleared (gates --grade --write flips it to ready)" in out + assert (root / P["02_gated_on_dev"]).read_text() == before + + +def test_grade_write_flips_gated_to_ready(tmp_path): + root = _copy(tmp_path) + out, rc = _grade(root, {ARRAY: _issue("closed", "completed"), FIT_PR: _pr(True), + GALAXY: _issue("closed"), LENS: _issue("closed"), FIT_ISSUE: _issue("closed")}, + write=True) + assert rc == 0 and "gated → ready, Gates-cleared: 2026-09-02" in out + f = _fields(root, P["02_gated_on_dev"]) + assert f["State"] == "ready" and f["Gates-cleared"] == "2026-09-02" + assert _problems(root) == [] + + +@pytest.mark.parametrize("info,needle", [ + (_pr(False), "dead gate"), + (_issue("closed", "not_planned"), "dead gate"), + (_issue("open"), "verdict: waiting"), + ("unreadable: HTTP 404", "unreadable ref — fails closed, nothing flipped"), +]) +def test_grade_write_does_not_flip_dead_open_or_unreadable(tmp_path, info, needle): + root = _copy(tmp_path) + out, rc = _grade(root, {ARRAY: info, FIT_PR: _pr(True), GALAXY: _issue("closed"), + LENS: _issue("closed"), FIT_ISSUE: _issue("closed")}, write=True) + assert needle in out + assert _fields(root, P["02_gated_on_dev"])["State"] == "gated" + assert rc == (1 if "unreadable" in needle else 0) + + +def test_grade_write_demotes_ready_on_reopen_but_not_with_override(tmp_path): + root = _copy(tmp_path) + states = {ARRAY: _issue("open"), FIT_PR: _pr(False, "open"), GALAXY: _issue("open"), + LENS: _issue("open"), FIT_ISSUE: _issue("open")} + out, rc = _grade(root, states) + assert "03_ready_cleared.md: ready" in out and "(gates --grade --write demotes it to gated)" in out + assert _fields(root, P["03_ready_cleared"])["State"] == "ready" + out, rc = _grade(root, states, write=True) + assert "03_ready_cleared.md: ready\n PyAutoGalaxy#486 → open (issue open)\n verdict: reopened PyAutoGalaxy#486 — ready → gated" in out + f = _fields(root, P["03_ready_cleared"]) + assert f["State"] == "gated" and "Gates-cleared" not in f + # submitted+ phases: reported, never enforced; override: kept ready + assert "04_submitted_override.md: submitted\n PyAutoLens#900 → open (issue open)\n verdict: reopened PyAutoLens#900 — reported, never enforced past ready" in out + assert "08_accepted.md" not in out # accepted is not graded + assert _fields(root, P["04_submitted_override"])["State"] == "submitted" + _edit(root, P["03_ready_cleared"], "State: gated\n", "State: ready\nGate-override: on purpose\n") + out, rc = _grade(root, states, write=True) + assert "Gate-override: present, kept ready" in out + assert _fields(root, P["03_ready_cleared"])["State"] == "ready" + assert _problems(root) == [] + + +def test_grade_unreadable_ready_phase_is_not_demoted(tmp_path): + root = _copy(tmp_path) + out, rc = _grade(root, {GALAXY: "unreadable: HTTP 500"}, write=True) + assert rc == 1 + assert _fields(root, P["03_ready_cleared"])["State"] == "ready" + + +def test_gh_jq_shape_and_http_fallback_user_agent(): + assert cortex.GATE_JQ == ('{state, state_reason, merged_at: .pull_request.merged_at, ' + 'is_pr: (.pull_request != null)}') + src = SCRIPT.read_text() + assert '"User-Agent": "pyautocortex"' in src + + +# --------------------------------------------------------------------------- # +# new +# --------------------------------------------------------------------------- # +def test_new_writes_a_parseable_phase(tmp_path): + root = _copy(tmp_path) + r = _run("new", "example", "11_next", "--phase", "11", "--gates", "PyAutoLens#1, https://github.com/o/r/pull/2", + "--epic", "example-programme", "--budget", "6:00", "--minutes", "5", "--witness", "a claim", + "--today", TODAY, root=root) + assert r.returncode == 0, r.stderr + assert r.stdout.strip() == "wrote phases/example/11_next.md" + text = (root / "phases/example/11_next.md").read_text() + assert text.startswith("# Example — phase 11: 11 next\n\nProject: example\nPhase: 11\nState: planned\n") + f = _fields(root, "phases/example/11_next.md") + assert f["Gates"] == "PyAutoLens#1, https://github.com/o/r/pull/2" and f["Lane"] == "local-dev" + assert f["Filed"] == TODAY and f["Budget"] == "6:00" and f["Review-minutes"] == "5" + for name in cortex.PHASE_SECTIONS: + assert f"\n## {name}\n" in text + assert _problems(root) == [] + assert _move(root, "phases/example/11_next.md", "gated").returncode == 0 + + +def test_new_refuses_duplicates_and_unknown_projects(tmp_path): + root = _copy(tmp_path) + r = _run("new", "example", "03_again", "--phase", "3", root=root) + assert r.returncode == 1 and "phase 3 already exists: phases/example/03_ready_cleared.md" in r.stderr + r = _run("new", "example", "03_ready_cleared", "--phase", "11", root=root) + assert r.returncode == 1 and "already exists" in r.stderr + r = _run("new", "nobody", "x", "--phase", "1", root=root) + assert r.returncode == 1 and "not a projects.yaml key" in r.stderr + r = _run("new", "example", "x", "--phase", "11", "--gates", "PyAutoLabs/PyAutoLens#1", root=root) + assert r.returncode == 1 and "no owner/Repo#N form" in r.stderr + assert not (root / "phases/example/x.md").exists() + + +def test_new_legacy_born_phase_is_ready_and_pullable(tmp_path): + root = _copy(tmp_path) + r = _run("new", "example", "12_legacy", "--phase", "12", "--legacy-run", "300000", + "--legacy-wrong", "300001_[0-3]", "--today", TODAY, root=root) + assert r.returncode == 1 and "--where" in r.stderr + r = _run("new", "example", "12_legacy", "--phase", "12", "--legacy-run", "300000", + "--legacy-wrong", "300001_[0-3]", "--where", "/mnt/q", "--witness", "w", "--today", TODAY, root=root) + assert r.returncode == 0, r.stderr + f = _fields(root, "phases/example/12_legacy.md") + assert f["State"] == "ready" and f["Runs"] == "300000, 300001" + text = (root / "phases/example/12_legacy.md").read_text() + assert "- 300000: legacy — gpu — submitted 2026-09-02 — wall 0:00 — pre-Cortex run, migrated\n where: /mnt/q\n" in text + assert "- 300001_[0-3]: legacy_wrong — gpu" in text + assert _problems(root) == [] + assert _move(root, "phases/example/12_legacy.md", "pulled").returncode == 0 + text = (root / "phases/example/12_legacy.md").read_text() + # the reusable run gets pulled_to: (its where:), the wrong one does not + assert " where: /mnt/q\n pulled_to: /mnt/q\n- 300001_[0-3]: legacy_wrong" in text + assert text.count("pulled_to:") == 1 + assert _problems(root) == [] + # legacy_wrong-only: nothing to pull — rule drop is the door + r = _run("new", "example", "14", "--phase", "14", "--legacy-wrong", "2", "--where", "/q", + "--witness", "w", root=root) + assert r.returncode == 0, r.stderr + r = _move(root, "phases/example/14.md", "pulled") + assert r.returncode == 1 and "no legacy run to pull" in r.stderr + r = _run("new", "example", "13", "--phase", "13", "--legacy-run", "1", "--where", "/q", + "--gates", "PyAutoLens#1", root=root) + assert r.returncode == 1 and "cannot be gated" in r.stderr + + +def test_partition_comes_from_the_project_row_or_the_flag(tmp_path): + root = _copy(tmp_path) + _edit(root, "projects.yaml", " partition: gpu\n", " partition: both\n") + r = _move(root, P["03_ready_cleared"], "submitted", "--run", "1") + assert r.returncode == 1 and "--partition" in r.stderr + r = _move(root, P["03_ready_cleared"], "submitted", "--run", "1", "--partition", "ral") + assert r.returncode == 0 + assert "- 1: submitted — ral — " in (root / P["03_ready_cleared"]).read_text() + + +def test_bad_root_and_bad_today(): + r = _run("check", root=Path("/nonexistent/tree")) + assert r.returncode == 2 + r = _run("move", P["01_scope"], "ready", "--today", "yesterday", root=SKELETON) + assert r.returncode == 1 and "--today must be YYYY-MM-DD" in r.stderr diff --git a/tests/test_ledger_merge.py b/tests/test_ledger_merge.py new file mode 100644 index 0000000..03015ed --- /dev/null +++ b/tests/test_ledger_merge.py @@ -0,0 +1,228 @@ +"""Tests for scripts/ledger_merge.py — the auto-merge safety gate. + +This gate decides what lands on `main` with no human in the loop, so the +properties that matter are the refusals: default deny for anything +unclassified, no traversal or dotfile route past the ledger prefixes, no +pytest-collectable file smuggled in beside a packet, every code home the repo +has (scripts/, tests/, .github/, policy/, docs/, projects.yaml, the prose +pages) staying on the human side of the line — and, the Cortex's own addition, +a MODIFIED, DELETED or RENAMED ruling never auto-merging (rulings/ is +append-only). +""" + +import subprocess +import sys +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "scripts")) + +import ledger_merge # noqa: E402 + +SCRIPT = Path(ledger_merge.__file__) +REPO = SCRIPT.resolve().parents[1] + + +def test_ledger_dirs_and_registry_files_are_ledger(): + for path in ( + "phases/example/01_scope.md", + "phases/newproject/01_first.md", + "rulings/2026/09/R-20260901-01.md", + "batches/2026-09-01-pm.md", + "batches/reviews/2026-09-01-pm.md", + "batches/packets/2026-09-01-pm.html", + "epics.md", + ): + assert ledger_merge.is_ledger_path(path), path + + +def test_every_code_home_needs_a_human(): + for path in ( + "scripts/cortex.py", + "scripts/ledger_merge.py", + "tests/test_cortex.py", + "tests/fixtures/skeleton/phases/example/01_scope.md", + ".github/workflows/ledger_merge.yml", + ".github/workflows/cortex_check.yml", + ".gitignore", + ".claude/settings.json", + ".claude/hooks/session-start.sh", + "policy/never_rewrite_history.md", + "docs/schema_decisions.md", + # CODE: sync_cli and local_path are paths the conductor executes under. + "projects.yaml", + "README.md", + "AGENTS.md", + "CLAUDE.md", + "REFERENCE.md", + "LICENSE", + ): + assert not ledger_merge.is_ledger_path(path), path + + +def test_unclassified_paths_default_to_deny(): + """A root file or top-level folder nobody has thought about is code.""" + for path in ("brand_new_root_file.md", "newfolder/thing.md", "notes.txt", "skills/x/SKILL.md"): + assert not ledger_merge.is_ledger_path(path), path + + +def test_traversal_cannot_smuggle_code_behind_a_ledger_prefix(): + for path in ("phases/../scripts/evil.py", "rulings/../../etc/passwd", ".."): + assert not ledger_merge.is_ledger_path(path), path + + +def test_dot_paths_are_never_ledger_wherever_they_sit(): + for path in ("phases/.github/workflows/x.yml", "batches/.hidden", ".phases/x.md"): + assert not ledger_merge.is_ledger_path(path), path + + +def test_inert_assets_ride_along_but_collectable_tests_do_not(): + assert ledger_merge.is_ledger_path("batches/packets/2026-09-01-pm.html") + for path in ( + "batches/packets/conftest.py", + "phases/example/test_thing.py", + "rulings/2026/09/thing_test.py", + ): + assert not ledger_merge.is_ledger_path(path), path + + +def test_classify_splits_and_dedupes_preserving_order(): + ledger, blocked = ledger_merge.classify( + ["epics.md", "scripts/x.py", "epics.md", "phases/a/b.md", "", " "] + ) + assert ledger == ["epics.md", "phases/a/b.md"] + assert blocked == ["scripts/x.py"] + + +# --------------------------------------------------------------------------- # +# append-only rulings/ +# --------------------------------------------------------------------------- # +def test_added_ruling_is_ledger_but_any_other_status_is_code(): + rid = "rulings/2026/09/R-20260901-01.md" + assert not ledger_merge.is_append_only_violation("A", rid) + for status in ("M", "D", "R100", "R", "T", "C75", "U"): + assert ledger_merge.is_append_only_violation(status, rid), status + # the leg is rulings/ only — a modified phase or batch record is still ledger + assert not ledger_merge.is_append_only_violation("M", "phases/example/01_scope.md") + assert not ledger_merge.is_append_only_violation("D", "batches/2026-09-01-pm.md") + + +def test_classify_entries_blocks_a_modified_ruling(): + ledger, blocked = ledger_merge.classify_entries([ + ("A", "rulings/2026/09/R-20260901-06.md"), + ("M", "rulings/2026/09/R-20260901-01.md"), + ("M", "phases/example/08_accepted.md"), + ("D", "batches/2026-09-01-pm.md"), + ("A", "scripts/new.py"), + ]) + assert ledger == ["rulings/2026/09/R-20260901-06.md", "phases/example/08_accepted.md", + "batches/2026-09-01-pm.md"] + assert blocked == ["rulings/2026/09/R-20260901-01.md", "scripts/new.py"] + + +def _git(cwd, *args): + return subprocess.run(["git", *args], cwd=cwd, capture_output=True, text=True, check=True) + + +def _repo(tmp_path): + repo = tmp_path / "repo" + repo.mkdir() + _git(repo, "init", "-q", "-b", "main") + _git(repo, "config", "user.email", "t@example.com") + _git(repo, "config", "user.name", "t") + (repo / "rulings" / "2026" / "09").mkdir(parents=True) + (repo / "phases" / "p").mkdir(parents=True) + (repo / "rulings/2026/09/R-20260901-01.md").write_text("# R-20260901-01\n\nRuling: accept\n") + (repo / "phases/p/01_a.md").write_text("# a\n\nState: ready\n") + (repo / "epics.md").write_text("# Epics\n") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "base") + _git(repo, "checkout", "-q", "-b", "claude/work") + return repo + + +def test_changed_entries_reads_name_status_from_git(tmp_path): + repo = _repo(tmp_path) + (repo / "rulings/2026/09/R-20260901-02.md").write_text("# R-20260901-02\n") + (repo / "phases/p/01_a.md").write_text("# a\n\nState: submitted\n") + (repo / "rulings/2026/09/R-20260901-01.md").write_text("# R-20260901-01\n\nRuling: drop\n") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "work") + entries = ledger_merge.changed_entries("main", cwd=repo) + assert sorted(entries) == [("A", "rulings/2026/09/R-20260901-02.md"), + ("M", "phases/p/01_a.md"), + ("M", "rulings/2026/09/R-20260901-01.md")] + ledger, blocked = ledger_merge.classify_entries(entries) + assert blocked == ["rulings/2026/09/R-20260901-01.md"] + assert set(ledger) == {"rulings/2026/09/R-20260901-02.md", "phases/p/01_a.md"} + + +def test_a_renamed_or_deleted_ruling_is_code_but_added_ones_merge(tmp_path): + repo = _repo(tmp_path) + _git(repo, "mv", "rulings/2026/09/R-20260901-01.md", "rulings/2026/09/R-20260901-03.md") + _git(repo, "commit", "-q", "-m", "rename") + entries = ledger_merge.changed_entries("main", cwd=repo) + _, blocked = ledger_merge.classify_entries(entries) + assert "rulings/2026/09/R-20260901-01.md" in blocked + assert "rulings/2026/09/R-20260901-03.md" in blocked + # a branch that only ADDS rulings and moves phases is ledger-only + _git(repo, "checkout", "-q", "main") + _git(repo, "checkout", "-q", "-b", "claude/clean") + (repo / "rulings/2026/09/R-20260901-04.md").write_text("# R-20260901-04\n") + (repo / "phases/p/01_a.md").write_text("# a\n\nState: accepted\n") + (repo / "epics.md").write_text("# Epics\n\n## x\n") + _git(repo, "add", "-A") + _git(repo, "commit", "-q", "-m", "clean") + ledger, blocked = ledger_merge.classify_entries(ledger_merge.changed_entries("main", cwd=repo)) + assert blocked == [] and len(ledger) == 3 + + +def _run(*args, stdin=""): + return subprocess.run( + [sys.executable, str(SCRIPT), "classify", *args], + input=stdin, + capture_output=True, + text=True, + ) + + +def test_cli_exit_codes_separate_ledger_from_code(): + assert _run("epics.md", "phases/a/b.md", "rulings/2026/09/R-20260901-01.md").returncode == 0 + result = _run("epics.md", "scripts/cortex.py") + assert result.returncode == 1 + assert "scripts/cortex.py" in result.stdout + + +def test_explicit_paths_carry_no_status_and_stay_path_only(): + """Without a git diff there is no status to read, so a ruling path given + by hand is judged on its path alone — as in the Mind.""" + assert _run("rulings/2026/09/R-20260901-01.md").returncode == 0 + assert _run(stdin="rulings/2026/09/R-20260901-01.md\n").returncode == 0 + + +def test_an_empty_diff_is_not_permission_to_merge(): + """Exit 0 means 'go'. Nothing to merge must never read as go.""" + result = _run(stdin="\n") + assert result.returncode == 1 + assert "nothing to merge" in result.stdout + + +def test_every_fixture_path_is_classified(): + """The witness: every path under tests/fixtures/ gets a verdict, and it is + code — `tests/` is a code home, whatever shape the files inside take.""" + out = subprocess.run(["git", "ls-files", "tests/fixtures"], cwd=REPO, + capture_output=True, text=True, check=True).stdout.split() + assert out, "no fixture paths tracked" + for path in out: + assert not ledger_merge.is_ledger_path(path), path + + +def test_this_repos_own_workflow_cannot_auto_merge_itself(): + """Self-consistency: the gate is on the code side of its own line.""" + assert not ledger_merge.is_ledger_path(".github/workflows/ledger_merge.yml") + assert not ledger_merge.is_ledger_path("scripts/ledger_merge.py") + + +def test_workflow_invokes_the_script_as_implemented(): + workflow = (REPO / ".github" / "workflows" / "ledger_merge.yml").read_text() + assert "python3 scripts/ledger_merge.py classify --base origin/main" in workflow + assert "python3 scripts/cortex.py check" in workflow