From 28e03e95b0a98202897ca118ffedefea2d9a9888 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:44:00 +0100 Subject: [PATCH 01/92] docs(run-to-fix): write the adapter contract The adapter is the one file that holds everything about an agent: its checkout, its commands with their placeholders, the limits of the run and the commands that cost money. The loop checks each ref in a detached worktree and removes it when the check ends. --- .../skills/run-to-fix/references/adapter.md | 151 ++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 .claude/skills/run-to-fix/references/adapter.md diff --git a/.claude/skills/run-to-fix/references/adapter.md b/.claude/skills/run-to-fix/references/adapter.md new file mode 100644 index 00000000..96ee2b38 --- /dev/null +++ b/.claude/skills/run-to-fix/references/adapter.md @@ -0,0 +1,151 @@ +# The adapter contract + +An adapter is one JSON file per agent, `adapter.json` in the loop folder, schema `run-to-fix.adapter/v1`. Every +value that belongs to one agent comes from this file: where its code is, how its tests run, how a run of chosen +tasks starts, and the limits of the run that was read. The skill itself holds nothing about any agent. + +For an agent of the Nullius catalogue, the assistant drafts the adapter from +`app/nullius/catalogue/agents//agent.json` (`versions..upstream.repository` and `tag`) and the person +confirms it. `loop.py read` refuses an adapter that does not satisfy this page, and records the SHA-256 of the file +in `round-NN/input.json`. + +## Keys + +| key | type | meaning | +|---|---|---| +| `schema` | string | `run-to-fix.adapter/v1` | +| `agent` | string | the value of `agent_info.name` in the trials of the run; `read` refuses a run whose trials name another agent | +| `source.checkout` | path | the agent's git checkout on this machine | +| `source.base_ref` | git ref | the commit the run was built from | +| `source.work_branch` | branch | the branch that receives test and fix commits | +| `commands.test` | command or null | runs the agent's whole test suite on `{worktree}` | +| `commands.test_one` | command or null | runs one test file, `{test}`, on `{worktree}` | +| `commands.build` | command or null | builds what a run installs from `{worktree}` and prints its path as the last line of its output; null when a run installs the checkout | +| `commands.run_tasks` | command | runs the tasks of `{units_file}`, `{repeat}` trials each, into `{jobs_dir}`, from `{build}` or `{ref}`; `{budget_usd}` is the budget the person named, for a runner that takes a spending cap | +| `commands.replay` | command or null | replays the recorded trials of `{job_dir}` on the code of `{ref}`, checked out at `{worktree}`, and writes `{out_json}` | +| `commands.classes` | command or null | writes the agent's own classes and flags for the trials of `{job_dir}` to `{out_json}` | +| `test_failed_pattern` | regular expression or null | one capture group that takes the id of a failed test from the output of `commands.test`; required when `commands.test` is set | +| `replay_success_labels` | list of strings | the replay labels that mean the agent reports success, for example `["done"]`; required when `commands.replay` is set | +| `limits.step_limit` | integer or null | the run's limit on model calls per trial | +| `limits.output_token_limit` | integer or null | the run's limit on completion tokens per model reply | +| `limits.time_limit_s` | number or null | the run's limit on the agent phase of a trial, in seconds | +| `paid` | list of command names | the commands that cost money, for example `["run_tasks"]` | + +Every key is present, and a value the agent does not have is null. `read` refuses an adapter with a missing key. + +## Placeholders + +| placeholder | given to | value | +|---|---|---| +| `{worktree}` | `test`, `test_one`, `build`, `replay` | a detached git worktree of the ref under check, made by the loop under `/worktrees/` | +| `{test}` | `test_one` | the path of one test file, relative to `{worktree}` | +| `{ref}` | `replay`, `run_tasks` | the full commit id under check | +| `{job_dir}` | `replay`, `classes` | the job folder the person gave to `loop.py read` | +| `{out_json}` | `replay`, `classes` | the file the command writes; the loop reads it after a zero exit | +| `{units_file}` | `run_tasks` | a text file with one task name per line | +| `{repeat}` | `run_tasks` | trials per task | +| `{jobs_dir}` | `run_tasks` | where the run writes its job folder | +| `{budget_usd}` | `run_tasks` | the budget the person named for this run, in US dollars | +| `{build}` | `run_tasks` | the path `commands.build` printed; empty when `commands.build` is null | + +A command is one line of POSIX shell. The loop replaces each placeholder with its value quoted for the shell. +Write each placeholder without quotes around it. A placeholder that a command does not take is refused when +the adapter is read, and so is a command that leaves out a placeholder it needs to do its work (`{out_json}` for +`replay` and `classes`, `{test}` for `test_one`). + +## How the loop runs a command + +1. The loop and every command run on one machine. A command with `{worktree}` runs with that worktree as its + working directory; every other command runs in `source.checkout`. +2. For a check on a ref, the loop makes a detached worktree of that ref with `git worktree add --detach`, runs the + command, and removes the worktree when the check ends, on failure as well. The checkout stays on its branch and + its working tree is never changed. The command decides how the agent's environment reaches the code in + `{worktree}`, for example by putting it on `PYTHONPATH` or by installing it into a temporary environment. +3. A command named in `paid` is never started by the loop. `loop.py` writes it, with every placeholder filled, to + `round-NN/commands/.sh` and stops. A person starts it, and the loop goes on from the job folder it + produced. The written line is complete as text: the task list, the repeats, the job folder and the budget are all + in it. +4. A null command switches its check off. The decision page then says that the check did not run, and which + evidence is missing because of it. +5. The loop writes the output of every command to a file under `round-NN/`. A command that has not + exited within the time `loop.py` was given is stopped, and the check is recorded as not finished. + +## Exit codes and output + +| command | success | what the loop reads | +|---|---|---| +| `test` | any exit code | the failed test ids that `test_failed_pattern` takes from the output. A non-zero exit with no failed test in the output is a broken suite and stops the check | +| `test_one` | 0 means the test passed, any other code means it failed | the exit code | +| `build` | 0 | the last line of the output: the path of what was built, recorded in `round-NN/build.json` with its SHA-256 | +| `replay` | 0 | `{out_json}`, below | +| `classes` | 0 | `{out_json}`, below | +| `run_tasks` | written, not run | the job folder under `{jobs_dir}` that the person names afterwards | + +A command other than `test` and `test_one` that exits non-zero stops the check, and the loop prints the last lines +of its output. + +The failures of `commands.test` are compared by id: a candidate keeps the failures the suite had on `base_ref` and +adds none. A failure on `base_ref` that a candidate removes is reported as removed. + +### The replay file + +```json +{ + "ref": "", + "trials": { + "": {"label": "done", "touched": true} + }, + "kept_recorded": [""] +} +``` + +1. `trials` has one entry for every trial of `{job_dir}`. `label` is what the agent reports for that trial on the + code of `ref`. +2. `touched` is optional: true when the code that changed between `base_ref` and `ref` ran in the trial. Guard tasks + of a trial run come from this field. Without it the list of guards is empty, and the decision page says so. +3. `kept_recorded` is optional: the trials the replay could not reproduce, which keep the label of the recorded run. + The decision page gives their count. + +The loop crosses each label with the trial's recorded pass or fail. A label in `replay_success_labels` on a failed +trial is a false success. A candidate's table before and after is these four counts on `base_ref` and on the +candidate, and the false successes it adds are the difference. + +### The classes file + +```json +{ + "trials": { + "": {"classes": [""], "flags": [""]} + } +} +``` + +Each class and each flag becomes an extra column of `trials.csv`. The seven classes of `references/classes.md` +stay as they are: an agent's own classes are extra columns beside them. + +## A minimal adapter + +An agent with no test suite, no replay and no known limits: + +```json +{ + "schema": "run-to-fix.adapter/v1", + "agent": "example-agent", + "source": {"checkout": "/home/me/example-agent", "base_ref": "a1b2c3d", "work_branch": "fix/run-to-fix"}, + "commands": { + "test": null, + "test_one": null, + "build": null, + "run_tasks": "example-run --tasks {units_file} --repeat {repeat} --out {jobs_dir} --max-usd {budget_usd} --ref {ref}", + "replay": null, + "classes": null + }, + "test_failed_pattern": null, + "replay_success_labels": [], + "limits": {"step_limit": null, "output_token_limit": null, "time_limit_s": null}, + "paid": ["run_tasks"] +} +``` + +With this adapter the loop still reads, classifies and compares runs. It cannot prove a cause with a test, compare +a candidate on recorded trials, or pick guard tasks. Every decision page names the checks that did not run. From df72eef3b415dc6830879c517e905d007ff7e12a Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:45:37 +0100 Subject: [PATCH 02/92] docs(run-to-fix): name the placeholders each adapter command needs --- .../skills/run-to-fix/references/adapter.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/.claude/skills/run-to-fix/references/adapter.md b/.claude/skills/run-to-fix/references/adapter.md index 96ee2b38..2002a56d 100644 --- a/.claude/skills/run-to-fix/references/adapter.md +++ b/.claude/skills/run-to-fix/references/adapter.md @@ -21,7 +21,7 @@ in `round-NN/input.json`. | `commands.test` | command or null | runs the agent's whole test suite on `{worktree}` | | `commands.test_one` | command or null | runs one test file, `{test}`, on `{worktree}` | | `commands.build` | command or null | builds what a run installs from `{worktree}` and prints its path as the last line of its output; null when a run installs the checkout | -| `commands.run_tasks` | command | runs the tasks of `{units_file}`, `{repeat}` trials each, into `{jobs_dir}`, from `{build}` or `{ref}`; `{budget_usd}` is the budget the person named, for a runner that takes a spending cap | +| `commands.run_tasks` | command or null | runs the tasks of `{units_file}`, `{repeat}` trials each, into `{jobs_dir}`, from `{build}` or `{ref}`; `{budget_usd}` is the budget the person named, for a runner that takes a spending cap | | `commands.replay` | command or null | replays the recorded trials of `{job_dir}` on the code of `{ref}`, checked out at `{worktree}`, and writes `{out_json}` | | `commands.classes` | command or null | writes the agent's own classes and flags for the trials of `{job_dir}` to `{out_json}` | | `test_failed_pattern` | regular expression or null | one capture group that takes the id of a failed test from the output of `commands.test`; required when `commands.test` is set | @@ -49,9 +49,20 @@ Every key is present, and a value the agent does not have is null. `read` refuse | `{build}` | `run_tasks` | the path `commands.build` printed; empty when `commands.build` is null | A command is one line of POSIX shell. The loop replaces each placeholder with its value quoted for the shell. -Write each placeholder without quotes around it. A placeholder that a command does not take is refused when -the adapter is read, and so is a command that leaves out a placeholder it needs to do its work (`{out_json}` for -`replay` and `classes`, `{test}` for `test_one`). +Write each placeholder without quotes around it. When the adapter is read, the loop refuses a placeholder that a +command does not take, and a command that leaves out one it needs: + +| command | needs | +|---|---| +| `test` | `{worktree}` | +| `test_one` | `{worktree}`, `{test}` | +| `build` | `{worktree}` | +| `replay` | `{worktree}`, `{ref}`, `{job_dir}`, `{out_json}` | +| `classes` | `{job_dir}`, `{out_json}` | +| `run_tasks` | `{units_file}`, `{repeat}`, `{jobs_dir}`, and `{build}` or `{ref}` | + +When `commands.run_tasks` is null, the decision page lists the tasks and the person starts the run by other +means. ## How the loop runs a command From 7530ae977f5272eed50b307f8308ab024e2ba985 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:48:39 +0100 Subject: [PATCH 03/92] test(run-to-fix): cover read and classify on hand-made jobs --- .claude/skills/run-to-fix/tests/jobs.py | 195 ++++++++++ .../skills/run-to-fix/tests/test_classify.py | 349 ++++++++++++++++++ .claude/skills/run-to-fix/tests/test_read.py | 168 +++++++++ 3 files changed, 712 insertions(+) create mode 100644 .claude/skills/run-to-fix/tests/jobs.py create mode 100644 .claude/skills/run-to-fix/tests/test_classify.py create mode 100644 .claude/skills/run-to-fix/tests/test_read.py diff --git a/.claude/skills/run-to-fix/tests/jobs.py b/.claude/skills/run-to-fix/tests/jobs.py new file mode 100644 index 00000000..51c1ef66 --- /dev/null +++ b/.claude/skills/run-to-fix/tests/jobs.py @@ -0,0 +1,195 @@ +"""Hand-made job folders in the two shapes Nullius writes, and the helpers the tests share.""" +from __future__ import annotations + +import csv +import datetime as dt +import json +import os +import shutil +import subprocess +import sys +import tempfile +import unittest +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +HERE = os.path.dirname(os.path.abspath(__file__)) +SKILL = os.path.dirname(HERE) +LOOP = os.path.join(SKILL, "scripts", "loop.py") +T0 = dt.datetime(2026, 11, 1, 12, 0, 0) + + +@dataclass +class Trial: + name: str + task: str + reward: Optional[float] + exception: Optional[Tuple[str, str]] + agent_s: float + calls: int + max_output: int + verifier_text: str + files: Dict[str, str] + + +def trial(name: str, task: str, reward: Optional[float]) -> Trial: + """A trial of 5 model calls, 60 s, 100 tokens per reply, no exception, no extra files.""" + return Trial(name=name, task=task, reward=reward, exception=None, agent_s=60.0, calls=5, max_output=100, + verifier_text="", files={}) + + +def iso(t: dt.datetime) -> str: + return t.isoformat() + + +def write(path: str, text: str) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8", newline="\n") as fh: + fh.write(text) + + +def write_json(path: str, data: object) -> None: + write(path, json.dumps(data, indent=1)) + + +def trajectory(t: Trial) -> dict: + steps: List[dict] = [{"step_id": 0, "source": "user", "timestamp": iso(T0), "message": "task"}] + for i in range(t.calls): + out = t.max_output if i == t.calls - 1 else min(100, t.max_output) + steps.append({"step_id": i + 1, "source": "agent", "timestamp": iso(T0 + dt.timedelta(seconds=i + 1)), + "message": "step", "metrics": {"prompt_tokens": 1000, "completion_tokens": out}}) + return {"schema_version": "ATIF-v1.2", "agent": {"name": "fixture"}, "steps": steps} + + +def write_trial(job_dir: str, t: Trial, agent: str, model: str, runner: str, index: int) -> None: + d = os.path.join(job_dir, t.name) + start = T0 + dt.timedelta(minutes=index) + agent_start = start + dt.timedelta(seconds=10) + agent_end = agent_start + dt.timedelta(seconds=t.agent_s) + end = agent_end + dt.timedelta(seconds=5) + res: dict = { + "task_name": t.task, "trial_name": t.name, "started_at": iso(start), "finished_at": iso(end), + "agent_info": {"name": agent, "version": "1.0", "model_info": {"name": model}}, + "environment_setup": {"started_at": iso(start), "finished_at": iso(agent_start)}, + "agent_execution": {"started_at": iso(agent_start), "finished_at": iso(agent_end)}, + "agent_result": {"n_input_tokens": 1000 * t.calls, "n_output_tokens": 100 * t.calls, "n_cache_tokens": 0, + "cost_usd": 0.01}, + } + if t.reward is not None: + res["verifier"] = {"started_at": iso(agent_end), "finished_at": iso(end)} + res["verifier_result"] = {"rewards": {"reward": t.reward}} + if t.exception is not None: + res["exception_info"] = {"exception_type": t.exception[0], "exception_message": t.exception[1]} + write(os.path.join(d, "exception.txt"), t.exception[1] + "\n") + if runner == "standard": + res["agent_result"]["n_model_calls"] = t.calls + res["standard"] = {"benchmark": "made-up-bench", "unit": t.task.split("/")[-1], "plugged": f"{agent}@1.0", + "job": os.path.basename(job_dir)} + if t.reward is not None: + write_json(os.path.join(d, "verifier", "score.json"), + {"reward": t.reward, "reason": t.verifier_text, "says": "pass"}) + elif t.reward is not None: + write(os.path.join(d, "verifier", "reward.txt"), f"{t.reward}\n") + write(os.path.join(d, "verifier", "test-stdout.txt"), t.verifier_text + "\n") + write_json(os.path.join(d, "agent", "trajectory.json"), trajectory(t)) + for rel, text in t.files.items(): + write(os.path.join(d, rel), text) + write_json(os.path.join(d, "result.json"), res) + + +def write_job(root: str, name: str, agent: str, runner: str, trials: List[Trial]) -> str: + """A finished job of `trials` under root/name; runner is "harbor" or "standard".""" + job_dir = os.path.join(root, name) + os.makedirs(job_dir, exist_ok=True) + config: dict = {"job_name": name, "n_concurrent_trials": 2, + "agents": [{"name": agent, "model_name": "made-up-model"}]} + if runner == "standard": + config.update({"runner": "standard", "runner_label": "Nullius", + "standard": {"benchmark": "made-up-bench", "plugged": f"{agent}@1.0", "kind": "agent"}}) + write_json(os.path.join(job_dir, "config.json"), config) + for i, t in enumerate(trials): + write_trial(job_dir, t, agent, "made-up-model", runner, i) + write_json(os.path.join(job_dir, "result.json"), { + "started_at": iso(T0), "finished_at": iso(T0 + dt.timedelta(hours=1)), "n_total_trials": len(trials), + "stats": {"n_completed_trials": len(trials), "n_cancelled_trials": 0}}) + return job_dir + + +def adapter(agent: str, limits: Dict[str, Optional[float]], commands: Dict[str, Optional[str]]) -> dict: + """An adapter whose checkout is never used: every command not named in `commands` is null.""" + names = ("test", "test_one", "build", "run_tasks", "replay", "classes") + return { + "schema": "run-to-fix.adapter/v1", + "agent": agent, + "source": {"checkout": HERE, "base_ref": "HEAD", "work_branch": "fix/made-up"}, + "commands": {n: commands.get(n) for n in names}, + "test_failed_pattern": None, + "replay_success_labels": [], + "limits": {"step_limit": limits.get("step_limit"), "output_token_limit": limits.get("output_token_limit"), + "time_limit_s": limits.get("time_limit_s")}, + "paid": [], + } + + +NO_LIMITS: Dict[str, Optional[float]] = {} +LIMITS: Dict[str, Optional[float]] = {"step_limit": 20, "output_token_limit": 4096, "time_limit_s": 600} + + +@dataclass +class Result: + code: int + out: str + err: str + + +def run(args: List[str]) -> Result: + env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1") + p = subprocess.run([sys.executable, LOOP] + args, capture_output=True, text=True, env=env, + stdin=subprocess.DEVNULL, timeout=120) + return Result(p.returncode, p.stdout, p.stderr) + + +def load(path: str) -> object: + with open(path, encoding="utf-8") as fh: + return json.load(fh) + + +class LoopCase(unittest.TestCase): + """A temporary folder with `jobs/` and `loop/`, removed after each test.""" + + def setUp(self) -> None: + self.tmp = tempfile.mkdtemp(prefix="run-to-fix-test-") + self.jobs = os.path.join(self.tmp, "jobs") + self.loop = os.path.join(self.tmp, "loop") + os.makedirs(self.jobs) + os.makedirs(self.loop) + + def tearDown(self) -> None: + shutil.rmtree(self.tmp) + + def set_adapter(self, data: dict) -> None: + write_json(os.path.join(self.loop, "adapter.json"), data) + + def read(self, run_dir: str, baseline: Optional[str]) -> Result: + args = ["read", "--loop", self.loop, "--run", run_dir, "--budget-usd", "10", "--pass-threshold", "1"] + if baseline is not None: + args += ["--baseline", baseline] + return run(args) + + def read_ok(self, run_dir: str, baseline: Optional[str]) -> None: + r = self.read(run_dir, baseline) + self.assertEqual(r.code, 0, r.out + r.err) + + def classify_ok(self) -> None: + r = run(["classify", "--loop", self.loop]) + self.assertEqual(r.code, 0, r.out + r.err) + + def round_file(self, name: str) -> str: + return os.path.join(self.loop, "round-01", name) + + def rows(self) -> List[Dict[str, str]]: + with open(self.round_file("trials.csv"), encoding="utf-8", newline="") as fh: + return list(csv.DictReader(fh)) + + def by_trial(self) -> Dict[str, Dict[str, str]]: + return {r["trial"]: r for r in self.rows()} diff --git a/.claude/skills/run-to-fix/tests/test_classify.py b/.claude/skills/run-to-fix/tests/test_classify.py new file mode 100644 index 00000000..1c88d55e --- /dev/null +++ b/.claude/skills/run-to-fix/tests/test_classify.py @@ -0,0 +1,349 @@ +"""`loop.py classify`: the seven classes, signatures, groups, warnings and the agent's own classes.""" +from __future__ import annotations + +import os +import sys +from typing import Dict, List + +from jobs import (LIMITS, NO_LIMITS, LoopCase, Trial, adapter, load, run, trial, write, write_job, write_json) + +FIVE = {"pass", "time_limit", "exception", "no_score", "verifier_failed"} +SEVEN = FIVE | {"step_limit", "output_limit"} + + +def precedence_trials() -> List[Trial]: + passed_slowly = trial("pass_slow__a", "bench/pass-slow", 1.0) + passed_slowly.agent_s = 700 + timeout = trial("timeout__a", "bench/timeout", None) + timeout.exception = ("AgentTimeoutError", "the agent did not end in 600 s") + long_phase = trial("long_phase__a", "bench/long-phase", 0.0) + long_phase.agent_s = 600 + crash_after_long_reply = trial("crash_long_reply__a", "bench/crash-long-reply", None) + crash_after_long_reply.exception = ("RuntimeError", "repeated stream chunk 12") + crash_after_long_reply.max_output = 4096 + no_score = trial("no_score__a", "bench/no-score", None) + no_score_many_steps = trial("no_score_steps__a", "bench/no-score-steps", None) + no_score_many_steps.calls = 20 + steps = trial("steps__a", "bench/steps", 0.0) + steps.calls = 20 + output = trial("output__a", "bench/output", 0.0) + output.max_output = 4096 + failed = trial("failed__a", "bench/failed", 0.0) + partial = trial("partial__a", "bench/partial", 0.5) + return [trial("pass__a", "bench/pass", 1.0), passed_slowly, timeout, long_phase, crash_after_long_reply, + no_score, no_score_many_steps, steps, output, failed, partial] + + +class Classes(LoopCase): + def classes(self) -> Dict[str, str]: + return {name: row["class"] for name, row in self.by_trial().items()} + + def test_precedence_with_every_limit(self) -> None: + run_dir = write_job(self.jobs, "run", "alpha", "harbor", precedence_trials()) + self.set_adapter(adapter("alpha", LIMITS, {})) + self.read_ok(run_dir, None) + self.classify_ok() + self.assertEqual(self.classes(), { + "pass__a": "pass", "pass_slow__a": "pass", "timeout__a": "time_limit", "long_phase__a": "time_limit", + "crash_long_reply__a": "exception", "no_score__a": "no_score", "no_score_steps__a": "no_score", + "steps__a": "step_limit", "output__a": "output_limit", "failed__a": "verifier_failed", + "partial__a": "verifier_failed"}) + groups = load(self.round_file("groups.json")) + self.assertEqual(groups["classes"], {"pass": 2, "time_limit": 2, "exception": 1, "no_score": 2, + "step_limit": 1, "output_limit": 1, "verifier_failed": 2}) + self.assertEqual(groups["rules_off"], []) + + def test_a_null_limit_switches_its_class_off(self) -> None: + run_dir = write_job(self.jobs, "run", "alpha", "harbor", precedence_trials()) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(run_dir, None) + self.classify_ok() + got = self.classes() + self.assertEqual(got["timeout__a"], "time_limit") + self.assertEqual(got["long_phase__a"], "verifier_failed") + self.assertEqual(got["steps__a"], "verifier_failed") + self.assertEqual(got["output__a"], "verifier_failed") + self.assertEqual(load(self.round_file("groups.json"))["rules_off"], + ["time_limit by the agent phase", "step_limit", "output_limit"]) + + def test_a_made_up_agent_with_no_limits_and_no_commands(self) -> None: + timeout = trial("t2__z", "made-up-bench/u2", None) + timeout.exception = ("AgentTimeoutError", "the agent did not answer in time") + crash = trial("t3__z", "made-up-bench/u3", None) + crash.exception = ("ConnectionError", "relay closed") + capped = trial("t5__z", "made-up-bench/u5", 0.0) + capped.calls = 500 + capped.max_output = 100000 + capped.agent_s = 99999 + capped.files = {"agent/output.json": '{"status": "step_cap", "steps": 500}'} + trials = [trial("t1__z", "made-up-bench/u1", 1.0), timeout, crash, trial("t4__z", "made-up-bench/u4", None), + capped, trial("t6__z", "made-up-bench/u6", 0.0)] + run_dir = write_job(self.jobs, "zephyr-run", "zephyr", "standard", trials) + self.set_adapter(adapter("zephyr", NO_LIMITS, {})) + self.read_ok(run_dir, None) + self.classify_ok() + got = self.classes() + self.assertTrue(set(got.values()) <= FIVE, got) + self.assertEqual(set(got.values()), FIVE) + self.assertEqual(got["t5__z"], "verifier_failed") + + +class Signatures(LoopCase): + def test_numbers_are_masked_and_verifier_text_is_read(self) -> None: + e1 = trial("e1__a", "bench/e1", None) + e1.exception = ("ValueError", "Traceback (most recent call last):\n ...\nValueError: bad token at 12") + e2 = trial("e2__a", "bench/e2", None) + e2.exception = ("ValueError", "Traceback (most recent call last):\n ...\nValueError: bad token at 345") + ctrf = trial("ctrf__a", "bench/ctrf", 0.0) + ctrf.files = {"verifier/ctrf.json": '{"results": {"tests": [{"name": "test_a", "status": "passed"}, ' + '{"name": "test_b", "status": "failed"}]}}'} + stdout = trial("stdout__a", "bench/stdout", 0.0) + stdout.verifier_text = "ok test_one\nFAILED test_total - assert 3 == 4\nFAILED test_other" + silent = trial("silent__a", "bench/silent", 0.0) + run_dir = write_job(self.jobs, "run", "alpha", "harbor", [e1, e2, ctrf, stdout, silent, + trial("p__a", "bench/p", 1.0)]) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(run_dir, None) + self.classify_ok() + rows = self.by_trial() + self.assertEqual(rows["e1__a"]["signature"], "exception: ValueError: bad token at #") + self.assertEqual(rows["e2__a"]["signature"], rows["e1__a"]["signature"]) + self.assertEqual(rows["ctrf__a"]["signature"], "verifier_failed: test_b") + self.assertEqual(rows["stdout__a"]["signature"], "verifier_failed: FAILED test_total - assert # == #") + self.assertEqual(rows["silent__a"]["signature"], "verifier_failed") + self.assertEqual(rows["p__a"]["signature"], "pass") + + def test_the_standard_score_reason(self) -> None: + wrong = trial("w__s", "made-up-bench/w", 0.0) + wrong.verifier_text = "expected 42, got 41" + run_dir = write_job(self.jobs, "run", "alpha", "standard", [wrong, trial("p__s", "made-up-bench/p", 1.0)]) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(run_dir, None) + self.classify_ok() + self.assertEqual(self.by_trial()["w__s"]["signature"], "verifier_failed: expected #, got #") + + +def shared_failure(name: str, task: str) -> Trial: + t = trial(name, task, 0.0) + t.files = {"verifier/ctrf.json": '{"results": {"tests": [{"name": "test_x", "status": "failed"}]}}'} + return t + + +class GroupsAndPairs(LoopCase): + def test_groups_warnings_and_pairs(self) -> None: + run_trials = [shared_failure(f"{t}__r", f"bench/{t}") for t in "abcd"] + run_trials += [trial("e__r", "bench/e", 0.0), trial("f__r", "bench/f", 1.0)] + run_trials += [trial(f"p{i}__r", f"bench/p{i}", 1.0) for i in range(14)] + base_trials = [shared_failure(f"{t}__b", f"bench/{t}") for t in "abcd"] + base_trials += [trial("e__b", "bench/e", 1.0), trial("f__b", "bench/f", 0.0)] + base_trials += [trial(f"p{i}__b", f"bench/p{i}", 1.0) for i in range(14)] + run_dir = write_job(self.jobs, "run", "alpha", "harbor", run_trials) + base_dir = write_job(self.jobs, "base", "alpha", "harbor", base_trials) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(run_dir, base_dir) + self.classify_ok() + + rows = self.by_trial() + self.assertEqual(rows["a__r"]["pair"], "neither") + self.assertEqual(rows["e__r"]["pair"], "baseline_only") + self.assertEqual(rows["f__r"]["pair"], "run_only") + self.assertEqual(rows["p0__r"]["pair"], "both") + + groups = load(self.round_file("groups.json")) + self.assertEqual(groups["failures"], 5) + shared = next(g for g in groups["groups"] if g["signature"] == "verifier_failed: test_x") + self.assertEqual(shared["trials"], 4) + self.assertEqual(sorted(shared["trial_names"]), ["a__r", "b__r", "c__r", "d__r"]) + kinds = {w["kind"]: w for w in shared["warnings"]} + self.assertEqual(set(kinds), {"frequent", "fails_on_baseline"}) + self.assertIn("benchmark", kinds["fails_on_baseline"]["text"]) + self.assertEqual(kinds["frequent"]["trials"], 4) + lone = next(g for g in groups["groups"] if g["trial_names"] == ["e__r"]) + self.assertEqual(lone["warnings"], []) + self.assertEqual(groups["groups"][0]["id"], "g01") + self.assertEqual(groups["groups"][0]["signature"], "verifier_failed: test_x") + + def test_fails_in_every_repeat(self) -> None: + trials = [shared_failure("a1__r", "bench/a"), shared_failure("a2__r", "bench/a"), + shared_failure("b1__r", "bench/b"), shared_failure("b2__r", "bench/b"), + shared_failure("c1__r", "bench/c"), trial("c2__r", "bench/c", 1.0), + trial("p1__r", "bench/p", 1.0), trial("p2__r", "bench/p", 1.0)] + run_dir = write_job(self.jobs, "run", "alpha", "harbor", trials) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(run_dir, None) + self.classify_ok() + group = load(self.round_file("groups.json"))["groups"][0] + self.assertEqual(group["trials"], 5) + repeat = [w for w in group["warnings"] if w["kind"] == "fails_in_every_repeat"] + self.assertEqual(len(repeat), 1) + self.assertEqual(repeat[0]["tasks"], 2) + + def test_a_count_that_disagrees_with_run_analysis_stops(self) -> None: + run_dir = write_job(self.jobs, "run", "alpha", "harbor", + [trial("p__r", "bench/p", 1.0), trial("f__r", "bench/f", 0.0)]) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(run_dir, None) + metrics = load(self.round_file("metrics.json")) + metrics["runs"][0]["trials_passed"] = 2 + write_json(self.round_file("metrics.json"), metrics) + r = run(["classify", "--loop", self.loop]) + self.assertNotEqual(r.code, 0) + self.assertIn("metrics.json", r.err) + self.assertFalse(os.path.exists(self.round_file("trials.csv"))) + + +class BothRunPaths(LoopCase): + def test_the_same_columns_for_harbor_and_standard(self) -> None: + headers, classes = [], [] + for runner in ("harbor", "standard"): + self.loop = os.path.join(self.tmp, f"loop-{runner}") + os.makedirs(self.loop) + timeout = trial("t__x", "bench/t", None) + timeout.exception = ("AgentTimeoutError", "late") + run_dir = write_job(self.jobs, runner, "alpha", runner, + [trial("p__x", "bench/p", 1.0), trial("f__x", "bench/f", 0.0), timeout]) + self.set_adapter(adapter("alpha", LIMITS, {})) + self.read_ok(run_dir, None) + self.classify_ok() + rows = self.rows() + headers.append(list(rows[0].keys())) + classes.append({r["trial"]: r["class"] for r in rows}) + self.assertEqual(headers[0], headers[1]) + self.assertEqual(classes[0], classes[1]) + for column in ("trial", "task", "class", "signature", "reward", "pair", "cost_usd", "model_calls", + "agent_s", "max_completion_tokens", "exception_type"): + self.assertIn(column, headers[0]) + + +STUB_CLASSES = """ +import json, os, sys +job_dir, out = sys.argv[1], sys.argv[2] +trials = {} +for name in sorted(os.listdir(job_dir)): + if os.path.isfile(os.path.join(job_dir, name, "result.json")): + trials[name] = {"classes": ["reply_overflow"] if name.startswith("f") else [], "flags": ["slow"]} +json.dump({"trials": trials}, open(out, "w")) +""" + + +class AgentClasses(LoopCase): + def job(self) -> str: + return write_job(self.jobs, "run", "alpha", "harbor", + [trial("p__r", "bench/p", 1.0), trial("f1__r", "bench/f1", 0.0), + trial("f2__r", "bench/f2", 0.0)]) + + def test_extra_columns_beside_the_seven(self) -> None: + stub = os.path.join(self.tmp, "stub_classes.py") + write(stub, STUB_CLASSES) + run_dir = self.job() + self.set_adapter(adapter("alpha", NO_LIMITS, {"classes": f"{sys.executable} {stub} {{job_dir}} {{out_json}}"})) + self.read_ok(run_dir, None) + self.classify_ok() + rows = self.by_trial() + self.assertEqual(rows["f1__r"]["class"], "verifier_failed") + self.assertEqual(rows["f1__r"]["agent_class:reply_overflow"], "1") + self.assertEqual(rows["p__r"]["agent_class:reply_overflow"], "0") + self.assertEqual(rows["p__r"]["agent_flag:slow"], "1") + agent = load(self.round_file("groups.json"))["agent_classes"] + self.assertTrue(agent["ran"]) + self.assertEqual(agent["classes"], {"reply_overflow": 2}) + self.assertEqual(agent["flags"], {"slow": 3}) + + def test_a_null_command_is_a_check_that_did_not_run(self) -> None: + run_dir = self.job() + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(run_dir, None) + self.classify_ok() + agent = load(self.round_file("groups.json"))["agent_classes"] + self.assertFalse(agent["ran"]) + self.assertIn("commands.classes is null", agent["reason"]) + self.assertFalse(any(k.startswith("agent_") for k in self.rows()[0])) + + def test_a_failing_command_stops(self) -> None: + run_dir = self.job() + self.set_adapter(adapter("alpha", NO_LIMITS, {"classes": "echo broken classes; exit 3 # {job_dir} {out_json}"})) + self.read_ok(run_dir, None) + r = run(["classify", "--loop", self.loop]) + self.assertNotEqual(r.code, 0) + self.assertIn("commands.classes", r.err) + self.assertIn("broken classes", r.err) + + +class FoundClasses(LoopCase): + def job(self) -> str: + own = trial("f1__r", "bench/f1", 0.0) + own.files = {"agent/output.json": '{"status": "step_cap"}'} + broken = trial("f3__r", "bench/f3", 0.0) + broken.files = {"agent/output.json": '{"status": '} + run_dir = write_job(self.jobs, "run", "alpha", "harbor", + [trial("p__r", "bench/p", 1.0), own, trial("f2__r", "bench/f2", 0.0), broken]) + for name in ("p__r", "f2__r"): + path = os.path.join(run_dir, name, "agent", "trajectory.json") + data = load(path) + data["steps"][-1]["extra"] = {"finish_reason": "length"} + write_json(path, data) + return run_dir + + def rules(self, rules: List[dict]) -> None: + write_json(os.path.join(self.loop, "found-classes.json"), + {"schema": "run-to-fix.found-classes/v1", "rules": rules}) + + def test_rules_count_every_trial(self) -> None: + run_dir = self.job() + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.rules([ + {"id": "reply_cut", "meaning": "a model reply ended at the length limit.", + "file": "agent/trajectory.json", "json_path": "steps.*.extra.finish_reason", "value": "length"}, + {"id": "own_step_cap", "meaning": "the agent wrote that it stopped at its step cap.", + "file": "agent/output.json", "regex": "\"status\":\\s*\"step_cap\""}, + {"id": "never", "meaning": "a text no trial has.", "file": "agent/trajectory.json", + "regex": "no-such-text-anywhere"}, + ]) + self.read_ok(run_dir, None) + self.classify_ok() + rows = self.by_trial() + self.assertEqual({n: r["found:reply_cut"] for n, r in rows.items()}, + {"p__r": "1", "f1__r": "0", "f2__r": "1", "f3__r": "0"}) + self.assertEqual({n: r["found:own_step_cap"] for n, r in rows.items()}, + {"p__r": "0", "f1__r": "1", "f2__r": "0", "f3__r": "0"}) + self.assertEqual({r["found:never"] for r in rows.values()}, {"0"}) + self.assertEqual(rows["f1__r"]["class"], "verifier_failed") + found = load(self.round_file("groups.json"))["found_classes"] + self.assertEqual(found["reply_cut"]["trials"], 2) + self.assertEqual(found["reply_cut"]["failing_trials"], 1) + self.assertEqual(found["own_step_cap"]["trials"], 1) + self.assertEqual(found["never"]["trials"], 0) + + def test_an_unreadable_file_is_counted_apart(self) -> None: + run_dir = self.job() + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.rules([{"id": "status", "meaning": "the agent wrote a step cap status.", "file": "agent/output.json", + "json_path": "status", "value": "step_cap"}]) + self.read_ok(run_dir, None) + self.classify_ok() + rows = self.by_trial() + self.assertEqual(rows["f3__r"]["found:status"], "unreadable") + self.assertEqual(rows["f1__r"]["found:status"], "1") + self.assertEqual(rows["p__r"]["found:status"], "0") + found = load(self.round_file("groups.json"))["found_classes"]["status"] + self.assertEqual((found["trials"], found["unreadable"]), (1, 1)) + + def test_a_rule_outside_the_trial_is_refused(self) -> None: + run_dir = self.job() + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.rules([{"id": "escape", "meaning": "reads a file of another trial.", "file": "../p__r/result.json", + "regex": "x"}]) + self.read_ok(run_dir, None) + r = run(["classify", "--loop", self.loop]) + self.assertNotEqual(r.code, 0) + self.assertIn("escape", r.err) + + def test_a_rule_with_two_patterns_is_refused(self) -> None: + run_dir = self.job() + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.rules([{"id": "both", "meaning": "two patterns.", "file": "result.json", "regex": "x", + "json_path": "a", "value": 1}]) + self.read_ok(run_dir, None) + r = run(["classify", "--loop", self.loop]) + self.assertNotEqual(r.code, 0) + self.assertIn("both", r.err) diff --git a/.claude/skills/run-to-fix/tests/test_read.py b/.claude/skills/run-to-fix/tests/test_read.py new file mode 100644 index 00000000..f27819af --- /dev/null +++ b/.claude/skills/run-to-fix/tests/test_read.py @@ -0,0 +1,168 @@ +"""`loop.py read`: stage 1 files, the adapter check and the refusals.""" +from __future__ import annotations + +import hashlib +import os +import shutil +import subprocess +import sys + +from jobs import (LIMITS, LOOP, NO_LIMITS, LoopCase, adapter, load, run, trial, write_job, write_json) + + +def healthy(root: str, name: str, agent: str) -> str: + return write_job(root, name, agent, "harbor", [ + trial(f"t{i}__a", f"bench/task-{i}", 1.0 if i < 4 else 0.0) for i in range(6)]) + + +class ReadWritesStageOne(LoopCase): + def test_writes_input_health_and_metrics(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + base_dir = healthy(self.jobs, "base", "alpha") + self.set_adapter(adapter("alpha", LIMITS, {})) + self.read_ok(run_dir, base_dir) + + data = load(self.round_file("input.json")) + with open(os.path.join(self.loop, "adapter.json"), "rb") as fh: + self.assertEqual(data["adapter_sha256"], hashlib.sha256(fh.read()).hexdigest()) + self.assertEqual(data["run"]["job_dir"], run_dir) + self.assertEqual(data["baseline"]["job_dir"], base_dir) + self.assertEqual(data["budget_usd"], 10.0) + self.assertEqual(data["pass_threshold"], 1.0) + + with open(self.round_file("health.txt"), encoding="utf-8") as fh: + health = fh.read() + self.assertIn(run_dir, health) + self.assertIn(base_dir, health) + + metrics = load(self.round_file("metrics.json")) + self.assertEqual([r["label"] for r in metrics["runs"]], ["baseline", "run"]) + self.assertEqual([r["trials_passed"] for r in metrics["runs"]], [4, 4]) + + def test_without_baseline(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(run_dir, None) + self.assertIsNone(load(self.round_file("input.json"))["baseline"]) + self.assertEqual([r["label"] for r in load(self.round_file("metrics.json"))["runs"]], ["run"]) + + def test_a_finished_round_starts_the_next(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(run_dir, None) + write_json(self.round_file("result.json"), {"verdict": "noise"}) + self.read_ok(run_dir, None) + self.assertTrue(os.path.isfile(os.path.join(self.loop, "round-02", "input.json"))) + + def test_an_open_round_past_stage_one_is_refused(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(run_dir, None) + self.classify_ok() + r = self.read(run_dir, None) + self.assertNotEqual(r.code, 0) + self.assertIn("round-01", r.err) + self.assertFalse(os.path.exists(os.path.join(self.loop, "round-02"))) + + +class ReadRefuses(LoopCase): + def test_a_systemic_failure_names_the_shared_error(self) -> None: + trials = [] + for i in range(8): + t = trial(f"t{i}__a", f"bench/task-{i}", None) + t.exception = ("BadRequestError", f"Error code: 402 - insufficient credits for request {1000 + i}") + trials.append(t) + run_dir = write_job(self.jobs, "stopped", "alpha", "harbor", trials) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + r = self.read(run_dir, None) + self.assertNotEqual(r.code, 0) + self.assertIn("not analysable", r.err) + self.assertIn("systemic failure", r.err) + self.assertIn("insufficient credits", r.err) + self.assertTrue(load(self.round_file("input.json"))["refused"]) + self.assertTrue(os.path.isfile(self.round_file("health.txt"))) + self.assertFalse(os.path.exists(self.round_file("metrics.json"))) + c = run(["classify", "--loop", self.loop]) + self.assertNotEqual(c.code, 0) + self.assertFalse(os.path.exists(self.round_file("trials.csv"))) + + def test_a_run_of_another_agent(self) -> None: + run_dir = healthy(self.jobs, "run", "beta") + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + r = self.read(run_dir, None) + self.assertNotEqual(r.code, 0) + self.assertIn("'alpha'", r.err) + self.assertIn("'beta'", r.err) + + def test_an_adapter_without_a_key(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + data = adapter("alpha", NO_LIMITS, {}) + del data["limits"] + self.set_adapter(data) + r = self.read(run_dir, None) + self.assertNotEqual(r.code, 0) + self.assertIn("limits", r.err) + + def test_an_adapter_of_another_schema(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + data = adapter("alpha", NO_LIMITS, {}) + data["schema"] = "run-to-fix.adapter/v0" + self.set_adapter(data) + r = self.read(run_dir, None) + self.assertNotEqual(r.code, 0) + self.assertIn("run-to-fix.adapter/v1", r.err) + + def test_a_command_without_a_placeholder_it_needs(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + self.set_adapter(adapter("alpha", NO_LIMITS, {"classes": "make-classes {job_dir}"})) + r = self.read(run_dir, None) + self.assertNotEqual(r.code, 0) + self.assertIn("classes", r.err) + self.assertIn("{out_json}", r.err) + + def test_a_placeholder_the_command_does_not_take(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + self.set_adapter(adapter("alpha", NO_LIMITS, {"classes": "make-classes {job_dir} {out_json} {worktree}"})) + r = self.read(run_dir, None) + self.assertNotEqual(r.code, 0) + self.assertIn("{worktree}", r.err) + + def test_a_test_command_without_its_pattern(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + self.set_adapter(adapter("alpha", NO_LIMITS, {"test": "pytest {worktree}"})) + r = self.read(run_dir, None) + self.assertNotEqual(r.code, 0) + self.assertIn("test_failed_pattern", r.err) + + def test_a_pattern_without_one_group(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + data = adapter("alpha", NO_LIMITS, {"test": "pytest {worktree}"}) + data["test_failed_pattern"] = "FAILED .*" + self.set_adapter(data) + r = self.read(run_dir, None) + self.assertNotEqual(r.code, 0) + self.assertIn("one group", r.err) + + def test_a_zip_is_refused(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + archive = shutil.make_archive(os.path.join(self.tmp, "run"), "zip", run_dir) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + r = self.read(archive, None) + self.assertNotEqual(r.code, 0) + self.assertIn("unpack", r.err) + + +class ReadWithoutRunAnalysis(LoopCase): + def test_names_the_missing_skill(self) -> None: + lone = os.path.join(self.tmp, "skills", "run-to-fix") + shutil.copytree(os.path.dirname(os.path.dirname(LOOP)), lone, + ignore=shutil.ignore_patterns("tests", "__pycache__")) + run_dir = healthy(self.jobs, "run", "alpha") + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + p = subprocess.run([sys.executable, os.path.join(lone, "scripts", "loop.py"), "read", "--loop", self.loop, + "--run", run_dir, "--budget-usd", "1", "--pass-threshold", "1"], + capture_output=True, text=True, stdin=subprocess.DEVNULL, timeout=60, + env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1")) + self.assertNotEqual(p.returncode, 0) + self.assertIn("run-analysis", p.stderr) + self.assertIn("runs.py", p.stderr) From c1caf999cc64e7de2c24df876191351a616eb94f Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:53:17 +0100 Subject: [PATCH 04/92] test(run-to-fix): look for agent class columns by their prefix --- .claude/skills/run-to-fix/tests/test_classify.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/tests/test_classify.py b/.claude/skills/run-to-fix/tests/test_classify.py index 1c88d55e..1517d12d 100644 --- a/.claude/skills/run-to-fix/tests/test_classify.py +++ b/.claude/skills/run-to-fix/tests/test_classify.py @@ -257,7 +257,7 @@ def test_a_null_command_is_a_check_that_did_not_run(self) -> None: agent = load(self.round_file("groups.json"))["agent_classes"] self.assertFalse(agent["ran"]) self.assertIn("commands.classes is null", agent["reason"]) - self.assertFalse(any(k.startswith("agent_") for k in self.rows()[0])) + self.assertFalse(any(k.startswith(("agent_class:", "agent_flag:")) for k in self.rows()[0])) def test_a_failing_command_stops(self) -> None: run_dir = self.job() From 76006761edffea2cf2b9484b045566b2ff4f3a2a Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 18:53:17 +0100 Subject: [PATCH 05/92] feat(run-to-fix): read a run and classify its trials read runs health and analyze of run-analysis, refuses a run that is not analysable or belongs to another agent, and records the round's input. classify gives each trial one of seven classes and a signature, groups the failing trials by signature with their warnings, and adds columns for found classes and for the agent's own classes. --- .claude/skills/run-to-fix/scripts/adapter.py | 147 +++++++++ .claude/skills/run-to-fix/scripts/found.py | 103 +++++++ .claude/skills/run-to-fix/scripts/loop.py | 308 +++++++++++++++++++ .claude/skills/run-to-fix/scripts/rounds.py | 154 ++++++++++ .claude/skills/run-to-fix/scripts/trials.py | 231 ++++++++++++++ 5 files changed, 943 insertions(+) create mode 100644 .claude/skills/run-to-fix/scripts/adapter.py create mode 100644 .claude/skills/run-to-fix/scripts/found.py create mode 100644 .claude/skills/run-to-fix/scripts/loop.py create mode 100644 .claude/skills/run-to-fix/scripts/rounds.py create mode 100644 .claude/skills/run-to-fix/scripts/trials.py diff --git a/.claude/skills/run-to-fix/scripts/adapter.py b/.claude/skills/run-to-fix/scripts/adapter.py new file mode 100644 index 00000000..b1c42afc --- /dev/null +++ b/.claude/skills/run-to-fix/scripts/adapter.py @@ -0,0 +1,147 @@ +"""The adapter file, `run-to-fix.adapter/v1`, read and checked against `references/adapter.md`.""" +from __future__ import annotations + +import hashlib +import json +import re +from dataclasses import dataclass +from typing import Dict, FrozenSet, List, Optional + +SCHEMA = "run-to-fix.adapter/v1" +COMMANDS = ("test", "test_one", "build", "run_tasks", "replay", "classes") +TAKES: Dict[str, FrozenSet[str]] = { + "test": frozenset({"worktree"}), + "test_one": frozenset({"worktree", "test"}), + "build": frozenset({"worktree"}), + "replay": frozenset({"worktree", "ref", "job_dir", "out_json"}), + "classes": frozenset({"job_dir", "out_json"}), + "run_tasks": frozenset({"units_file", "repeat", "jobs_dir", "budget_usd", "build", "ref"}), +} +NEEDS: Dict[str, FrozenSet[str]] = { + "test": frozenset({"worktree"}), + "test_one": frozenset({"worktree", "test"}), + "build": frozenset({"worktree"}), + "replay": frozenset({"worktree", "ref", "job_dir", "out_json"}), + "classes": frozenset({"job_dir", "out_json"}), + "run_tasks": frozenset({"units_file", "repeat", "jobs_dir"}), +} +PLACEHOLDER = re.compile(r"\{([a-z_]+)\}") + + +class AdapterError(Exception): + """The adapter does not satisfy the contract; the message names the key.""" + + +@dataclass(frozen=True) +class Limits: + step_limit: Optional[int] + output_token_limit: Optional[int] + time_limit_s: Optional[float] + + +@dataclass(frozen=True) +class Adapter: + agent: str + checkout: str + base_ref: str + work_branch: str + commands: Dict[str, Optional[str]] + test_failed_pattern: Optional["re.Pattern[str]"] + replay_success_labels: List[str] + limits: Limits + paid: List[str] + sha256: str + + +def _key(data: dict, key: str, where: str) -> object: + if not isinstance(data, dict) or key not in data: + raise AdapterError(f"adapter.json has no key {where}{key}; every key of references/adapter.md is present, " + "null when the agent has no value") + return data[key] + + +def _text(value: object, name: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise AdapterError(f"adapter.json: {name} must be a nonempty string, got {value!r}") + return value + + +def _limit(value: object, name: str) -> Optional[float]: + if value is None: + return None + if isinstance(value, bool) or not isinstance(value, (int, float)) or value <= 0: + raise AdapterError(f"adapter.json: limits.{name} must be a positive number or null, got {value!r}") + return value + + +def _command(name: str, value: object) -> Optional[str]: + if value is None: + return None + line = _text(value, f"commands.{name}") + used = set(PLACEHOLDER.findall(line)) + extra = sorted(used - TAKES[name]) + if extra: + raise AdapterError(f"adapter.json: commands.{name} uses " + ", ".join("{" + p + "}" for p in extra) + + ", which it does not take; it takes " + ", ".join("{" + p + "}" for p in sorted(TAKES[name]))) + missing = sorted(NEEDS[name] - used) + if missing: + raise AdapterError(f"adapter.json: commands.{name} needs " + ", ".join("{" + p + "}" for p in missing)) + if name == "run_tasks" and not used & {"build", "ref"}: + raise AdapterError("adapter.json: commands.run_tasks needs {build} or {ref}, so the run installs the code under check") + return line + + +def load_adapter(path: str) -> Adapter: + with open(path, "rb") as fh: + raw = fh.read() + try: + data = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError) as exc: + raise AdapterError(f"adapter.json is not JSON: {exc}") from exc + if _key(data, "schema", "") != SCHEMA: + raise AdapterError(f"adapter.json: schema must be {SCHEMA}, got {data.get('schema')!r}") + agent = _text(_key(data, "agent", ""), "agent") + source = _key(data, "source", "") + checkout = _text(_key(source, "checkout", "source."), "source.checkout") + base_ref = _text(_key(source, "base_ref", "source."), "source.base_ref") + work_branch = _text(_key(source, "work_branch", "source."), "source.work_branch") + raw_commands = _key(data, "commands", "") + commands = {name: _command(name, _key(raw_commands, name, "commands.")) for name in COMMANDS} + unknown = sorted(set(raw_commands) - set(COMMANDS)) + if unknown: + raise AdapterError(f"adapter.json: unknown commands {unknown}; the commands are {list(COMMANDS)}") + + pattern_text = _key(data, "test_failed_pattern", "") + pattern = None + if pattern_text is not None: + try: + pattern = re.compile(_text(pattern_text, "test_failed_pattern")) + except re.error as exc: + raise AdapterError(f"adapter.json: test_failed_pattern is not a regular expression: {exc}") from exc + if pattern.groups != 1: + raise AdapterError(f"adapter.json: test_failed_pattern must have exactly one group, it has {pattern.groups}") + if commands["test"] is not None and pattern is None: + raise AdapterError("adapter.json: commands.test is set, so test_failed_pattern is required") + + labels = _key(data, "replay_success_labels", "") + if not isinstance(labels, list) or not all(isinstance(x, str) and x for x in labels): + raise AdapterError(f"adapter.json: replay_success_labels must be a list of strings, got {labels!r}") + if commands["replay"] is not None and not labels: + raise AdapterError("adapter.json: commands.replay is set, so replay_success_labels needs at least one label") + + raw_limits = _key(data, "limits", "") + step = _limit(_key(raw_limits, "step_limit", "limits."), "step_limit") + output = _limit(_key(raw_limits, "output_token_limit", "limits."), "output_token_limit") + time_s = _limit(_key(raw_limits, "time_limit_s", "limits."), "time_limit_s") + paid = _key(data, "paid", "") + if not isinstance(paid, list) or not all(isinstance(x, str) for x in paid): + raise AdapterError(f"adapter.json: paid must be a list of command names, got {paid!r}") + strange = sorted(set(paid) - set(COMMANDS)) + if strange: + raise AdapterError(f"adapter.json: paid names {strange}, which are not commands") + return Adapter( + agent=agent, checkout=checkout, base_ref=base_ref, work_branch=work_branch, commands=commands, + test_failed_pattern=pattern, replay_success_labels=list(labels), + limits=Limits(step_limit=None if step is None else int(step), + output_token_limit=None if output is None else int(output), time_limit_s=time_s), + paid=list(paid), sha256=hashlib.sha256(raw).hexdigest()) diff --git a/.claude/skills/run-to-fix/scripts/found.py b/.claude/skills/run-to-fix/scripts/found.py new file mode 100644 index 00000000..3a22970c --- /dev/null +++ b/.claude/skills/run-to-fix/scripts/found.py @@ -0,0 +1,103 @@ +"""Found classes: rules in `/found-classes.json` that the assistant added while reading trials.""" +from __future__ import annotations + +import json +import os +import re +from dataclasses import dataclass +from typing import List, Optional + +SCHEMA = "run-to-fix.found-classes/v1" +ID = re.compile(r"^[a-z][a-z0-9_]{0,47}$") +UNREADABLE = "unreadable" + + +class RuleError(Exception): + """A rule of found-classes.json is malformed; the message names the rule.""" + + +@dataclass(frozen=True) +class Rule: + id: str + meaning: str + file: str + regex: Optional["re.Pattern[str]"] + json_path: Optional[List[str]] + value: object + + +def _rule(raw: object, index: int) -> Rule: + if not isinstance(raw, dict): + raise RuleError(f"found-classes.json: rule {index} is not an object") + rid = raw.get("id") + if not isinstance(rid, str) or not ID.match(rid): + raise RuleError(f"found-classes.json: rule {index} needs an id of lower-case letters, digits and _, got {rid!r}") + meaning = raw.get("meaning") + if not isinstance(meaning, str) or not meaning.strip(): + raise RuleError(f"found-classes.json: rule {rid} needs a one-sentence meaning") + rel = raw.get("file") + if not isinstance(rel, str) or not rel or os.path.isabs(rel) or ".." in re.split(r"[\\/]", rel): + raise RuleError(f"found-classes.json: rule {rid} must name a file inside the trial folder, got {rel!r}") + has_regex, has_path = "regex" in raw, "json_path" in raw + if has_regex == has_path: + raise RuleError(f"found-classes.json: rule {rid} needs either regex, or json_path with value; it has " + + ("both" if has_regex else "neither")) + if has_regex: + try: + pattern = re.compile(str(raw["regex"])) + except re.error as exc: + raise RuleError(f"found-classes.json: rule {rid} has a bad regex: {exc}") from exc + return Rule(id=rid, meaning=meaning, file=rel, regex=pattern, json_path=None, value=None) + path = raw["json_path"] + if not isinstance(path, str) or not path or "value" not in raw: + raise RuleError(f"found-classes.json: rule {rid} needs a json_path string and a value") + return Rule(id=rid, meaning=meaning, file=rel, regex=None, json_path=path.split("."), value=raw["value"]) + + +def load_rules(loop_dir: str) -> List[Rule]: + path = os.path.join(loop_dir, "found-classes.json") + if not os.path.isfile(path): + return [] + with open(path, encoding="utf-8") as fh: + try: + data = json.load(fh) + except json.JSONDecodeError as exc: + raise RuleError(f"found-classes.json is not JSON: {exc}") from exc + if not isinstance(data, dict) or data.get("schema") != SCHEMA or not isinstance(data.get("rules"), list): + raise RuleError(f"found-classes.json needs schema {SCHEMA} and a list of rules") + rules = [_rule(r, i) for i, r in enumerate(data["rules"])] + ids = [r.id for r in rules] + if len(ids) != len(set(ids)): + raise RuleError(f"found-classes.json: rule ids repeat: {sorted(i for i in set(ids) if ids.count(i) > 1)}") + return rules + + +def _walk(node: object, path: List[str]) -> List[object]: + """Values at `path`: a segment is a key, a list index, or * for every element.""" + if not path: + return [node] + head, rest = path[0], path[1:] + if head == "*": + children = list(node.values()) if isinstance(node, dict) else node if isinstance(node, list) else [] + return [v for child in children for v in _walk(child, rest)] + if isinstance(node, dict) and head in node: + return _walk(node[head], rest) + if isinstance(node, list) and head.isdigit() and int(head) < len(node): + return _walk(node[int(head)], rest) + return [] + + +def apply(rule: Rule, trial_dir: str) -> str: + """"1" when the rule matches the trial, "0" when it does not or the file is absent, "unreadable" otherwise.""" + path = os.path.join(trial_dir, rule.file) + if not os.path.isfile(path): + return "0" + with open(path, encoding="utf-8", errors="replace") as fh: + text = fh.read() + if rule.regex is not None: + return "1" if rule.regex.search(text) else "0" + try: + data = json.loads(text) + except json.JSONDecodeError: + return UNREADABLE + return "1" if any(v == rule.value for v in _walk(data, rule.json_path or [])) else "0" diff --git a/.claude/skills/run-to-fix/scripts/loop.py b/.claude/skills/run-to-fix/scripts/loop.py new file mode 100644 index 00000000..ca1ac59f --- /dev/null +++ b/.claude/skills/run-to-fix/scripts/loop.py @@ -0,0 +1,308 @@ +#!/usr/bin/env python3 +"""Counting and bookkeeping of the run-to-fix loop. Standard library only, Python 3.8+. + + read stage 1: health and metrics of the run and its baseline; refuses a run that is not analysable + classify stage 2: one class and signature per trial, groups and their warnings + +Every file is written under the loop folder; see references/files.md. +""" +from __future__ import annotations + +import argparse +import collections +import csv +import json +import os +import sys +from typing import Dict, List, Optional, Tuple + +from adapter import Adapter, AdapterError, load_adapter +from found import UNREADABLE, Rule, RuleError, apply, load_rules +from rounds import (LoopError, call_runs, now, open_round, read_input, read_json, round_for_read, run_command, + runs_py, sha256_of, write_json, write_paid) +from trials import CLASSES, TrialError, TrialRow, classify, read_trial, rules_off, signature, solved_tasks, trial_dirs + +#: verdicts of `runs.py health` that make a run not analysable (they are matched by their opening words) +REFUSING = ("no trials ran", "every trial raised an exception", "systemic failure", "incomplete:", + "the job never wrote a finish time") +FREQUENT_MIN_TRIALS = 3 +FREQUENT_MIN_SHARE = 0.05 +BASE_COLUMNS = ["trial", "task", "class", "signature", "reward", "pair", "cost_usd", "model_calls", "agent_s", + "max_completion_tokens", "exception_type"] + + +def adapter_of(loop_dir: str) -> Adapter: + return load_adapter(os.path.join(loop_dir, "adapter.json")) + + +def split_spec(spec: str) -> Tuple[str, Optional[str]]: + path, _, job = spec.partition("::") + path = os.path.abspath(os.path.expanduser(path)) + if os.path.isfile(path): + raise LoopError(f"{path} is a file; unpack the archive first and pass its folder") + if not os.path.isdir(path): + raise LoopError(f"{path} is not a folder") + return path, job or None + + +def health_job(spec: str, threshold: float) -> Tuple[dict, List[str]]: + """The health report of the one job `spec` names, and the verdicts that refuse it.""" + path, job = split_spec(spec) + reports = json.loads(call_runs(["health", "--json", "--pass-threshold", str(threshold), path])) + report = reports[0] + jobs = report["jobs"] + if job is not None: + jobs = [j for j in jobs if j["job"] == job] + if len(jobs) > 1: + raise LoopError(f"{path} holds several jobs: {[j['job'] for j in jobs]}. Name one with PATH::JOB_NAME.") + verdicts = list(report["verdicts"]) + (jobs[0]["verdicts"] if jobs else []) + if not jobs and not any(v.startswith("no trials ran") for v in verdicts): + verdicts.append(f"no trials ran: no job {job!r} under {path}") + refusing = [v for v in verdicts if v.startswith(REFUSING)] + return (jobs[0] if jobs else {}), refusing + + +def cmd_read(args: argparse.Namespace) -> None: + adapter = adapter_of(args.loop) + runs = runs_py() + specs = [("run", args.run)] + ([("baseline", args.baseline)] if args.baseline else []) + for _, spec in specs: + split_spec(spec) + round_dir = round_for_read(args.loop) + for stale in ("metrics.json", "input.json", "health.txt"): + if os.path.exists(os.path.join(round_dir, stale)): + os.remove(os.path.join(round_dir, stale)) + + texts, refused, agents = [], [], {} + for label, spec in specs: + path, _ = split_spec(spec) + texts.append(f"# {label}: {spec}\n" + call_runs(["health", "--pass-threshold", str(args.pass_threshold), path])) + job, refusing = health_job(spec, args.pass_threshold) + agents[label] = job.get("agents") or [] + for v in refusing: + refused.append(f"{label}: {v}") + for message, example in (job.get("error_examples") or {}).items(): + if message in v and example: + refused.append(f"{label}: e.g. {example}") + with open(os.path.join(round_dir, "health.txt"), "w", encoding="utf-8", newline="\n") as fh: + fh.write("\n".join(texts)) + + record = { + "schema": "run-to-fix.input/v1", "created": now(), "round": os.path.basename(round_dir), + "run": {"spec": args.run, "job_dir": None}, "baseline": {"spec": args.baseline, "job_dir": None} if args.baseline else None, + "adapter_sha256": adapter.sha256, "budget_usd": args.budget_usd, "pass_threshold": args.pass_threshold, + "run_analysis": {"runs_py": runs, "sha256": sha256_of(runs)}, "refused": bool(refused), + "reason": "; ".join(refused) or None, + } + if refused: + write_json(os.path.join(round_dir, "input.json"), record) + raise LoopError("the run is not analysable, so the loop stops here. health.txt has the report.\n " + + "\n ".join(refused)) + if agents["run"] != [adapter.agent]: + raise LoopError(f"adapter.json is for the agent {adapter.agent!r}, and the trials of the run name " + + ", ".join(repr(a) for a in agents["run"])) + + metrics = os.path.join(round_dir, "metrics.json") + order = ([("baseline", args.baseline)] if args.baseline else []) + [("run", args.run)] + paths, labels = [], [] + for label, spec in order: + path, job = split_spec(spec) + paths.append(path + (f"::{job}" if job else "")) + labels += ["--label", label] + print(call_runs(["analyze"] + paths + labels + ["--pass-threshold", str(args.pass_threshold), "--out", metrics])) + by_label = {r["label"]: r for r in read_json(metrics)["runs"]} + record["run"]["job_dir"] = by_label["run"]["dir"] + if args.baseline: + record["baseline"]["job_dir"] = by_label["baseline"]["dir"] + write_json(os.path.join(round_dir, "input.json"), record) + print(f"wrote {round_dir}: input.json, health.txt, metrics.json") + + +def pair_of(task: str, run_solved: Dict[str, bool], base_solved: Optional[Dict[str, bool]]) -> str: + if base_solved is None or task not in base_solved: + return "" + r, b = run_solved[task], base_solved[task] + return "both" if r and b else "run_only" if r else "baseline_only" if b else "neither" + + +def cross_check(metrics: dict, rows: List[TrialRow], base_rows: Optional[List[TrialRow]]) -> None: + """Stop when the counts of these trial readers and of run-analysis differ.""" + by_label = {r["label"]: r for r in metrics["runs"]} + problems = [] + for label, got in (("run", rows), ("baseline", base_rows)): + if got is None: + continue + want = by_label[label] + mine = (len(got), sum(r.passed for r in got)) + theirs = (want["trials"], want["trials_passed"]) + if mine != theirs: + problems.append(f"{label}: trials and passes {mine} here, {theirs} in metrics.json") + if base_rows is not None: + run_solved, _ = solved_tasks(rows) + base_solved, _ = solved_tasks(base_rows) + common = set(run_solved) & set(base_solved) + counts = collections.Counter(pair_of(t, run_solved, base_solved) for t in common) + pair = metrics["pairs"][0] + mine_p = (counts["both"], counts["baseline_only"], counts["run_only"], counts["neither"]) + theirs_p = (pair["both"], pair["first_only"], pair["second_only"], pair["neither"]) + if mine_p != theirs_p: + problems.append(f"pairs (both, baseline only, run only, neither) {mine_p} here, {theirs_p} in metrics.json") + if problems: + raise LoopError("the trials read here disagree with run-analysis, so nothing is written: " + "; ".join(problems)) + + +def agent_classes(adapter: Adapter, round_dir: str, job_dir: str, timeout_s: float) -> dict: + line = adapter.commands["classes"] + if line is None: + return {"ran": False, "reason": "commands.classes is null: the agent's own classes did not run", "trials": {}} + out_json = os.path.join(round_dir, "agent-classes.json") + values = {"job_dir": job_dir, "out_json": out_json} + if "classes" in adapter.paid: + path = write_paid(round_dir, "classes", line, values, adapter.checkout) + return {"ran": False, "reason": f"commands.classes costs money: written to {path}, not started", "trials": {}} + ran = run_command(line, values, adapter.checkout, os.path.join(round_dir, "logs", "classes.log"), timeout_s) + if ran.code != 0: + status = "did not finish in time" if ran.code is None else f"exited with code {ran.code}" + raise LoopError(f"commands.classes {status}; the log is {ran.log}:\n{ran.tail}") + data = read_json(out_json) + trials = data.get("trials") if isinstance(data, dict) else None + if not isinstance(trials, dict) or not all( + isinstance(v, dict) and isinstance(v.get("classes", []), list) and isinstance(v.get("flags", []), list) + for v in trials.values()): + raise LoopError(f"{out_json} must be {{\"trials\": {{\"\": {{\"classes\": [...], \"flags\": [...]}}}}}}") + return {"ran": True, "reason": None, "trials": trials} + + +def warnings_of(group_rows: List[TrialRow], failures: int, by_task: Dict[str, List[TrialRow]], + base_solved: Optional[Dict[str, bool]]) -> List[dict]: + out = [] + n = len(group_rows) + if n >= FREQUENT_MIN_TRIALS and n >= FREQUENT_MIN_SHARE * failures: + out.append({"kind": "frequent", "trials": n, + "text": f"{n} of {failures} failing trials share this signature. Read two or three of them."}) + tasks = sorted({r.task for r in group_rows}) + if base_solved is not None and all(t in base_solved and not base_solved[t] for t in tasks): + out.append({"kind": "fails_on_baseline", "tasks": len(tasks), + "text": f"All {len(tasks)} tasks of this group also fail on the baseline, so the benchmark is a " + "suspect. Check one task's verifier and expected answer before any claim that the " + "benchmark is defective."}) + repeated = [t for t in tasks if len(by_task[t]) > 1] + every = [t for t in repeated if not any(r.passed for r in by_task[t])] + if every: + out.append({"kind": "fails_in_every_repeat", "tasks": len(every), + "text": f"{len(every)} of the {len(tasks)} tasks of this group fail in every one of their trials."}) + return out + + +def cmd_classify(args: argparse.Namespace) -> None: + adapter = adapter_of(args.loop) + round_dir = open_round(args.loop) + record = read_input(round_dir) + rules: List[Rule] = load_rules(args.loop) + threshold = float(record["pass_threshold"]) + rows = [read_trial(d, threshold) for d in trial_dirs(record["run"]["job_dir"])] + base_rows = ([read_trial(d, threshold) for d in trial_dirs(record["baseline"]["job_dir"])] + if record["baseline"] else None) + cross_check(read_json(os.path.join(round_dir, "metrics.json")), rows, base_rows) + agent = agent_classes(adapter, round_dir, record["run"]["job_dir"], args.command_timeout_s) + + run_solved, by_task = solved_tasks(rows) + base_solved = solved_tasks(base_rows)[0] if base_rows is not None else None + own_classes = sorted({c for v in agent["trials"].values() for c in v.get("classes", [])}) + own_flags = sorted({f for v in agent["trials"].values() for f in v.get("flags", [])}) + columns = (BASE_COLUMNS + [f"found:{r.id}" for r in rules] + [f"agent_class:{c}" for c in own_classes] + + [f"agent_flag:{f}" for f in own_flags]) + + table, classes, signatures = [], {}, {} + found_counts = {r.id: collections.Counter() for r in rules} + for row in rows: + cls = classify(row, adapter.limits) + sig = signature(row, cls) + classes[row.trial], signatures[row.trial] = cls, sig + cells = {"trial": row.trial, "task": row.task, "class": cls, "signature": sig, + "reward": "" if row.reward is None else row.reward, "pair": pair_of(row.task, run_solved, base_solved), + "cost_usd": "" if row.cost_usd is None else row.cost_usd, + "model_calls": "" if row.model_calls is None else row.model_calls, + "agent_s": "" if row.agent_s is None else round(row.agent_s, 1), + "max_completion_tokens": "" if row.max_completion_tokens is None else row.max_completion_tokens, + "exception_type": row.exception_type or ""} + for rule in rules: + value = apply(rule, row.dir) + cells[f"found:{rule.id}"] = value + found_counts[rule.id][value] += 1 + if value == "1" and cls != "pass": + found_counts[rule.id]["failing"] += 1 + own = agent["trials"].get(row.trial) or {} + for c in own_classes: + cells[f"agent_class:{c}"] = int(c in own.get("classes", [])) + for f in own_flags: + cells[f"agent_flag:{f}"] = int(f in own.get("flags", [])) + table.append(cells) + + failing = [r for r in rows if classes[r.trial] != "pass"] + by_sig: Dict[str, List[TrialRow]] = collections.defaultdict(list) + for r in failing: + by_sig[signatures[r.trial]].append(r) + ordered = sorted(by_sig.items(), key=lambda kv: (-len(kv[1]), kv[0])) + groups = [{ + "id": f"g{i + 1:02d}", "class": classes[members[0].trial], "signature": sig, "trials": len(members), + "tasks": len({m.task for m in members}), "trial_names": sorted(m.trial for m in members), + "task_names": sorted({m.task for m in members}), + "warnings": warnings_of(members, len(failing), by_task, base_solved), + } for i, (sig, members) in enumerate(ordered)] + + with open(os.path.join(round_dir, "trials.csv"), "w", encoding="utf-8", newline="") as fh: + writer = csv.DictWriter(fh, fieldnames=columns, lineterminator="\n") + writer.writeheader() + writer.writerows(table) + count = collections.Counter(classes.values()) + write_json(os.path.join(round_dir, "groups.json"), { + "schema": "run-to-fix.groups/v1", + "counts": "trials of the run; each trial has one class, and a group is the failing trials of one signature", + "trials": len(rows), "failures": len(failing), + "classes": {c: count[c] for c in CLASSES}, "rules_off": rules_off(adapter.limits), + "groups": groups, + "found_classes": {r.id: {"meaning": r.meaning, "file": r.file, "trials": found_counts[r.id]["1"], + "failing_trials": found_counts[r.id]["failing"], + "unreadable": found_counts[r.id][UNREADABLE]} for r in rules}, + "agent_classes": {"ran": agent["ran"], "reason": agent["reason"], + "classes": {c: sum(c in v.get("classes", []) for v in agent["trials"].values()) for c in own_classes}, + "flags": {f: sum(f in v.get("flags", []) for v in agent["trials"].values()) for f in own_flags}}, + }) + print(f"wrote {round_dir}: trials.csv, groups.json") + print(f"{len(rows)} trials of the run, {len(failing)} failing, in {len(groups)} groups") + for c in CLASSES: + print(f" {c:16s} {count[c]:5d} trials") + for g in groups[:15]: + marks = ", ".join(w["kind"] for w in g["warnings"]) + print(f" {g['id']} {g['trials']:4d} trials {g['signature'][:110]}" + (f" [{marks}]" if marks else "")) + if not agent["ran"]: + print(f" check not run: {agent['reason']}") + + +def main(argv: List[str]) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + rd = sub.add_parser("read", help="stage 1: read the run and its baseline") + rd.add_argument("--loop", required=True, help="the loop folder a person named; it holds adapter.json") + rd.add_argument("--run", required=True, help="the job folder of the run, or a folder holding it as PATH::JOB") + rd.add_argument("--baseline", help="the job folder the run is compared with, as for --run") + rd.add_argument("--budget-usd", type=float, required=True, help="the budget the person named for this round") + rd.add_argument("--pass-threshold", type=float, default=1.0, help="reward at or above this is a pass (default 1)") + rd.set_defaults(func=cmd_read) + cl = sub.add_parser("classify", help="stage 2: classes, signatures and groups of the open round") + cl.add_argument("--loop", required=True) + cl.add_argument("--command-timeout-s", type=float, default=3600.0, + help="an adapter command still running after this many seconds is stopped (default 3600)") + cl.set_defaults(func=cmd_classify) + args = ap.parse_args(argv) + try: + args.func(args) + except (LoopError, AdapterError, RuleError, TrialError) as exc: + print(f"run-to-fix: {exc}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/.claude/skills/run-to-fix/scripts/rounds.py b/.claude/skills/run-to-fix/scripts/rounds.py new file mode 100644 index 00000000..70a47b88 --- /dev/null +++ b/.claude/skills/run-to-fix/scripts/rounds.py @@ -0,0 +1,154 @@ +"""The loop folder and its rounds, the bridge to run-analysis, and the runner of adapter commands.""" +from __future__ import annotations + +import datetime as dt +import hashlib +import json +import os +import re +import shlex +import signal +import subprocess +import sys +from dataclasses import dataclass +from typing import Dict, List, Optional + +SCRIPTS = os.path.dirname(os.path.abspath(__file__)) +RUNS_PY = os.path.join(os.path.dirname(os.path.dirname(SCRIPTS)), "run-analysis", "scripts", "runs.py") +ROUND = re.compile(r"^round-(\d{2,})$") +TAIL_LINES = 12 + + +class LoopError(Exception): + """The loop cannot go on; the message says why and what to do.""" + + +def now() -> str: + return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def sha256_of(path: str) -> str: + with open(path, "rb") as fh: + return hashlib.sha256(fh.read()).hexdigest() + + +def write_json(path: str, data: object) -> None: + os.makedirs(os.path.dirname(path), exist_ok=True) + tmp = path + ".tmp" + with open(tmp, "w", encoding="utf-8", newline="\n") as fh: + json.dump(data, fh, indent=1, sort_keys=False) + fh.write("\n") + os.replace(tmp, path) + + +def read_json(path: str) -> object: + with open(path, encoding="utf-8") as fh: + return json.load(fh) + + +def runs_py() -> str: + if not os.path.isfile(RUNS_PY): + raise LoopError(f"run-to-fix needs the run-analysis skill beside it, and {RUNS_PY} is missing. " + "Copy the whole run-analysis skill folder next to the run-to-fix folder.") + return RUNS_PY + + +def call_runs(args: List[str]) -> str: + """Run a command of run-analysis's runs.py and return its standard output.""" + p = subprocess.run([sys.executable, runs_py()] + args, capture_output=True, text=True, + stdin=subprocess.DEVNULL, env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1")) + if p.returncode != 0: + raise LoopError(f"runs.py {args[0]} failed with exit code {p.returncode}: {(p.stderr or p.stdout).strip()}") + return p.stdout + + +def rounds(loop_dir: str) -> List[str]: + found = [] + for name in os.listdir(loop_dir): + m = ROUND.match(name) + if m and os.path.isdir(os.path.join(loop_dir, name)): + found.append((int(m.group(1)), os.path.join(loop_dir, name))) + return [p for _, p in sorted(found)] + + +def round_for_read(loop_dir: str) -> str: + """The round `read` writes: the open round while it holds stage 1 only, else a new one after a finished round.""" + existing = rounds(loop_dir) + if existing: + last = existing[-1] + if not os.path.exists(os.path.join(last, "result.json")): + if os.path.exists(os.path.join(last, "trials.csv")): + raise LoopError(f"{os.path.basename(last)} is open past stage 1. Finish it with loop.py compare, " + "or start a new loop folder.") + return last + path = os.path.join(loop_dir, f"round-{len(existing) + 1:02d}") + os.makedirs(path) + return path + + +def open_round(loop_dir: str) -> str: + existing = rounds(loop_dir) + if not existing or os.path.exists(os.path.join(existing[-1], "result.json")): + raise LoopError(f"{loop_dir} has no open round; start one with loop.py read") + return existing[-1] + + +def read_input(round_dir: str) -> dict: + path = os.path.join(round_dir, "input.json") + if not os.path.isfile(path): + raise LoopError(f"{path} is missing; run loop.py read first") + data = read_json(path) + if not isinstance(data, dict): + raise LoopError(f"{path} is not an object") + if data.get("refused"): + raise LoopError(f"{os.path.basename(round_dir)}: read refused this run ({data.get('reason')}); " + "start a new loop folder for another run") + return data + + +def fill(line: str, values: Dict[str, str]) -> str: + """The command line with each placeholder replaced by its value quoted for the shell.""" + def one(m: "re.Match[str]") -> str: + key = m.group(1) + if key not in values: + raise LoopError(f"no value for {{{key}}} in {line!r}") + return shlex.quote(values[key]) + return re.sub(r"\{([a-z_]+)\}", one, line) + + +@dataclass(frozen=True) +class Ran: + code: Optional[int] + tail: str + log: str + + +def run_command(line: str, values: Dict[str, str], cwd: str, log_path: str, timeout_s: float) -> Ran: + """Run one adapter command with bash; its output goes to log_path. A command past timeout_s is stopped + with its whole process group, and `code` is None.""" + filled = fill(line, values) + os.makedirs(os.path.dirname(log_path), exist_ok=True) + with open(log_path, "w", encoding="utf-8") as log: + log.write(f"$ {filled}\n") + log.flush() + proc = subprocess.Popen(["bash", "-c", filled], cwd=cwd, stdout=log, stderr=subprocess.STDOUT, + stdin=subprocess.DEVNULL, start_new_session=True) + try: + code: Optional[int] = proc.wait(timeout=timeout_s) + except subprocess.TimeoutExpired: + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + code = None + with open(log_path, encoding="utf-8", errors="replace") as fh: + lines = fh.read().splitlines() + return Ran(code=code, tail="\n".join(lines[-TAIL_LINES:]), log=log_path) + + +def write_paid(round_dir: str, name: str, line: str, values: Dict[str, str], cwd: str) -> str: + """Write a paid command, filled, to round-NN/commands/.sh for a person to start.""" + path = os.path.join(round_dir, "commands", f"{name}.sh") + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "w", encoding="utf-8", newline="\n") as fh: + fh.write("#!/usr/bin/env bash\nset -euo pipefail\n") + fh.write(f"cd {shlex.quote(cwd)}\n{fill(line, values)}\n") + return path diff --git a/.claude/skills/run-to-fix/scripts/trials.py b/.claude/skills/run-to-fix/scripts/trials.py new file mode 100644 index 00000000..03883ff8 --- /dev/null +++ b/.claude/skills/run-to-fix/scripts/trials.py @@ -0,0 +1,231 @@ +"""One trial read from the files both run paths write, and its class and signature (`references/classes.md`).""" +from __future__ import annotations + +import datetime as dt +import json +import os +import re +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +from adapter import Limits + +CLASSES = ("pass", "time_limit", "exception", "no_score", "step_limit", "output_limit", "verifier_failed") +TIMEOUT_TYPE = "AgentTimeoutError" +NUMBER = re.compile(r"\b[0-9a-f]{8,}\b|\d+(?:\.\d+)?") +FAILING_LINE = re.compile(r"\b(FAIL(?:ED|URE)?|ERROR|Error|AssertionError|assert)\b") +SCORE_TEXT_KEYS = ("reason", "error") +DETAIL_CHARS = 200 + + +class TrialError(Exception): + """A trial file cannot be read; the message names it.""" + + +@dataclass(frozen=True) +class TrialRow: + trial: str + task: str + dir: str + reward: Optional[float] + passed: bool + exception_type: Optional[str] + exception_message: Optional[str] + agent_s: Optional[float] + model_calls: Optional[int] + max_completion_tokens: Optional[int] + cost_usd: Optional[float] + agent: Optional[str] + verifier_ran: bool + + +def read_json(path: str) -> object: + """A JSON file; a file that is not JSON raises with its path.""" + with open(path, encoding="utf-8") as fh: + try: + return json.load(fh) + except json.JSONDecodeError as exc: + raise TrialError(f"{path} is not JSON: {exc}") from exc + + +def _time(value: object) -> Optional[dt.datetime]: + if not value: + return None + t = dt.datetime.fromisoformat(str(value).replace("Z", "+00:00")) + if t.tzinfo is not None: + t = t.astimezone(dt.timezone.utc).replace(tzinfo=None) + return t + + +def _span(block: object) -> Optional[float]: + if not isinstance(block, dict): + return None + a, b = _time(block.get("started_at")), _time(block.get("finished_at")) + return (b - a).total_seconds() if a and b else None + + +def primary_reward(result: dict) -> Optional[float]: + """`rewards.reward`, else the first numeric reward: the rule of run-analysis.""" + rewards = (result.get("verifier_result") or {}).get("rewards") or {} + if not isinstance(rewards, dict): + return None + numeric = {k: v for k, v in rewards.items() if isinstance(v, (int, float)) and not isinstance(v, bool)} + if "reward" in numeric: + return float(numeric["reward"]) + return float(next(iter(numeric.values()))) if numeric else None + + +def _usage(trial_dir: str) -> Tuple[Optional[int], Optional[int]]: + """Model calls and the largest completion of one reply, from the ATIF trajectory. + + Model calls follow run-analysis: per-call usage records (`extra.raw_usage` or steps with metrics, whichever is + more), else the agent's steps.""" + path = os.path.join(trial_dir, "agent", "trajectory.json") + if not os.path.isfile(path): + return None, None + data = read_json(path) + if not isinstance(data, dict) or not isinstance(data.get("steps"), list): + return None, None + agent_steps, with_metrics, outputs = 0, 0, [] + for step in data["steps"]: + if not isinstance(step, dict) or step.get("source") != "agent": + continue + agent_steps += 1 + metrics = step.get("metrics") + if isinstance(metrics, dict) and metrics.get("prompt_tokens") is not None: + with_metrics += 1 + if isinstance(metrics, dict) and isinstance(metrics.get("completion_tokens"), int): + outputs.append(metrics["completion_tokens"]) + raw = (data.get("extra") or {}).get("raw_usage") if isinstance(data.get("extra"), dict) else None + raw = [u for u in raw if isinstance(u, dict)] if isinstance(raw, list) else [] + outputs += [u["output_tokens"] for u in raw if isinstance(u.get("output_tokens"), int)] + calls = max(len(raw), with_metrics) or agent_steps or None + return calls, (max(outputs) if outputs else None) + + +def read_trial(trial_dir: str, pass_threshold: float) -> TrialRow: + result = read_json(os.path.join(trial_dir, "result.json")) + if not isinstance(result, dict) or "task_name" not in result: + raise TrialError(f"{trial_dir}/result.json has no task_name") + reward = primary_reward(result) + exc = result.get("exception_info") or {} + calls, max_out = _usage(trial_dir) + info = result.get("agent_info") or {} + cost = (result.get("agent_result") or {}).get("cost_usd") + return TrialRow( + trial=str(result.get("trial_name") or os.path.basename(trial_dir)), task=str(result["task_name"]), + dir=trial_dir, reward=reward, passed=reward is not None and reward >= pass_threshold, + exception_type=str(exc.get("exception_type") or "Exception") if exc else None, + exception_message=str(exc.get("exception_message") or "") if exc else None, + agent_s=_span(result.get("agent_execution")), model_calls=calls, max_completion_tokens=max_out, + cost_usd=float(cost) if isinstance(cost, (int, float)) else None, agent=info.get("name"), + verifier_ran=reward is not None) + + +def classify(row: TrialRow, limits: Limits) -> str: + """The first class of `CLASSES` whose rule holds; a null limit switches its rule off.""" + if row.passed: + return "pass" + if row.exception_type == TIMEOUT_TYPE or ( + limits.time_limit_s is not None and row.agent_s is not None and row.agent_s >= limits.time_limit_s): + return "time_limit" + if row.exception_type is not None: + return "exception" + if row.reward is None: + return "no_score" + if limits.step_limit is not None and row.model_calls is not None and row.model_calls >= limits.step_limit: + return "step_limit" + if (limits.output_token_limit is not None and row.max_completion_tokens is not None + and row.max_completion_tokens >= limits.output_token_limit): + return "output_limit" + return "verifier_failed" + + +def rules_off(limits: Limits) -> List[str]: + off = [] + if limits.time_limit_s is None: + off.append("time_limit by the agent phase") + if limits.step_limit is None: + off.append("step_limit") + if limits.output_token_limit is None: + off.append("output_limit") + return off + + +def mask(text: str) -> str: + return re.sub(r"\s+", " ", NUMBER.sub("#", text)).strip()[:DETAIL_CHARS] + + +def _last_line(message: str) -> str: + lines = [ln.strip() for ln in message.splitlines() if ln.strip()] + return lines[-1] if lines else "" + + +def _exception_detail(row: TrialRow) -> str: + line = _last_line(row.exception_message or "") + prefix = f"{row.exception_type}: " + if line.startswith(prefix): + line = line[len(prefix):] + return mask(f"{row.exception_type}: {line}" if line else str(row.exception_type)) + + +def _ctrf_failure(path: str) -> Optional[str]: + data = read_json(path) + tests = ((data or {}).get("results") or {}).get("tests") if isinstance(data, dict) else None + for test in tests if isinstance(tests, list) else []: + if isinstance(test, dict) and test.get("status") in ("failed", "error"): + return str(test.get("name") or "") + return None + + +def _verifier_detail(trial_dir: str) -> str: + """The first failing line of the verifier's text: ctrf.json, then test-stdout.txt, then score.json.""" + verifier = os.path.join(trial_dir, "verifier") + ctrf = os.path.join(verifier, "ctrf.json") + if os.path.isfile(ctrf): + name = _ctrf_failure(ctrf) + if name: + return mask(name) + stdout = os.path.join(verifier, "test-stdout.txt") + if os.path.isfile(stdout): + with open(stdout, encoding="utf-8", errors="replace") as fh: + for line in fh: + if FAILING_LINE.search(line): + return mask(line) + score = os.path.join(verifier, "score.json") + if os.path.isfile(score): + data = read_json(score) + for key in SCORE_TEXT_KEYS: + value = data.get(key) if isinstance(data, dict) else None + if isinstance(value, str) and value.strip(): + return mask(_last_line(value)) + return "" + + +def signature(row: TrialRow, cls: str) -> str: + detail = "" + if row.exception_type is not None and cls in ("time_limit", "exception"): + detail = _exception_detail(row) + elif cls == "verifier_failed": + detail = _verifier_detail(row.dir) + return f"{cls}: {detail}" if detail else cls + + +def trial_dirs(job_dir: str) -> List[str]: + """The trial folders of a job: each child holding a result.json with a task_name.""" + out = [] + for name in sorted(os.listdir(job_dir)): + path = os.path.join(job_dir, name, "result.json") + if os.path.isfile(path): + data = read_json(path) + if isinstance(data, dict) and "task_name" in data: + out.append(os.path.join(job_dir, name)) + return out + + +def solved_tasks(rows: List[TrialRow]) -> Tuple[Dict[str, bool], Dict[str, List[TrialRow]]]: + """A task is solved when at least half of its trials pass: the rule of run-analysis.""" + by_task: Dict[str, List[TrialRow]] = {} + for r in rows: + by_task.setdefault(r.task, []).append(r) + return {t: sum(r.passed for r in v) / len(v) >= 0.5 for t, v in by_task.items()}, by_task From 09e4a2faa0a987c627502b5829af11be68fc2559 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:12:16 +0100 Subject: [PATCH 06/92] test(run-to-fix): cover compare, the exact tests and the stop rule --- .../skills/run-to-fix/tests/test_compare.py | 220 ++++++++++++++++++ 1 file changed, 220 insertions(+) create mode 100644 .claude/skills/run-to-fix/tests/test_compare.py diff --git a/.claude/skills/run-to-fix/tests/test_compare.py b/.claude/skills/run-to-fix/tests/test_compare.py new file mode 100644 index 00000000..e041e54a --- /dev/null +++ b/.claude/skills/run-to-fix/tests/test_compare.py @@ -0,0 +1,220 @@ +"""`loop.py compare`: pairs, the exact tests of D7, the targets, the stop rule and the closing of a round.""" +from __future__ import annotations + +import os +from typing import List, Tuple + +from jobs import (NO_LIMITS, LoopCase, Trial, adapter, load, run, trial, write_job, write_json) + + +def paired(both: int, won: int, lost: int, neither: int) -> Tuple[List[Trial], List[Trial]]: + """Trials of a run before and a new run, one per task, with the given counts of changed tasks.""" + before, new = [], [] + kinds = ["both"] * both + ["won"] * won + ["lost"] * lost + ["neither"] * neither + for i, kind in enumerate(kinds): + task = f"bench/task-{i:03d}" + before.append(trial(f"task-{i:03d}__b", task, 1.0 if kind in ("both", "lost") else 0.0)) + new.append(trial(f"task-{i:03d}__n", task, 1.0 if kind in ("both", "won") else 0.0)) + return before, new + + +def shared_failure(name: str, task: str) -> Trial: + t = trial(name, task, 0.0) + t.files = {"verifier/ctrf.json": '{"results": {"tests": [{"name": "test_x", "status": "failed"}]}}'} + return t + + +class Verdicts(LoopCase): + def compare(self, both: int, won: int, lost: int, neither: int) -> dict: + before, new = paired(both, won, lost, neither) + before_dir = write_job(self.jobs, "before", "alpha", "harbor", before) + new_dir = write_job(self.jobs, "new", "alpha", "harbor", new) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(before_dir, None) + r = run(["compare", "--loop", self.loop, "--new", new_dir, "--against", "run"]) + self.assertEqual(r.code, 0, r.out + r.err) + self.last_out = r.out + return load(self.round_file("result.json")) + + def test_ten_won_and_thirteen_lost_is_noise(self) -> None: + result = self.compare(271 // 10, 10, 13, 6) + self.assertEqual(result["pairs"], {"common_tasks": 27 + 10 + 13 + 6, "both": 27, "won": 10, "lost": 13, + "neither": 6}) + self.assertAlmostEqual(result["test"]["p"], 0.6776394844055176, places=9) + self.assertEqual(result["verdict"], "noise") + self.assertIn("noise", self.last_out) + + def test_twelve_won_and_one_lost_is_improved(self) -> None: + result = self.compare(20, 12, 1, 3) + self.assertAlmostEqual(result["test"]["p"], 0.00341796875, places=9) + self.assertEqual(result["verdict"], "improved") + + def test_one_won_and_twelve_lost_is_regressed(self) -> None: + self.assertEqual(self.compare(20, 1, 12, 3)["verdict"], "regressed") + + def test_no_changed_task_is_noise(self) -> None: + result = self.compare(10, 0, 0, 2) + self.assertEqual(result["test"]["p"], 1.0) + self.assertEqual(result["verdict"], "noise") + + def test_pass_rates_carry_their_standard_error(self) -> None: + result = self.compare(20, 12, 1, 3) + new = result["pass_rate"]["new"] + self.assertAlmostEqual(new["rate"], 32 / 36) + self.assertAlmostEqual(new["standard_error"], ((32 / 36) * (4 / 36) / 36) ** 0.5) + self.assertEqual((new["tasks"], new["trials"]), (36, 36)) + self.assertIn("standard error", self.last_out) + + def test_the_round_is_closed_and_recorded(self) -> None: + self.compare(20, 1, 1, 3) + ledger = load(os.path.join(self.loop, "ledger.json")) + self.assertEqual(ledger["rounds"][0]["round"], "round-01") + self.assertEqual(ledger["rounds"][0]["result"]["verdict"], "noise") + self.assertTrue(os.path.isfile(self.round_file("report.html"))) + self.assertTrue(os.path.isfile(self.round_file("compare-metrics.json"))) + + +class AgainstTheBaseline(LoopCase): + def test_the_round_run_against_its_baseline(self) -> None: + before, new = paired(5, 2, 1, 1) + before_dir = write_job(self.jobs, "before", "alpha", "harbor", before) + new_dir = write_job(self.jobs, "new", "alpha", "harbor", new) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(new_dir, before_dir) + r = run(["compare", "--loop", self.loop, "--new", new_dir, "--against", "baseline"]) + self.assertEqual(r.code, 0, r.out + r.err) + result = load(self.round_file("result.json")) + self.assertEqual((result["pairs"]["won"], result["pairs"]["lost"]), (2, 1)) + self.assertEqual(result["against"]["job_dir"], before_dir) + self.assertEqual(result["stop_rule"]["spent_usd"], 0.0) + + def test_against_a_baseline_the_round_does_not_have(self) -> None: + before, new = paired(5, 2, 1, 1) + before_dir = write_job(self.jobs, "before", "alpha", "harbor", before) + new_dir = write_job(self.jobs, "new", "alpha", "harbor", new) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(before_dir, None) + r = run(["compare", "--loop", self.loop, "--new", new_dir, "--against", "baseline"]) + self.assertNotEqual(r.code, 0) + self.assertIn("baseline", r.err) + + def test_a_new_run_that_is_not_analysable(self) -> None: + before, _ = paired(5, 0, 0, 1) + broken = [] + for i in range(6): + t = trial(f"task-{i:03d}__n", f"bench/task-{i:03d}", None) + t.exception = ("BadRequestError", "Error code: 402 - insufficient credits") + broken.append(t) + before_dir = write_job(self.jobs, "before", "alpha", "harbor", before) + new_dir = write_job(self.jobs, "new", "alpha", "harbor", broken) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(before_dir, None) + r = run(["compare", "--loop", self.loop, "--new", new_dir, "--against", "run"]) + self.assertNotEqual(r.code, 0) + self.assertIn("insufficient credits", r.err) + self.assertFalse(os.path.exists(self.round_file("result.json"))) + + +class Targets(LoopCase): + def run_targets(self, before_pass: List[bool], after_pass: List[bool]) -> dict: + tasks = ["bench/tg-0", "bench/tg-1", "bench/tg-2"] + before = [trial(f"tg-{i}__b", t, 1.0 if before_pass[i] else 0.0) for i, t in enumerate(tasks)] + before += [trial(f"p{i}__b", f"bench/p{i}", 1.0) for i in range(5)] + new = [] + for i, t in enumerate(tasks): + for k in range(3): + new.append(trial(f"tg-{i}-{k}__n", t, 1.0 if after_pass[3 * i + k] else 0.0)) + new += [trial(f"p{i}__n", f"bench/p{i}", 1.0) for i in range(5)] + before_dir = write_job(self.jobs, "before", "alpha", "harbor", before) + new_dir = write_job(self.jobs, "new", "alpha", "harbor", new) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(before_dir, None) + ledger_path = os.path.join(self.loop, "ledger.json") + ledger = load(ledger_path) + ledger["rounds"][-1]["selection"] = {"targets": tasks, "guards": [], "canaries": [], "repeat": 3} + write_json(ledger_path, ledger) + r = run(["compare", "--loop", self.loop, "--new", new_dir, "--against", "run"]) + self.assertEqual(r.code, 0, r.out + r.err) + return load(self.round_file("result.json"))["targets"] + + def test_targets_that_now_pass(self) -> None: + got = self.run_targets([False] * 3, [True] * 9) + self.assertEqual(got["tasks"], 3) + self.assertEqual(got["before"], {"trials": 3, "passed": 0}) + self.assertEqual(got["after"], {"trials": 9, "passed": 9}) + self.assertAlmostEqual(got["p"], 0.004545454545454545, places=9) + self.assertEqual(got["verdict"], "improved") + self.assertIn("3 tasks", got["text"]) + self.assertIn("12 trials", got["text"]) + + def test_targets_within_noise(self) -> None: + got = self.run_targets([True, False, False], [True, True, False, True, False, True, False, True, False]) + self.assertEqual(got["after"], {"trials": 9, "passed": 5}) + self.assertAlmostEqual(got["p"], 1.0, places=9) + self.assertEqual(got["verdict"], "noise") + + def test_without_a_selection(self) -> None: + before, new = paired(5, 1, 1, 1) + before_dir = write_job(self.jobs, "before", "alpha", "harbor", before) + new_dir = write_job(self.jobs, "new", "alpha", "harbor", new) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(before_dir, None) + r = run(["compare", "--loop", self.loop, "--new", new_dir, "--against", "run"]) + self.assertEqual(r.code, 0, r.out + r.err) + got = load(self.round_file("result.json"))["targets"] + self.assertIsNone(got["p"]) + self.assertIn("no targets", got["text"]) + + +class StopRule(LoopCase): + def compare_with_group(self, budget: str, with_group: bool) -> dict: + before, new = paired(20, 1, 1, 1) + if with_group: + new = new[:-3] + [shared_failure(f"g{i}__n", f"bench/g{i}") for i in range(3)] + before = before[:-3] + [trial(f"g{i}__b", f"bench/g{i}", 1.0) for i in range(3)] + before_dir = write_job(self.jobs, "before", "alpha", "harbor", before) + new_dir = write_job(self.jobs, "new", "alpha", "harbor", new) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + r = run(["read", "--loop", self.loop, "--run", before_dir, "--budget-usd", budget, "--pass-threshold", "1"]) + self.assertEqual(r.code, 0, r.out + r.err) + r = run(["compare", "--loop", self.loop, "--new", new_dir, "--against", "run"]) + self.assertEqual(r.code, 0, r.out + r.err) + self.last_out = r.out + return load(self.round_file("result.json"))["stop_rule"] + + def test_noise_and_no_group_of_three_stops(self) -> None: + rule = self.compare_with_group("10", False) + self.assertTrue(rule["stop"]) + self.assertEqual(rule["groups_of_3_or_more"], 0) + self.assertTrue(any("noise" in r for r in rule["reasons"])) + self.assertIn("stop rule: stop", self.last_out) + + def test_a_group_of_three_goes_on(self) -> None: + rule = self.compare_with_group("10", True) + self.assertEqual(rule["groups_of_3_or_more"], 1) + self.assertAlmostEqual(rule["spent_usd"], 0.23) + self.assertAlmostEqual(rule["left_usd"], 9.77) + self.assertAlmostEqual(rule["cheapest_option"]["usd"], 0.09) + self.assertFalse(rule["stop"], rule) + + def test_a_budget_under_the_cheapest_option_stops(self) -> None: + rule = self.compare_with_group("0.3", True) + self.assertAlmostEqual(rule["left_usd"], 0.07) + self.assertTrue(rule["stop"]) + self.assertTrue(any("budget" in r for r in rule["reasons"])) + + def test_the_next_read_after_a_stop_needs_the_person(self) -> None: + self.compare_with_group("10", False) + run_dir = os.path.join(self.jobs, "new") + r = run(["read", "--loop", self.loop, "--run", run_dir, "--budget-usd", "5", "--pass-threshold", "1"]) + self.assertNotEqual(r.code, 0) + self.assertIn("stop rule", r.err) + self.assertFalse(os.path.exists(os.path.join(self.loop, "round-02"))) + words = "Go on: the second fix of the step cap is ready." + r = run(["read", "--loop", self.loop, "--run", run_dir, "--budget-usd", "5", "--pass-threshold", "1", + "--continue-after-stop", words]) + self.assertEqual(r.code, 0, r.out + r.err) + answers = load(os.path.join(self.loop, "ledger.json"))["rounds"][1]["answers"] + self.assertEqual(answers[0]["about"], "continue after the stop rule of round-01") + self.assertEqual(answers[0]["text"], words) + self.assertRegex(answers[0]["date"], r"^\d{4}-\d{2}-\d{2}T") From eb4d087059a1b39f02c915bc92812b82c15a6f72 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:16:49 +0100 Subject: [PATCH 07/92] feat(run-to-fix): compare a new run and apply the stop rule compare gives the pass rates with their standard error, the won and lost tasks with an exact two-sided binomial test, the targets' trials with Fisher's exact test, and the stop rule, then closes the round. read asks for the person's words before a round that follows a stop. Each stage lives in its own module, and loop.py parses the command line. --- .../skills/run-to-fix/scripts/classifying.py | 201 ++++++++++++ .../skills/run-to-fix/scripts/comparing.py | 144 +++++++++ .claude/skills/run-to-fix/scripts/loop.py | 285 ++---------------- .claude/skills/run-to-fix/scripts/reading.py | 137 +++++++++ .claude/skills/run-to-fix/scripts/rounds.py | 40 ++- .claude/skills/run-to-fix/scripts/stats.py | 48 +++ 6 files changed, 586 insertions(+), 269 deletions(-) create mode 100644 .claude/skills/run-to-fix/scripts/classifying.py create mode 100644 .claude/skills/run-to-fix/scripts/comparing.py create mode 100644 .claude/skills/run-to-fix/scripts/reading.py create mode 100644 .claude/skills/run-to-fix/scripts/stats.py diff --git a/.claude/skills/run-to-fix/scripts/classifying.py b/.claude/skills/run-to-fix/scripts/classifying.py new file mode 100644 index 00000000..395b1261 --- /dev/null +++ b/.claude/skills/run-to-fix/scripts/classifying.py @@ -0,0 +1,201 @@ +"""Stage 2, `loop.py classify`: one class and signature per trial, the groups, their warnings, extra columns.""" +from __future__ import annotations + +import argparse +import collections +import csv +import os +from typing import Dict, List, Optional, Tuple + +from adapter import Adapter, Limits +from found import UNREADABLE, Rule, apply, load_rules +from reading import adapter_of +from rounds import LoopError, open_round, read_input, read_json, run_command, write_json, write_paid +from trials import CLASSES, TrialRow, classify, read_trial, rules_off, signature, solved_tasks, trial_dirs + +FREQUENT_MIN_TRIALS = 3 +FREQUENT_MIN_SHARE = 0.05 +BASE_COLUMNS = ["trial", "task", "class", "signature", "reward", "pair", "cost_usd", "model_calls", "agent_s", + "max_completion_tokens", "exception_type"] + + +def read_rows(job_dir: str, threshold: float) -> List[TrialRow]: + return [read_trial(d, threshold) for d in trial_dirs(job_dir)] + + +def classes_of(rows: List[TrialRow], limits: Limits) -> Tuple[Dict[str, str], Dict[str, str]]: + """The class and the signature of each trial, by trial name.""" + classes, signatures = {}, {} + for row in rows: + cls = classify(row, limits) + classes[row.trial], signatures[row.trial] = cls, signature(row, cls) + return classes, signatures + + +def pair_of(task: str, run_solved: Dict[str, bool], base_solved: Optional[Dict[str, bool]]) -> str: + if base_solved is None or task not in base_solved: + return "" + r, b = run_solved[task], base_solved[task] + return "both" if r and b else "run_only" if r else "baseline_only" if b else "neither" + + +def cross_check(metrics_path: str, runs: List[Tuple[str, List[TrialRow]]]) -> None: + """Stop when the counts of these trial readers and of run-analysis differ. `runs` is in the order of + `runs.py analyze`, so a pair is first against second.""" + metrics = read_json(metrics_path) + name = os.path.basename(metrics_path) + by_label = {r["label"]: r for r in metrics["runs"]} + problems = [] + for label, got in runs: + mine = (len(got), sum(r.passed for r in got)) + theirs = (by_label[label]["trials"], by_label[label]["trials_passed"]) + if mine != theirs: + problems.append(f"{label}: trials and passes {mine} here, {theirs} in {name}") + if len(runs) == 2: + (first, a), (second, b) = runs + sa, sb = solved_tasks(a)[0], solved_tasks(b)[0] + common = set(sa) & set(sb) + mine_p = (sum(sa[t] and sb[t] for t in common), sum(sa[t] and not sb[t] for t in common), + sum(sb[t] and not sa[t] for t in common), sum(not sa[t] and not sb[t] for t in common)) + pair = metrics["pairs"][0] + theirs_p = (pair["both"], pair["first_only"], pair["second_only"], pair["neither"]) + if mine_p != theirs_p: + problems.append(f"pairs (both, {first} only, {second} only, neither) {mine_p} here, {theirs_p} in {name}") + if problems: + raise LoopError("the trials read here disagree with run-analysis, so nothing is written: " + "; ".join(problems)) + + +def agent_classes(adapter: Adapter, round_dir: str, job_dir: str, timeout_s: float) -> dict: + line = adapter.commands["classes"] + if line is None: + return {"ran": False, "reason": "commands.classes is null: the agent's own classes did not run", "trials": {}} + out_json = os.path.join(round_dir, "agent-classes.json") + values = {"job_dir": job_dir, "out_json": out_json} + if "classes" in adapter.paid: + path = write_paid(round_dir, "classes", line, values, adapter.checkout) + return {"ran": False, "reason": f"commands.classes costs money: written to {path}, not started", "trials": {}} + ran = run_command(line, values, adapter.checkout, os.path.join(round_dir, "logs", "classes.log"), timeout_s) + if ran.code != 0: + status = "did not finish in time" if ran.code is None else f"exited with code {ran.code}" + raise LoopError(f"commands.classes {status}; the log is {ran.log}:\n{ran.tail}") + data = read_json(out_json) + trials = data.get("trials") if isinstance(data, dict) else None + if not isinstance(trials, dict) or not all( + isinstance(v, dict) and isinstance(v.get("classes", []), list) and isinstance(v.get("flags", []), list) + for v in trials.values()): + raise LoopError(f"{out_json} must be {{\"trials\": {{\"\": {{\"classes\": [...], \"flags\": [...]}}}}}}") + return {"ran": True, "reason": None, "trials": trials} + + +def warnings_of(group_rows: List[TrialRow], failures: int, by_task: Dict[str, List[TrialRow]], + base_solved: Optional[Dict[str, bool]]) -> List[dict]: + out = [] + n = len(group_rows) + if n >= FREQUENT_MIN_TRIALS and n >= FREQUENT_MIN_SHARE * failures: + out.append({"kind": "frequent", "trials": n, + "text": f"{n} of {failures} failing trials share this signature. Read two or three of them."}) + tasks = sorted({r.task for r in group_rows}) + if base_solved is not None and all(t in base_solved and not base_solved[t] for t in tasks): + out.append({"kind": "fails_on_baseline", "tasks": len(tasks), + "text": f"All {len(tasks)} tasks of this group also fail on the baseline, so the benchmark is a " + "suspect. Check one task's verifier and expected answer before any claim that the " + "benchmark is defective."}) + repeated = [t for t in tasks if len(by_task[t]) > 1] + every = [t for t in repeated if not any(r.passed for r in by_task[t])] + if every: + out.append({"kind": "fails_in_every_repeat", "tasks": len(every), + "text": f"{len(every)} of the {len(tasks)} tasks of this group fail in every one of their trials."}) + return out + + +def cmd_classify(args: argparse.Namespace) -> None: + adapter = adapter_of(args.loop) + round_dir = open_round(args.loop) + record = read_input(round_dir) + rules: List[Rule] = load_rules(args.loop) + threshold = float(record["pass_threshold"]) + rows = read_rows(record["run"]["job_dir"], threshold) + base_rows = read_rows(record["baseline"]["job_dir"], threshold) if record["baseline"] else None + cross_check(os.path.join(round_dir, "metrics.json"), + ([("baseline", base_rows)] if base_rows is not None else []) + [("run", rows)]) + agent = agent_classes(adapter, round_dir, record["run"]["job_dir"], args.command_timeout_s) + + run_solved, _ = solved_tasks(rows) + base_solved = solved_tasks(base_rows)[0] if base_rows is not None else None + own_classes = sorted({c for v in agent["trials"].values() for c in v.get("classes", [])}) + own_flags = sorted({f for v in agent["trials"].values() for f in v.get("flags", [])}) + columns = (BASE_COLUMNS + [f"found:{r.id}" for r in rules] + [f"agent_class:{c}" for c in own_classes] + + [f"agent_flag:{f}" for f in own_flags]) + + classes, signatures = classes_of(rows, adapter.limits) + table = [] + found_counts = {r.id: collections.Counter() for r in rules} + for row in rows: + cls, sig = classes[row.trial], signatures[row.trial] + cells = {"trial": row.trial, "task": row.task, "class": cls, "signature": sig, + "reward": "" if row.reward is None else row.reward, "pair": pair_of(row.task, run_solved, base_solved), + "cost_usd": "" if row.cost_usd is None else row.cost_usd, + "model_calls": "" if row.model_calls is None else row.model_calls, + "agent_s": "" if row.agent_s is None else round(row.agent_s, 1), + "max_completion_tokens": "" if row.max_completion_tokens is None else row.max_completion_tokens, + "exception_type": row.exception_type or ""} + for rule in rules: + value = apply(rule, row.dir) + cells[f"found:{rule.id}"] = value + found_counts[rule.id][value] += 1 + if value == "1" and cls != "pass": + found_counts[rule.id]["failing"] += 1 + own = agent["trials"].get(row.trial) or {} + for c in own_classes: + cells[f"agent_class:{c}"] = int(c in own.get("classes", [])) + for f in own_flags: + cells[f"agent_flag:{f}"] = int(f in own.get("flags", [])) + table.append(cells) + + failing = [r for r in rows if classes[r.trial] != "pass"] + groups = failure_groups(rows, classes, signatures, base_solved) + + with open(os.path.join(round_dir, "trials.csv"), "w", encoding="utf-8", newline="") as fh: + writer = csv.DictWriter(fh, fieldnames=columns, lineterminator="\n") + writer.writeheader() + writer.writerows(table) + count = collections.Counter(classes.values()) + write_json(os.path.join(round_dir, "groups.json"), { + "schema": "run-to-fix.groups/v1", + "counts": "trials of the run; each trial has one class, and a group is the failing trials of one signature", + "trials": len(rows), "failures": len(failing), + "classes": {c: count[c] for c in CLASSES}, "rules_off": rules_off(adapter.limits), + "groups": groups, + "found_classes": {r.id: {"meaning": r.meaning, "file": r.file, "trials": found_counts[r.id]["1"], + "failing_trials": found_counts[r.id]["failing"], + "unreadable": found_counts[r.id][UNREADABLE]} for r in rules}, + "agent_classes": {"ran": agent["ran"], "reason": agent["reason"], + "classes": {c: sum(c in v.get("classes", []) for v in agent["trials"].values()) for c in own_classes}, + "flags": {f: sum(f in v.get("flags", []) for v in agent["trials"].values()) for f in own_flags}}, + }) + print(f"wrote {round_dir}: trials.csv, groups.json") + print(f"{len(rows)} trials of the run, {len(failing)} failing, in {len(groups)} groups") + for c in CLASSES: + print(f" {c:16s} {count[c]:5d} trials") + for g in groups[:15]: + marks = ", ".join(w["kind"] for w in g["warnings"]) + print(f" {g['id']} {g['trials']:4d} trials {g['signature'][:110]}" + (f" [{marks}]" if marks else "")) + if not agent["ran"]: + print(f" check not run: {agent['reason']}") + + +def failure_groups(rows: List[TrialRow], classes: Dict[str, str], signatures: Dict[str, str], + base_solved: Optional[Dict[str, bool]]) -> List[dict]: + """The failing trials grouped by signature, largest first, each with its warnings.""" + _, by_task = solved_tasks(rows) + failing = [r for r in rows if classes[r.trial] != "pass"] + by_sig: Dict[str, List[TrialRow]] = collections.defaultdict(list) + for r in failing: + by_sig[signatures[r.trial]].append(r) + ordered = sorted(by_sig.items(), key=lambda kv: (-len(kv[1]), kv[0])) + return [{ + "id": f"g{i + 1:02d}", "class": classes[members[0].trial], "signature": sig, "trials": len(members), + "tasks": len({m.task for m in members}), "trial_names": sorted(m.trial for m in members), + "task_names": sorted({m.task for m in members}), + "warnings": warnings_of(members, len(failing), by_task, base_solved), + } for i, (sig, members) in enumerate(ordered)] diff --git a/.claude/skills/run-to-fix/scripts/comparing.py b/.claude/skills/run-to-fix/scripts/comparing.py new file mode 100644 index 00000000..59001ef7 --- /dev/null +++ b/.claude/skills/run-to-fix/scripts/comparing.py @@ -0,0 +1,144 @@ +"""Stage 5, `loop.py compare`: a new run against the round's run or baseline, the verdict of D7, the stop rule.""" +from __future__ import annotations + +import argparse +import os +from typing import Dict, List, Optional + +from classifying import classes_of, cross_check, failure_groups, read_rows +from reading import adapter_of, analyze, health +from rounds import (LoopError, call_runs, ledger_round, load_ledger, now, open_round, read_input, read_json, + save_ledger, write_json) +from stats import binomial_two_sided, fisher_two_sided, verdict +from trials import TrialRow + +TARGET_TRIALS = 3 +GROUP_MIN_TRIALS = 3 +STOP_RULE = ("No new round when the verdict is noise and no group of failing trials of the new run has 3 trials " + "or more, or when the budget left is under the price of the cheapest option: a trial run of the tasks " + "of the largest such group, 3 trials each, at the new run's mean cost per trial.") + + +def rate_of(run: dict) -> dict: + return {"rate": run["pass_rate"], "standard_error": run["standard_error"], "tasks": run["tasks"], + "trials": run["trials"], "trials_passed": run["trials_passed"]} + + +def passes(rows: List[TrialRow]) -> dict: + return {"trials": len(rows), "passed": sum(r.passed for r in rows)} + + +def targets_of(selection: Optional[dict], before: List[TrialRow], new: List[TrialRow]) -> dict: + """D7.3: passes of the target tasks' trials before and after, with Fisher's exact test.""" + targets = set((selection or {}).get("targets") or []) + if not targets: + return {"tasks": 0, "before": None, "after": None, "p": None, "verdict": None, + "text": "no targets were selected in this round, so no trials of targets are compared"} + b = passes([r for r in before if r.task in targets]) + a = passes([r for r in new if r.task in targets]) + p = fisher_two_sided(b["passed"], b["trials"] - b["passed"], a["passed"], a["trials"] - a["passed"]) + share = lambda x: x["passed"] / x["trials"] if x["trials"] else 0.0 + return {"tasks": len(targets), "before": b, "after": a, "p": p, + "verdict": verdict(p, share(a) > share(b), share(a) < share(b)), + "text": f"{len(targets)} tasks: {b['trials']} trials before and {a['trials']} after, " + f"{b['trials'] + a['trials']} trials in all; Fisher's exact two-sided test gives p = {p:.3g}"} + + +def predictions_of(predictions: List[dict], new: List[TrialRow]) -> List[dict]: + """Each prediction of stage 3 beside the passes of its tasks in the new run.""" + out = [] + for pred in predictions: + tasks = set(pred.get("tasks") or []) + out.append(dict(pred, result=dict(passes([r for r in new if r.task in tasks]), tasks=len(tasks)))) + return out + + +def stop_rule(result_verdict: str, groups: List[dict], budget: float, spent: Optional[float], + cost_per_trial: Optional[float]) -> dict: + big = [g for g in groups if g["trials"] >= GROUP_MIN_TRIALS] + reasons, not_checked = [], [] + if result_verdict == "noise" and not big: + reasons.append(f"the verdict is noise and no group of the new run has {GROUP_MIN_TRIALS} failing trials or more") + cheapest = None + if big: + price = big[0]["tasks"] * TARGET_TRIALS * cost_per_trial if cost_per_trial is not None else None + cheapest = {"what": f"a trial run of the {big[0]['tasks']} tasks of {big[0]['id']}, {TARGET_TRIALS} trials each", + "usd": price} + left = budget - spent if spent is not None else None + if cheapest is not None: + if cheapest["usd"] is None or left is None: + not_checked.append("the budget: the new run reports no cost") + elif left < cheapest["usd"]: + reasons.append(f"the budget left, ${left:.2f}, is under the price of the cheapest option, ${cheapest['usd']:.2f}") + return {"rule": STOP_RULE, "stop": bool(reasons), "reasons": reasons, "not_checked": not_checked, + "groups_of_3_or_more": len(big), "budget_usd": budget, "spent_usd": spent, "left_usd": left, + "cheapest_option": cheapest} + + +def cmd_compare(args: argparse.Namespace) -> None: + adapter = adapter_of(args.loop) + round_dir = open_round(args.loop) + record = read_input(round_dir) + threshold = float(record["pass_threshold"]) + against = record["run"] if args.against == "run" else record["baseline"] + if against is None: + raise LoopError(f"{os.path.basename(round_dir)} has no baseline; compare --against run, or read the run " + "again with --baseline") + checked = health("new", args.new, threshold) + if checked.refused: + raise LoopError("the new run is not analysable, so it is not compared:\n " + "\n ".join(checked.refused)) + if checked.agents != [adapter.agent]: + raise LoopError(f"adapter.json is for the agent {adapter.agent!r}, and the trials of the new run name " + + ", ".join(repr(a) for a in checked.agents)) + + metrics_path = os.path.join(round_dir, "compare-metrics.json") + print(analyze([("before", against["job_dir"]), ("new", args.new)], threshold, metrics_path)) + metrics = read_json(metrics_path) + runs: Dict[str, dict] = {r["label"]: r for r in metrics["runs"]} + new_dir = runs["new"]["dir"] + before_rows, new_rows = read_rows(against["job_dir"], threshold), read_rows(new_dir, threshold) + cross_check(metrics_path, [("before", before_rows), ("new", new_rows)]) + + pair = metrics["pairs"][0] + won, lost = pair["second_only"], pair["first_only"] + p = binomial_two_sided(won, lost) + result_verdict = verdict(p, won > lost, lost > won) + ledger = load_ledger(args.loop) + entry = ledger_round(ledger, os.path.basename(round_dir)) + classes, signatures = classes_of(new_rows, adapter.limits) + groups = failure_groups(new_rows, classes, signatures, None) + spent = 0.0 if new_dir == record["run"]["job_dir"] else runs["new"]["cost_total"] + rule = stop_rule(result_verdict, groups, float(record["budget_usd"]), spent, runs["new"]["cost_mean"]) + result = { + "schema": "run-to-fix.result/v1", "created": now(), "round": os.path.basename(round_dir), + "counts": "tasks for pairs; a task is solved when at least half of its trials pass. Trials for targets.", + "against": {"label": args.against, "job_dir": against["job_dir"]}, "new": {"spec": args.new, "job_dir": new_dir}, + "pass_rate": {"before": rate_of(runs["before"]), "new": rate_of(runs["new"])}, + "pairs": {"common_tasks": metrics["overlap"]["common_tasks"], "both": pair["both"], "won": won, "lost": lost, + "neither": pair["neither"]}, + "test": {"name": "exact two-sided binomial test on won and lost tasks", "p": p}, "verdict": result_verdict, + "targets": targets_of(entry.get("selection"), before_rows, new_rows), + "predictions": predictions_of(entry.get("predictions") or [], new_rows), + "groups": [{k: g[k] for k in ("id", "class", "signature", "trials", "tasks")} for g in groups], + "stop_rule": rule, + } + report = os.path.join(round_dir, "report.html") + call_runs(["render", metrics_path, "--out", report]) + entry["result"] = {"verdict": result_verdict, "p": p, "won": won, "lost": lost, "stop": rule["stop"], + "reasons": rule["reasons"], "file": os.path.join(os.path.basename(round_dir), "result.json")} + save_ledger(args.loop, ledger) + write_json(os.path.join(round_dir, "result.json"), result) + + for label in ("before", "new"): + r = result["pass_rate"][label] + print(f"pass rate {label}: {r['rate']:.3f}, standard error {r['standard_error']:.3f}, " + f"over {r['tasks']} tasks ({r['trials_passed']} of {r['trials']} trials passed)") + pr = result["pairs"] + print(f"tasks on both: {pr['common_tasks']}; solved by both {pr['both']}, won {won}, lost {lost}, " + f"neither {pr['neither']}") + print(f"verdict: {result_verdict} (exact two-sided binomial test on won and lost, p = {p:.3g})") + print(f"targets: {result['targets']['text']}") + print("stop rule: " + ("stop: " + "; ".join(rule["reasons"]) if rule["stop"] else "go on") + + ("" if not rule["not_checked"] else "; not checked: " + "; ".join(rule["not_checked"]))) + print(f"wrote {round_dir}: compare-metrics.json, result.json, report.html (its prose slots are open: fill them " + "with runs.py render --slots, then runs.py check)") diff --git a/.claude/skills/run-to-fix/scripts/loop.py b/.claude/skills/run-to-fix/scripts/loop.py index ca1ac59f..a5c6a461 100644 --- a/.claude/skills/run-to-fix/scripts/loop.py +++ b/.claude/skills/run-to-fix/scripts/loop.py @@ -1,283 +1,25 @@ #!/usr/bin/env python3 -"""Counting and bookkeeping of the run-to-fix loop. Standard library only, Python 3.8+. +"""Counting and bookkeeping of the run-to-fix loop. Standard library only. read stage 1: health and metrics of the run and its baseline; refuses a run that is not analysable classify stage 2: one class and signature per trial, groups and their warnings + compare stage 5: a new run against the round's run or baseline, the verdict, the stop rule Every file is written under the loop folder; see references/files.md. """ from __future__ import annotations import argparse -import collections -import csv -import json -import os import sys -from typing import Dict, List, Optional, Tuple +from typing import List -from adapter import Adapter, AdapterError, load_adapter -from found import UNREADABLE, Rule, RuleError, apply, load_rules -from rounds import (LoopError, call_runs, now, open_round, read_input, read_json, round_for_read, run_command, - runs_py, sha256_of, write_json, write_paid) -from trials import CLASSES, TrialError, TrialRow, classify, read_trial, rules_off, signature, solved_tasks, trial_dirs - -#: verdicts of `runs.py health` that make a run not analysable (they are matched by their opening words) -REFUSING = ("no trials ran", "every trial raised an exception", "systemic failure", "incomplete:", - "the job never wrote a finish time") -FREQUENT_MIN_TRIALS = 3 -FREQUENT_MIN_SHARE = 0.05 -BASE_COLUMNS = ["trial", "task", "class", "signature", "reward", "pair", "cost_usd", "model_calls", "agent_s", - "max_completion_tokens", "exception_type"] - - -def adapter_of(loop_dir: str) -> Adapter: - return load_adapter(os.path.join(loop_dir, "adapter.json")) - - -def split_spec(spec: str) -> Tuple[str, Optional[str]]: - path, _, job = spec.partition("::") - path = os.path.abspath(os.path.expanduser(path)) - if os.path.isfile(path): - raise LoopError(f"{path} is a file; unpack the archive first and pass its folder") - if not os.path.isdir(path): - raise LoopError(f"{path} is not a folder") - return path, job or None - - -def health_job(spec: str, threshold: float) -> Tuple[dict, List[str]]: - """The health report of the one job `spec` names, and the verdicts that refuse it.""" - path, job = split_spec(spec) - reports = json.loads(call_runs(["health", "--json", "--pass-threshold", str(threshold), path])) - report = reports[0] - jobs = report["jobs"] - if job is not None: - jobs = [j for j in jobs if j["job"] == job] - if len(jobs) > 1: - raise LoopError(f"{path} holds several jobs: {[j['job'] for j in jobs]}. Name one with PATH::JOB_NAME.") - verdicts = list(report["verdicts"]) + (jobs[0]["verdicts"] if jobs else []) - if not jobs and not any(v.startswith("no trials ran") for v in verdicts): - verdicts.append(f"no trials ran: no job {job!r} under {path}") - refusing = [v for v in verdicts if v.startswith(REFUSING)] - return (jobs[0] if jobs else {}), refusing - - -def cmd_read(args: argparse.Namespace) -> None: - adapter = adapter_of(args.loop) - runs = runs_py() - specs = [("run", args.run)] + ([("baseline", args.baseline)] if args.baseline else []) - for _, spec in specs: - split_spec(spec) - round_dir = round_for_read(args.loop) - for stale in ("metrics.json", "input.json", "health.txt"): - if os.path.exists(os.path.join(round_dir, stale)): - os.remove(os.path.join(round_dir, stale)) - - texts, refused, agents = [], [], {} - for label, spec in specs: - path, _ = split_spec(spec) - texts.append(f"# {label}: {spec}\n" + call_runs(["health", "--pass-threshold", str(args.pass_threshold), path])) - job, refusing = health_job(spec, args.pass_threshold) - agents[label] = job.get("agents") or [] - for v in refusing: - refused.append(f"{label}: {v}") - for message, example in (job.get("error_examples") or {}).items(): - if message in v and example: - refused.append(f"{label}: e.g. {example}") - with open(os.path.join(round_dir, "health.txt"), "w", encoding="utf-8", newline="\n") as fh: - fh.write("\n".join(texts)) - - record = { - "schema": "run-to-fix.input/v1", "created": now(), "round": os.path.basename(round_dir), - "run": {"spec": args.run, "job_dir": None}, "baseline": {"spec": args.baseline, "job_dir": None} if args.baseline else None, - "adapter_sha256": adapter.sha256, "budget_usd": args.budget_usd, "pass_threshold": args.pass_threshold, - "run_analysis": {"runs_py": runs, "sha256": sha256_of(runs)}, "refused": bool(refused), - "reason": "; ".join(refused) or None, - } - if refused: - write_json(os.path.join(round_dir, "input.json"), record) - raise LoopError("the run is not analysable, so the loop stops here. health.txt has the report.\n " - + "\n ".join(refused)) - if agents["run"] != [adapter.agent]: - raise LoopError(f"adapter.json is for the agent {adapter.agent!r}, and the trials of the run name " - + ", ".join(repr(a) for a in agents["run"])) - - metrics = os.path.join(round_dir, "metrics.json") - order = ([("baseline", args.baseline)] if args.baseline else []) + [("run", args.run)] - paths, labels = [], [] - for label, spec in order: - path, job = split_spec(spec) - paths.append(path + (f"::{job}" if job else "")) - labels += ["--label", label] - print(call_runs(["analyze"] + paths + labels + ["--pass-threshold", str(args.pass_threshold), "--out", metrics])) - by_label = {r["label"]: r for r in read_json(metrics)["runs"]} - record["run"]["job_dir"] = by_label["run"]["dir"] - if args.baseline: - record["baseline"]["job_dir"] = by_label["baseline"]["dir"] - write_json(os.path.join(round_dir, "input.json"), record) - print(f"wrote {round_dir}: input.json, health.txt, metrics.json") - - -def pair_of(task: str, run_solved: Dict[str, bool], base_solved: Optional[Dict[str, bool]]) -> str: - if base_solved is None or task not in base_solved: - return "" - r, b = run_solved[task], base_solved[task] - return "both" if r and b else "run_only" if r else "baseline_only" if b else "neither" - - -def cross_check(metrics: dict, rows: List[TrialRow], base_rows: Optional[List[TrialRow]]) -> None: - """Stop when the counts of these trial readers and of run-analysis differ.""" - by_label = {r["label"]: r for r in metrics["runs"]} - problems = [] - for label, got in (("run", rows), ("baseline", base_rows)): - if got is None: - continue - want = by_label[label] - mine = (len(got), sum(r.passed for r in got)) - theirs = (want["trials"], want["trials_passed"]) - if mine != theirs: - problems.append(f"{label}: trials and passes {mine} here, {theirs} in metrics.json") - if base_rows is not None: - run_solved, _ = solved_tasks(rows) - base_solved, _ = solved_tasks(base_rows) - common = set(run_solved) & set(base_solved) - counts = collections.Counter(pair_of(t, run_solved, base_solved) for t in common) - pair = metrics["pairs"][0] - mine_p = (counts["both"], counts["baseline_only"], counts["run_only"], counts["neither"]) - theirs_p = (pair["both"], pair["first_only"], pair["second_only"], pair["neither"]) - if mine_p != theirs_p: - problems.append(f"pairs (both, baseline only, run only, neither) {mine_p} here, {theirs_p} in metrics.json") - if problems: - raise LoopError("the trials read here disagree with run-analysis, so nothing is written: " + "; ".join(problems)) - - -def agent_classes(adapter: Adapter, round_dir: str, job_dir: str, timeout_s: float) -> dict: - line = adapter.commands["classes"] - if line is None: - return {"ran": False, "reason": "commands.classes is null: the agent's own classes did not run", "trials": {}} - out_json = os.path.join(round_dir, "agent-classes.json") - values = {"job_dir": job_dir, "out_json": out_json} - if "classes" in adapter.paid: - path = write_paid(round_dir, "classes", line, values, adapter.checkout) - return {"ran": False, "reason": f"commands.classes costs money: written to {path}, not started", "trials": {}} - ran = run_command(line, values, adapter.checkout, os.path.join(round_dir, "logs", "classes.log"), timeout_s) - if ran.code != 0: - status = "did not finish in time" if ran.code is None else f"exited with code {ran.code}" - raise LoopError(f"commands.classes {status}; the log is {ran.log}:\n{ran.tail}") - data = read_json(out_json) - trials = data.get("trials") if isinstance(data, dict) else None - if not isinstance(trials, dict) or not all( - isinstance(v, dict) and isinstance(v.get("classes", []), list) and isinstance(v.get("flags", []), list) - for v in trials.values()): - raise LoopError(f"{out_json} must be {{\"trials\": {{\"\": {{\"classes\": [...], \"flags\": [...]}}}}}}") - return {"ran": True, "reason": None, "trials": trials} - - -def warnings_of(group_rows: List[TrialRow], failures: int, by_task: Dict[str, List[TrialRow]], - base_solved: Optional[Dict[str, bool]]) -> List[dict]: - out = [] - n = len(group_rows) - if n >= FREQUENT_MIN_TRIALS and n >= FREQUENT_MIN_SHARE * failures: - out.append({"kind": "frequent", "trials": n, - "text": f"{n} of {failures} failing trials share this signature. Read two or three of them."}) - tasks = sorted({r.task for r in group_rows}) - if base_solved is not None and all(t in base_solved and not base_solved[t] for t in tasks): - out.append({"kind": "fails_on_baseline", "tasks": len(tasks), - "text": f"All {len(tasks)} tasks of this group also fail on the baseline, so the benchmark is a " - "suspect. Check one task's verifier and expected answer before any claim that the " - "benchmark is defective."}) - repeated = [t for t in tasks if len(by_task[t]) > 1] - every = [t for t in repeated if not any(r.passed for r in by_task[t])] - if every: - out.append({"kind": "fails_in_every_repeat", "tasks": len(every), - "text": f"{len(every)} of the {len(tasks)} tasks of this group fail in every one of their trials."}) - return out - - -def cmd_classify(args: argparse.Namespace) -> None: - adapter = adapter_of(args.loop) - round_dir = open_round(args.loop) - record = read_input(round_dir) - rules: List[Rule] = load_rules(args.loop) - threshold = float(record["pass_threshold"]) - rows = [read_trial(d, threshold) for d in trial_dirs(record["run"]["job_dir"])] - base_rows = ([read_trial(d, threshold) for d in trial_dirs(record["baseline"]["job_dir"])] - if record["baseline"] else None) - cross_check(read_json(os.path.join(round_dir, "metrics.json")), rows, base_rows) - agent = agent_classes(adapter, round_dir, record["run"]["job_dir"], args.command_timeout_s) - - run_solved, by_task = solved_tasks(rows) - base_solved = solved_tasks(base_rows)[0] if base_rows is not None else None - own_classes = sorted({c for v in agent["trials"].values() for c in v.get("classes", [])}) - own_flags = sorted({f for v in agent["trials"].values() for f in v.get("flags", [])}) - columns = (BASE_COLUMNS + [f"found:{r.id}" for r in rules] + [f"agent_class:{c}" for c in own_classes] - + [f"agent_flag:{f}" for f in own_flags]) - - table, classes, signatures = [], {}, {} - found_counts = {r.id: collections.Counter() for r in rules} - for row in rows: - cls = classify(row, adapter.limits) - sig = signature(row, cls) - classes[row.trial], signatures[row.trial] = cls, sig - cells = {"trial": row.trial, "task": row.task, "class": cls, "signature": sig, - "reward": "" if row.reward is None else row.reward, "pair": pair_of(row.task, run_solved, base_solved), - "cost_usd": "" if row.cost_usd is None else row.cost_usd, - "model_calls": "" if row.model_calls is None else row.model_calls, - "agent_s": "" if row.agent_s is None else round(row.agent_s, 1), - "max_completion_tokens": "" if row.max_completion_tokens is None else row.max_completion_tokens, - "exception_type": row.exception_type or ""} - for rule in rules: - value = apply(rule, row.dir) - cells[f"found:{rule.id}"] = value - found_counts[rule.id][value] += 1 - if value == "1" and cls != "pass": - found_counts[rule.id]["failing"] += 1 - own = agent["trials"].get(row.trial) or {} - for c in own_classes: - cells[f"agent_class:{c}"] = int(c in own.get("classes", [])) - for f in own_flags: - cells[f"agent_flag:{f}"] = int(f in own.get("flags", [])) - table.append(cells) - - failing = [r for r in rows if classes[r.trial] != "pass"] - by_sig: Dict[str, List[TrialRow]] = collections.defaultdict(list) - for r in failing: - by_sig[signatures[r.trial]].append(r) - ordered = sorted(by_sig.items(), key=lambda kv: (-len(kv[1]), kv[0])) - groups = [{ - "id": f"g{i + 1:02d}", "class": classes[members[0].trial], "signature": sig, "trials": len(members), - "tasks": len({m.task for m in members}), "trial_names": sorted(m.trial for m in members), - "task_names": sorted({m.task for m in members}), - "warnings": warnings_of(members, len(failing), by_task, base_solved), - } for i, (sig, members) in enumerate(ordered)] - - with open(os.path.join(round_dir, "trials.csv"), "w", encoding="utf-8", newline="") as fh: - writer = csv.DictWriter(fh, fieldnames=columns, lineterminator="\n") - writer.writeheader() - writer.writerows(table) - count = collections.Counter(classes.values()) - write_json(os.path.join(round_dir, "groups.json"), { - "schema": "run-to-fix.groups/v1", - "counts": "trials of the run; each trial has one class, and a group is the failing trials of one signature", - "trials": len(rows), "failures": len(failing), - "classes": {c: count[c] for c in CLASSES}, "rules_off": rules_off(adapter.limits), - "groups": groups, - "found_classes": {r.id: {"meaning": r.meaning, "file": r.file, "trials": found_counts[r.id]["1"], - "failing_trials": found_counts[r.id]["failing"], - "unreadable": found_counts[r.id][UNREADABLE]} for r in rules}, - "agent_classes": {"ran": agent["ran"], "reason": agent["reason"], - "classes": {c: sum(c in v.get("classes", []) for v in agent["trials"].values()) for c in own_classes}, - "flags": {f: sum(f in v.get("flags", []) for v in agent["trials"].values()) for f in own_flags}}, - }) - print(f"wrote {round_dir}: trials.csv, groups.json") - print(f"{len(rows)} trials of the run, {len(failing)} failing, in {len(groups)} groups") - for c in CLASSES: - print(f" {c:16s} {count[c]:5d} trials") - for g in groups[:15]: - marks = ", ".join(w["kind"] for w in g["warnings"]) - print(f" {g['id']} {g['trials']:4d} trials {g['signature'][:110]}" + (f" [{marks}]" if marks else "")) - if not agent["ran"]: - print(f" check not run: {agent['reason']}") +from adapter import AdapterError +from classifying import cmd_classify +from comparing import cmd_compare +from found import RuleError +from reading import cmd_read +from rounds import LoopError +from trials import TrialError def main(argv: List[str]) -> int: @@ -289,12 +31,19 @@ def main(argv: List[str]) -> int: rd.add_argument("--baseline", help="the job folder the run is compared with, as for --run") rd.add_argument("--budget-usd", type=float, required=True, help="the budget the person named for this round") rd.add_argument("--pass-threshold", type=float, default=1.0, help="reward at or above this is a pass (default 1)") + rd.add_argument("--continue-after-stop", help="the person's words to start a round after a stop rule said stop") rd.set_defaults(func=cmd_read) cl = sub.add_parser("classify", help="stage 2: classes, signatures and groups of the open round") cl.add_argument("--loop", required=True) cl.add_argument("--command-timeout-s", type=float, default=3600.0, help="an adapter command still running after this many seconds is stopped (default 3600)") cl.set_defaults(func=cmd_classify) + cp = sub.add_parser("compare", help="stage 5: compare a new run and close the round") + cp.add_argument("--loop", required=True) + cp.add_argument("--new", required=True, help="the job folder of the new run, as for read --run") + cp.add_argument("--against", required=True, choices=("run", "baseline"), + help="compare with the round's run (the code before the fix) or with its baseline") + cp.set_defaults(func=cmd_compare) args = ap.parse_args(argv) try: args.func(args) diff --git a/.claude/skills/run-to-fix/scripts/reading.py b/.claude/skills/run-to-fix/scripts/reading.py new file mode 100644 index 00000000..413f72f6 --- /dev/null +++ b/.claude/skills/run-to-fix/scripts/reading.py @@ -0,0 +1,137 @@ +"""Stage 1, `loop.py read`: health and metrics of the run and its baseline, through run-analysis.""" +from __future__ import annotations + +import argparse +import json +import os +from dataclasses import dataclass +from typing import List, Optional, Tuple + +from adapter import Adapter, load_adapter +from rounds import (LoopError, call_runs, last_result, ledger_round, load_ledger, now, round_for_read, runs_py, + save_ledger, sha256_of, write_json) + +#: verdicts of `runs.py health` that make a run not analysable, matched by their opening words +REFUSING = ("no trials ran", "every trial raised an exception", "systemic failure", "incomplete:", + "the job never wrote a finish time") + + +def adapter_of(loop_dir: str) -> Adapter: + return load_adapter(os.path.join(loop_dir, "adapter.json")) + + +def split_spec(spec: str) -> Tuple[str, Optional[str]]: + """PATH or PATH::JOB, with PATH made absolute; an archive is refused.""" + path, _, job = spec.partition("::") + path = os.path.abspath(os.path.expanduser(path)) + if os.path.isfile(path): + raise LoopError(f"{path} is a file; unpack the archive first and pass its folder") + if not os.path.isdir(path): + raise LoopError(f"{path} is not a folder") + return path, job or None + + +def full_spec(spec: str) -> str: + path, job = split_spec(spec) + return path + (f"::{job}" if job else "") + + +@dataclass(frozen=True) +class Health: + text: str + agents: List[str] + refused: List[str] + + +def health(label: str, spec: str, threshold: float) -> Health: + """The health of the one job `spec` names: the printed report, its agents, and the verdicts that refuse it.""" + path, job_name = split_spec(spec) + text = f"# {label}: {spec}\n" + call_runs(["health", "--pass-threshold", str(threshold), path]) + report = json.loads(call_runs(["health", "--json", "--pass-threshold", str(threshold), path]))[0] + jobs = [j for j in report["jobs"] if job_name is None or j["job"] == job_name] + if len(jobs) > 1: + raise LoopError(f"{path} holds several jobs: {[j['job'] for j in jobs]}. Name one with PATH::JOB_NAME.") + job = jobs[0] if jobs else {} + verdicts = list(report["verdicts"]) + list(job.get("verdicts") or []) + if not jobs and not any(v.startswith("no trials ran") for v in verdicts): + verdicts.append(f"no trials ran: no job {job_name!r} under {path}") + refused = [] + for v in verdicts: + if v.startswith(REFUSING): + refused.append(f"{label}: {v}") + refused += [f"{label}: e.g. {example}" for message, example in (job.get("error_examples") or {}).items() + if example and message in v] + return Health(text=text, agents=list(job.get("agents") or []), refused=refused) + + +def analyze(runs: List[Tuple[str, str]], threshold: float, out: str) -> str: + """`runs.py analyze` of (label, spec) pairs into `out`; the first is the base of every gap.""" + paths = [full_spec(spec) for _, spec in runs] + labels = [x for label, _ in runs for x in ("--label", label)] + return call_runs(["analyze"] + paths + labels + ["--pass-threshold", str(threshold), "--out", out]) + + +def gate(loop_dir: str, words: Optional[str]) -> Optional[dict]: + """The stop rule of the last finished round. A stop needs the person's words to go on; they are returned + as an answer for the ledger.""" + last = last_result(loop_dir) + if last is None: + return None + name, result = last + rule = result.get("stop_rule") or {} + if not rule.get("stop"): + return None + if not words: + raise LoopError(f"the stop rule of {name} says stop: " + "; ".join(rule.get("reasons") or []) + + ". To start another round anyway, pass the person's words with --continue-after-stop.") + return {"about": f"continue after the stop rule of {name}", "date": now(), "text": words} + + +def cmd_read(args: argparse.Namespace) -> None: + adapter = adapter_of(args.loop) + runs = runs_py() + specs = [("run", args.run)] + ([("baseline", args.baseline)] if args.baseline else []) + for _, spec in specs: + split_spec(spec) + answer = gate(args.loop, args.continue_after_stop) + round_dir = round_for_read(args.loop) + name = os.path.basename(round_dir) + for stale in ("metrics.json", "input.json", "health.txt"): + if os.path.exists(os.path.join(round_dir, stale)): + os.remove(os.path.join(round_dir, stale)) + ledger = load_ledger(args.loop) + entry = ledger_round(ledger, name) + if answer is not None: + entry["answers"].append(answer) + save_ledger(args.loop, ledger) + + checks = {label: health(label, spec, args.pass_threshold) for label, spec in specs} + with open(os.path.join(round_dir, "health.txt"), "w", encoding="utf-8", newline="\n") as fh: + fh.write("\n".join(h.text for h in checks.values())) + refused = [line for h in checks.values() for line in h.refused] + record = { + "schema": "run-to-fix.input/v1", "created": now(), "round": name, + "run": {"spec": args.run, "job_dir": None}, + "baseline": {"spec": args.baseline, "job_dir": None} if args.baseline else None, + "adapter_sha256": adapter.sha256, "budget_usd": args.budget_usd, "pass_threshold": args.pass_threshold, + "run_analysis": {"runs_py": runs, "sha256": sha256_of(runs)}, "refused": bool(refused), + "reason": "; ".join(refused) or None, + } + if refused: + write_json(os.path.join(round_dir, "input.json"), record) + raise LoopError("the run is not analysable, so the loop stops here. health.txt has the report.\n " + + "\n ".join(refused)) + if checks["run"].agents != [adapter.agent]: + raise LoopError(f"adapter.json is for the agent {adapter.agent!r}, and the trials of the run name " + + ", ".join(repr(a) for a in checks["run"].agents)) + + metrics = os.path.join(round_dir, "metrics.json") + order = ([("baseline", args.baseline)] if args.baseline else []) + [("run", args.run)] + print(analyze(order, args.pass_threshold, metrics)) + with open(metrics, encoding="utf-8") as fh: + by_label = {r["label"]: r for r in json.load(fh)["runs"]} + record["run"]["job_dir"] = by_label["run"]["dir"] + if args.baseline: + record["baseline"]["job_dir"] = by_label["baseline"]["dir"] + write_json(os.path.join(round_dir, "input.json"), record) + print(f"wrote {round_dir}: input.json, health.txt, metrics.json") diff --git a/.claude/skills/run-to-fix/scripts/rounds.py b/.claude/skills/run-to-fix/scripts/rounds.py index 70a47b88..be16d497 100644 --- a/.claude/skills/run-to-fix/scripts/rounds.py +++ b/.claude/skills/run-to-fix/scripts/rounds.py @@ -11,7 +11,7 @@ import subprocess import sys from dataclasses import dataclass -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple SCRIPTS = os.path.dirname(os.path.abspath(__file__)) RUNS_PY = os.path.join(os.path.dirname(os.path.dirname(SCRIPTS)), "run-analysis", "scripts", "runs.py") @@ -152,3 +152,41 @@ def write_paid(round_dir: str, name: str, line: str, values: Dict[str, str], cwd fh.write("#!/usr/bin/env bash\nset -euo pipefail\n") fh.write(f"cd {shlex.quote(cwd)}\n{fill(line, values)}\n") return path + + +LEDGER_SCHEMA = "run-to-fix.ledger/v1" + + +def load_ledger(loop_dir: str) -> dict: + path = os.path.join(loop_dir, "ledger.json") + if not os.path.isfile(path): + return {"schema": LEDGER_SCHEMA, "rounds": []} + data = read_json(path) + if not isinstance(data, dict) or data.get("schema") != LEDGER_SCHEMA or not isinstance(data.get("rounds"), list): + raise LoopError(f"{path} is not a ledger of schema {LEDGER_SCHEMA}") + return data + + +def save_ledger(loop_dir: str, ledger: dict) -> None: + write_json(os.path.join(loop_dir, "ledger.json"), ledger) + + +def ledger_round(ledger: dict, name: str) -> dict: + """The ledger entry of round `name`, added when it is not there yet.""" + for entry in ledger["rounds"]: + if entry["round"] == name: + return entry + entry = {"round": name, "started": now(), "answers": [], "selection": None, "predictions": [], "result": None} + ledger["rounds"].append(entry) + return entry + + +def last_result(loop_dir: str) -> Optional[Tuple[str, dict]]: + """The name and result.json of the last finished round, when the last round is finished.""" + existing = rounds(loop_dir) + if not existing or not os.path.isfile(os.path.join(existing[-1], "result.json")): + return None + data = read_json(os.path.join(existing[-1], "result.json")) + if not isinstance(data, dict): + raise LoopError(f"{existing[-1]}/result.json is not an object") + return os.path.basename(existing[-1]), data diff --git a/.claude/skills/run-to-fix/scripts/stats.py b/.claude/skills/run-to-fix/scripts/stats.py new file mode 100644 index 00000000..d81c8f2e --- /dev/null +++ b/.claude/skills/run-to-fix/scripts/stats.py @@ -0,0 +1,48 @@ +"""Exact tests of D7: a binomial test on won and lost tasks, Fisher's test on passes of trials.""" +from __future__ import annotations + +import math +from typing import Optional + +ALPHA = 0.05 +#: probabilities within this relative distance of the observed table's count as equally extreme +TIE = 1e-7 + + +def standard_error(rate: float, n: int) -> Optional[float]: + return math.sqrt(rate * (1 - rate) / n) if n else None + + +def binomial_two_sided(won: int, lost: int) -> float: + """p of an exact two-sided binomial test of won against lost, with even odds.""" + n = won + lost + if n == 0: + return 1.0 + k = min(won, lost) + return min(1.0, 2 * sum(math.comb(n, i) for i in range(k + 1)) / 2 ** n) + + +def fisher_two_sided(before_pass: int, before_fail: int, after_pass: int, after_fail: int) -> float: + """p of Fisher's exact two-sided test on a 2 x 2 table: the tables no more likely than the one observed.""" + n = before_pass + before_fail + after_pass + after_fail + rows = before_pass + before_fail + passes = before_pass + after_pass + if n == 0 or rows in (0, n) or passes in (0, n): + return 1.0 + total = math.comb(n, passes) + + def prob(x: int) -> float: + return math.comb(rows, x) * math.comb(n - rows, passes - x) / total + + observed = prob(before_pass) + lo, hi = max(0, passes - (n - rows)), min(rows, passes) + return min(1.0, sum(prob(x) for x in range(lo, hi + 1) if prob(x) <= observed * (1 + TIE))) + + +def verdict(p: float, better: bool, worse: bool) -> str: + """"improved" or "regressed" when p is under ALPHA, in the direction the counts point, else "noise".""" + if p < ALPHA and better: + return "improved" + if p < ALPHA and worse: + return "regressed" + return "noise" From a142991dfede586631827a969e1e8de5e064a295 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:19:16 +0100 Subject: [PATCH 08/92] fix(run-to-fix): print a negative budget left with its sign first --- .claude/skills/run-to-fix/scripts/comparing.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/scripts/comparing.py b/.claude/skills/run-to-fix/scripts/comparing.py index 59001ef7..28062f97 100644 --- a/.claude/skills/run-to-fix/scripts/comparing.py +++ b/.claude/skills/run-to-fix/scripts/comparing.py @@ -19,6 +19,10 @@ "of the largest such group, 3 trials each, at the new run's mean cost per trial.") +def usd(value: float) -> str: + return f"-${-value:.2f}" if value < 0 else f"${value:.2f}" + + def rate_of(run: dict) -> dict: return {"rate": run["pass_rate"], "standard_error": run["standard_error"], "tasks": run["tasks"], "trials": run["trials"], "trials_passed": run["trials_passed"]} @@ -69,7 +73,8 @@ def stop_rule(result_verdict: str, groups: List[dict], budget: float, spent: Opt if cheapest["usd"] is None or left is None: not_checked.append("the budget: the new run reports no cost") elif left < cheapest["usd"]: - reasons.append(f"the budget left, ${left:.2f}, is under the price of the cheapest option, ${cheapest['usd']:.2f}") + reasons.append(f"the budget left, {usd(left)}, is under the price of the cheapest option, " + f"{usd(cheapest['usd'])}") return {"rule": STOP_RULE, "stop": bool(reasons), "reasons": reasons, "not_checked": not_checked, "groups_of_3_or_more": len(big), "budget_usd": budget, "spent_usd": spent, "left_usd": left, "cheapest_option": cheapest} From c913145d5c1ae6a5b2f58287784f5f949f0074ee Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:28:16 +0100 Subject: [PATCH 09/92] test(run-to-fix): cover candidates, checks, both stops and the trial run --- .claude/skills/run-to-fix/tests/agentrepo.py | 151 +++++++++++++ .claude/skills/run-to-fix/tests/test_check.py | 163 ++++++++++++++ .../skills/run-to-fix/tests/test_decide.py | 211 ++++++++++++++++++ 3 files changed, 525 insertions(+) create mode 100644 .claude/skills/run-to-fix/tests/agentrepo.py create mode 100644 .claude/skills/run-to-fix/tests/test_check.py create mode 100644 .claude/skills/run-to-fix/tests/test_decide.py diff --git a/.claude/skills/run-to-fix/tests/agentrepo.py b/.claude/skills/run-to-fix/tests/agentrepo.py new file mode 100644 index 00000000..ce0b7f0a --- /dev/null +++ b/.claude/skills/run-to-fix/tests/agentrepo.py @@ -0,0 +1,151 @@ +"""A made-up agent in a git repository, its stub commands, and a recorded run of it.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from dataclasses import dataclass +from typing import Dict, List, Optional + +from jobs import LoopCase, Trial, adapter, run, trial, write, write_job + +SUITE = """import agent +print("FAILED test_old - a failure the suite has on every ref") +if agent.BREAKS_OTHER: + print("FAILED test_other - broken by the fix") +print("ok test_fix_area") +""" +TEST_FIX = """import sys +sys.path.insert(0, ".") +import agent +sys.exit(0 if agent.FIXED else 1) +""" +REPLAY = """import json, os, sys +worktree, job_dir, ref, out, marker = sys.argv[1:6] +labels = json.load(open(os.path.join(worktree, "labels.json"))) +names = sorted(n for n in os.listdir(job_dir) if os.path.isfile(os.path.join(job_dir, n, "result.json"))) +json.dump({"ref": ref, "trials": {n: labels[n] for n in names}, "kept_recorded": labels.get("_kept", [])}, open(out, "w")) +open(marker, "a").write(ref + "\\n") +""" +BUILD = """import hashlib, os, shutil, sys +worktree, out_dir = sys.argv[1:3] +data = open(os.path.join(worktree, "agent.py"), "rb").read() +os.makedirs(out_dir, exist_ok=True) +path = os.path.join(out_dir, "agent-" + hashlib.sha256(data).hexdigest()[:8] + ".txt") +open(path, "wb").write(data) +print("built the agent") +print(path) +""" + +BASE_LABELS = {"p1__r": {"label": "done"}, "p2__r": {"label": "done"}, "p3__r": {"label": "not done"}, + "f1__r": {"label": "done"}, "f2__r": {"label": "done"}, "f3__r": {"label": "not done"}} +GOOD_LABELS = {"p1__r": {"label": "done", "touched": True}, "p2__r": {"label": "done", "touched": False}, + "p3__r": {"label": "done", "touched": False}, "f1__r": {"label": "not done", "touched": True}, + "f2__r": {"label": "not done", "touched": True}, "f3__r": {"label": "not done", "touched": False}} +BAD_LABELS = dict(GOOD_LABELS, f3__r={"label": "done", "touched": True}) + + +def git(repo: str, args: List[str]) -> str: + p = subprocess.run(["git", "-C", repo, "-c", "user.name=Test", "-c", "user.email=test@example.invalid"] + args, + capture_output=True, text=True, check=True) + return p.stdout.strip() + + +def commit(repo: str, files: Dict[str, str], message: str) -> str: + for rel, text in files.items(): + write(os.path.join(repo, rel), text) + git(repo, ["add", "-A"]) + git(repo, ["commit", "-q", "-m", message]) + return git(repo, ["rev-parse", "HEAD"]) + + +def agent_file(fixed: bool, breaks_other: bool) -> str: + return f"FIXED = {fixed}\nBREAKS_OTHER = {breaks_other}\n" + + +@dataclass(frozen=True) +class Repo: + path: str + base: str + test_ref: str + good: str + bad: str + + +def make_repo(root: str) -> Repo: + path = os.path.join(root, "agent") + os.makedirs(path) + git(path, ["init", "-q", "-b", "main"]) + base = commit(path, {"agent.py": agent_file(False, False), "suite.py": SUITE, + "labels.json": json.dumps(BASE_LABELS)}, "base") + test_ref = commit(path, {"tests/test_fix.py": TEST_FIX}, "test: the trial f1 ends in the wrong state") + good = commit(path, {"agent.py": agent_file(True, False), "labels.json": json.dumps(GOOD_LABELS)}, "fix: good") + git(path, ["checkout", "-q", "--detach", test_ref]) + bad = commit(path, {"agent.py": agent_file(True, True), "labels.json": json.dumps(BAD_LABELS)}, "fix: bad") + git(path, ["checkout", "-q", "main"]) + return Repo(path=path, base=base, test_ref=test_ref, good=good, bad=bad) + + +def recorded_trials() -> List[Trial]: + out = [trial("p1__r", "bench/p1", 1.0), trial("p2__r", "bench/p2", 1.0), trial("p3__r", "bench/p3", 1.0)] + for name in ("f1", "f2"): + t = trial(f"{name}__r", f"bench/{name}", 0.0) + t.files = {"verifier/ctrf.json": '{"results": {"tests": [{"name": "test_state", "status": "failed"}]}}'} + out.append(t) + out.append(trial("f3__r", "bench/f3", 0.0)) + return out + + +class RepoCase(LoopCase): + """A loop over a recorded run of the made-up agent, read and classified, with one cause written.""" + + def commands(self) -> Dict[str, Optional[str]]: + py = sys.executable + return { + "test": f"cd {{worktree}} && {py} suite.py", + "test_one": f"cd {{worktree}} && {py} {{test}}", + "build": f"{py} {self.stubs}/build.py {{worktree}} {self.tmp}/built", + "run_tasks": f"{self.tmp}/no-such-runner --units {{units_file}} --repeat {{repeat}} --out {{jobs_dir}} " + f"--cap {{budget_usd}} --ref {{ref}} --build {{build}} --marker {self.tmp}/runner.marker", + "replay": f"{py} {self.stubs}/replay.py {{worktree}} {{job_dir}} {{ref}} {{out_json}} {self.tmp}/replay.marker", + "classes": None, + } + + def setUp(self) -> None: + super().setUp() + self.repo = make_repo(self.tmp) + self.stubs = os.path.join(self.tmp, "stubs") + write(os.path.join(self.stubs, "replay.py"), REPLAY) + write(os.path.join(self.stubs, "build.py"), BUILD) + self.run_dir = write_job(self.jobs, "recorded", "alpha", "harbor", recorded_trials()) + + def start(self, commands: Dict[str, Optional[str]], paid: List[str]) -> None: + data = adapter("alpha", {}, commands) + data["source"] = {"checkout": self.repo.path, "base_ref": self.repo.base, "work_branch": "main"} + data["test_failed_pattern"] = r"^FAILED (\S+)" + data["replay_success_labels"] = ["done"] + data["paid"] = paid + self.set_adapter(data) + self.read_ok(self.run_dir, None) + self.classify_ok() + write(os.path.join(self.loop, "round-01", "causes", "state.md"), + "# The agent reports done in the wrong state\n\nRule: g01\n\nTrials: 2 (f1__r, f2__r).\n") + + def candidate(self, cid: str, ref: str, test_ref: str) -> None: + r = run(["candidate", "--loop", self.loop, "--id", cid, "--cause", "state", "--ref", ref, + "--test", "tests/test_fix.py", "--test-ref", test_ref]) + self.assertEqual(r.code, 0, r.out + r.err) + + def check(self, cid: str) -> dict: + r = run(["check", "--loop", self.loop, "--candidate", cid]) + self.assertEqual(r.code, 0, r.out + r.err) + with open(self.round_file(os.path.join("candidates", cid, "checks.json")), encoding="utf-8") as fh: + return json.load(fh) + + def marker(self, name: str) -> List[str]: + path = os.path.join(self.tmp, name) + if not os.path.isfile(path): + return [] + with open(path, encoding="utf-8") as fh: + return fh.read().split() diff --git a/.claude/skills/run-to-fix/tests/test_check.py b/.claude/skills/run-to-fix/tests/test_check.py new file mode 100644 index 00000000..f1fb52a2 --- /dev/null +++ b/.claude/skills/run-to-fix/tests/test_check.py @@ -0,0 +1,163 @@ +"""`loop.py candidate` and `loop.py check`: the test, the suite and the replay of a candidate, in worktrees.""" +from __future__ import annotations + +import os + +from agentrepo import RepoCase, git +from jobs import load, run, write + + +class Candidate(RepoCase): + def test_records_refs_and_the_prediction_of_its_cause(self) -> None: + self.start(self.commands(), ["run_tasks"]) + self.candidate("good", self.repo.good[:8], self.repo.test_ref[:8]) + data = load(self.round_file("candidates/good/candidate.json")) + self.assertEqual((data["ref"], data["test_ref"]), (self.repo.good, self.repo.test_ref)) + self.assertEqual(data["cause"], "state") + self.assertEqual(data["rule"], "g01") + self.assertEqual(data["prediction"], {"trials": ["f1__r", "f2__r"], "tasks": ["bench/f1", "bench/f2"]}) + + def test_a_cause_without_a_rule_is_refused(self) -> None: + self.start(self.commands(), ["run_tasks"]) + write(self.round_file("causes/vague.md"), "# Something is wrong\n") + r = run(["candidate", "--loop", self.loop, "--id", "c1", "--cause", "vague", "--ref", self.repo.good, + "--test", "tests/test_fix.py", "--test-ref", self.repo.test_ref]) + self.assertNotEqual(r.code, 0) + self.assertIn("Rule:", r.err) + + def test_a_rule_that_counts_nothing_is_refused(self) -> None: + self.start(self.commands(), ["run_tasks"]) + write(self.round_file("causes/ghost.md"), "# A cause\n\nRule: g09\n") + r = run(["candidate", "--loop", self.loop, "--id", "c1", "--cause", "ghost", "--ref", self.repo.good, + "--test", "tests/test_fix.py", "--test-ref", self.repo.test_ref]) + self.assertNotEqual(r.code, 0) + self.assertIn("g09", r.err) + + def test_an_unknown_ref_is_refused(self) -> None: + self.start(self.commands(), ["run_tasks"]) + r = run(["candidate", "--loop", self.loop, "--id", "c1", "--cause", "state", "--ref", "no-such-ref", + "--test", "tests/test_fix.py", "--test-ref", self.repo.test_ref]) + self.assertNotEqual(r.code, 0) + self.assertIn("no-such-ref", r.err) + + +class Check(RepoCase): + def test_a_good_candidate(self) -> None: + self.start(self.commands(), ["run_tasks"]) + self.candidate("good", self.repo.good, self.repo.test_ref) + checks = self.check("good") + self.assertEqual(checks["test"]["ran"], True) + self.assertTrue(checks["test"]["fails_before"]) + self.assertTrue(checks["test"]["passes_after"]) + self.assertEqual(checks["suite"]["base_failures"], ["test_old"]) + self.assertEqual((checks["suite"]["added"], checks["suite"]["removed"]), ([], [])) + replay = checks["replay"] + self.assertTrue(replay["ran"]) + self.assertEqual(replay["before"], {"done_pass": 2, "done_fail": 2, "not_done_pass": 1, "not_done_fail": 1}) + self.assertEqual(replay["after"], {"done_pass": 3, "done_fail": 0, "not_done_pass": 0, "not_done_fail": 3}) + self.assertEqual(replay["false_success_added"], -2) + self.assertEqual(replay["touched"], ["f1__r", "f2__r", "p1__r"]) + self.assertEqual(checks["diff"]["files"], 2) + + def test_a_candidate_that_adds_failures_and_false_successes(self) -> None: + self.start(self.commands(), ["run_tasks"]) + self.candidate("bad", self.repo.bad, self.repo.test_ref) + checks = self.check("bad") + self.assertEqual(checks["suite"]["added"], ["test_other"]) + self.assertEqual(checks["replay"]["after"]["done_fail"], 1) + self.assertEqual(checks["replay"]["false_success_added"], -1) + self.assertEqual(checks["replay"]["false_success_new"], ["f3__r"]) + + def test_a_test_that_does_not_fail_before_the_fix(self) -> None: + self.start(self.commands(), ["run_tasks"]) + self.candidate("weak", self.repo.good, self.repo.good) + checks = self.check("weak") + self.assertFalse(checks["test"]["fails_before"]) + + def test_the_base_is_checked_once_per_round(self) -> None: + self.start(self.commands(), ["run_tasks"]) + self.candidate("good", self.repo.good, self.repo.test_ref) + self.candidate("bad", self.repo.bad, self.repo.test_ref) + self.check("good") + self.check("bad") + self.assertEqual(self.marker("replay.marker"), [self.repo.base, self.repo.good, self.repo.bad]) + + def test_worktrees_are_removed_and_the_checkout_is_untouched(self) -> None: + self.start(self.commands(), ["run_tasks"]) + head = git(self.repo.path, ["rev-parse", "HEAD"]) + self.candidate("good", self.repo.good, self.repo.test_ref) + self.check("good") + self.assertEqual(git(self.repo.path, ["worktree", "list", "--porcelain"]).count("worktree "), 1) + self.assertEqual(os.listdir(os.path.join(self.loop, "worktrees")), []) + self.assertEqual(git(self.repo.path, ["rev-parse", "HEAD"]), head) + self.assertEqual(git(self.repo.path, ["status", "--porcelain"]), "") + + def test_a_failing_replay_still_removes_its_worktree(self) -> None: + commands = self.commands() + commands["replay"] = "exit 4 # {worktree} {ref} {job_dir} {out_json}" + self.start(commands, ["run_tasks"]) + self.candidate("good", self.repo.good, self.repo.test_ref) + r = run(["check", "--loop", self.loop, "--candidate", "good"]) + self.assertNotEqual(r.code, 0) + self.assertIn("commands.replay", r.err) + self.assertEqual(git(self.repo.path, ["worktree", "list", "--porcelain"]).count("worktree "), 1) + + +class CheckFive(RepoCase): + """Known-answer check 5 of the brief, with the stub adapter.""" + + def test_a_replay_of_two_refs_gives_the_table_and_the_false_results(self) -> None: + self.start(self.commands(), ["run_tasks"]) + self.candidate("bad", self.repo.bad, self.repo.test_ref) + replay = self.check("bad")["replay"] + self.assertEqual(replay["base_ref"], self.repo.base) + self.assertEqual(replay["ref"], self.repo.bad) + self.assertEqual(replay["before"], {"done_pass": 2, "done_fail": 2, "not_done_pass": 1, "not_done_fail": 1}) + self.assertEqual(replay["after"], {"done_pass": 3, "done_fail": 1, "not_done_pass": 0, "not_done_fail": 2}) + self.assertEqual((replay["false_success_before"], replay["false_success_after"]), (2, 1)) + + def test_a_paid_command_is_written_and_not_started(self) -> None: + self.start(self.commands(), ["run_tasks", "replay"]) + self.candidate("good", self.repo.good, self.repo.test_ref) + replay = self.check("good")["replay"] + self.assertFalse(replay["ran"]) + self.assertIn("costs money", replay["reason"]) + self.assertEqual(self.marker("replay.marker"), []) + written = sorted(os.listdir(self.round_file("commands"))) + self.assertEqual(written, sorted([f"replay-{self.repo.base[:12]}.sh", f"replay-{self.repo.good[:12]}.sh"])) + with open(self.round_file(os.path.join("commands", f"replay-{self.repo.good[:12]}.sh")), encoding="utf-8") as fh: + script = fh.read() + self.assertIn(self.repo.good, script) + self.assertIn("worktree add --detach", script) + self.assertIn("worktree remove", script) + + def test_a_null_command_is_a_check_that_did_not_run(self) -> None: + commands = self.commands() + commands["test"] = None + commands["replay"] = None + self.start(commands, ["run_tasks"]) + self.candidate("good", self.repo.good, self.repo.test_ref) + checks = self.check("good") + self.assertFalse(checks["suite"]["ran"]) + self.assertIn("commands.test is null", checks["suite"]["reason"]) + self.assertFalse(checks["replay"]["ran"]) + self.assertIn("commands.replay is null", checks["replay"]["reason"]) + r = run(["decide", "--loop", self.loop, "--stop", "1"]) + self.assertEqual(r.code, 0, r.out + r.err) + with open(self.round_file("decision-1.md"), encoding="utf-8") as fh: + page = fh.read() + self.assertIn("did not run", page) + self.assertIn("commands.test is null", page) + self.assertIn("commands.replay is null", page) + self.assertNotIn("{{", page) + + def test_a_replay_that_leaves_out_a_trial_stops(self) -> None: + commands = self.commands() + commands["replay"] = (commands["replay"] or "") + " && python3 -c \"import json,sys; p=sys.argv[1]; " \ + "d=json.load(open(p)); d['trials'].pop('f3__r'); json.dump(d, open(p, 'w'))\" {out_json}" + self.start(commands, ["run_tasks"]) + self.candidate("good", self.repo.good, self.repo.test_ref) + r = run(["check", "--loop", self.loop, "--candidate", "good"]) + self.assertNotEqual(r.code, 0) + self.assertIn("f3__r", r.err) + diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py new file mode 100644 index 00000000..08833907 --- /dev/null +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -0,0 +1,211 @@ +"""`loop.py decide`, `answer` and `select`: the two stops, the person's answers and the trial run.""" +from __future__ import annotations + +import os +from typing import List + +from agentrepo import RepoCase +from jobs import Result, load, run, trial, write_job + +KEEP_GOOD = "Keep good. bad breaks test_other." + + +class Flow(RepoCase): + """The steps up to each stop, shared by the tests below.""" + + def checked(self) -> None: + self.start(self.commands(), ["run_tasks"]) + self.candidate("good", self.repo.good, self.repo.test_ref) + self.candidate("bad", self.repo.bad, self.repo.test_ref) + self.check("good") + self.check("bad") + + def page(self, name: str) -> str: + with open(self.round_file(name), encoding="utf-8") as fh: + return fh.read() + + def kept(self) -> None: + self.checked() + run(["decide", "--loop", self.loop, "--stop", "1"]) + r = run(["answer", "--loop", self.loop, "--stop", "1", "--keep", "good", "--text", KEEP_GOOD]) + self.assertEqual(r.code, 0, r.out + r.err) + + def select(self, extra: List[str]) -> Result: + return run(["select", "--loop", self.loop, "--ref", self.repo.good, "--canaries", "1"] + extra) + + def trial_job(self) -> str: + trials = [] + for task, passes in (("f1", 3), ("f2", 2), ("p1", 3)): + for k in range(3): + trials.append(trial(f"{task}-{k}__t", f"bench/{task}", 1.0 if k < passes else 0.0)) + return write_job(self.jobs, "trial", "alpha", "harbor", trials) + + def chosen(self) -> str: + self.kept() + r = self.select(["--size", "guards", "--budget-usd", "0.5", "--text", "targets and guards"]) + self.assertEqual(r.code, 0, r.out + r.err) + return self.trial_job() + + +class StopOne(Flow): + def test_one_row_per_candidate(self) -> None: + self.checked() + r = run(["decide", "--loop", self.loop, "--stop", "1"]) + self.assertEqual(r.code, 0, r.out + r.err) + page = self.page("decision-1.md") + rows = [ln for ln in page.splitlines() if ln.startswith("| good |") or ln.startswith("| bad |")] + self.assertEqual(len(rows), 2) + good = next(ln for ln in rows if ln.startswith("| good |")) + bad = next(ln for ln in rows if ln.startswith("| bad |")) + self.assertIn("2 / 2 / 1 / 1 → 3 / 0 / 0 / 3", good) + self.assertIn("2 / 2 / 1 / 1 → 3 / 1 / 0 / 2", bad) + self.assertIn("test_other", bad) + self.assertIn("f3__r", bad) + self.assertIn("fails, then passes", good) + self.assertNotIn("{{", page) + + def test_the_answer_is_stored_in_the_person_s_words(self) -> None: + self.checked() + run(["decide", "--loop", self.loop, "--stop", "1"]) + r = run(["answer", "--loop", self.loop, "--stop", "1", "--keep", "good", "--text", KEEP_GOOD]) + self.assertEqual(r.code, 0, r.out + r.err) + entry = load(os.path.join(self.loop, "ledger.json"))["rounds"][0] + self.assertEqual(entry["answers"][0]["text"], KEEP_GOOD) + self.assertEqual(entry["answers"][0]["about"], "stop 1: the candidates to keep") + self.assertEqual(entry["kept"], ["good"]) + self.assertEqual(entry["predictions"], [{"candidate": "good", "cause": "state", "rule": "g01", + "tasks": ["bench/f1", "bench/f2"], "expect": "pass"}]) + + def test_an_answer_before_the_page_is_refused(self) -> None: + self.checked() + r = run(["answer", "--loop", self.loop, "--stop", "1", "--keep", "good", "--text", KEEP_GOOD]) + self.assertNotEqual(r.code, 0) + self.assertIn("decision-1.md", r.err) + + def test_keeping_an_unchecked_candidate_is_refused(self) -> None: + self.checked() + run(["decide", "--loop", self.loop, "--stop", "1"]) + r = run(["answer", "--loop", self.loop, "--stop", "1", "--keep", "good,other", "--text", "keep both"]) + self.assertNotEqual(r.code, 0) + self.assertIn("other", r.err) + + +class Select(Flow): + def test_select_before_stop_one_is_refused(self) -> None: + self.checked() + r = self.select([]) + self.assertNotEqual(r.code, 0) + self.assertIn("stop 1", r.err) + + def test_the_proposal_prices_each_size(self) -> None: + self.kept() + r = self.select([]) + self.assertEqual(r.code, 0, r.out + r.err) + self.assertIn("targets: 2 tasks, 6 trials, $0.06", r.out) + self.assertIn("targets and guards: 3 tasks, 9 trials, $0.09", r.out) + self.assertIn("targets, guards and canaries: 4 tasks, 12 trials, $0.12", r.out) + self.assertFalse(os.path.exists(self.round_file("trial-tasks.txt"))) + + def test_a_ref_without_a_kept_candidate_is_refused(self) -> None: + self.kept() + r = run(["select", "--loop", self.loop, "--ref", self.repo.bad, "--canaries", "1"]) + self.assertNotEqual(r.code, 0) + self.assertIn("good", r.err) + + def test_choosing_a_size_writes_the_run_and_starts_nothing(self) -> None: + self.kept() + words = "Run targets and guards, 0.50 USD." + r = self.select(["--size", "guards", "--budget-usd", "0.5", "--text", words]) + self.assertEqual(r.code, 0, r.out + r.err) + with open(self.round_file("trial-tasks.txt"), encoding="utf-8") as fh: + self.assertEqual(fh.read().split(), ["bench/f1", "bench/f2", "bench/p1"]) + entry = load(os.path.join(self.loop, "ledger.json"))["rounds"][0] + sel = entry["selection"] + self.assertEqual((sel["targets"], sel["guards"], sel["canaries"]), (["bench/f1", "bench/f2"], ["bench/p1"], [])) + self.assertEqual((sel["repeat"], sel["budget_usd"], sel["ref"]), (3, 0.5, self.repo.good)) + self.assertAlmostEqual(sel["price_usd"], 0.09) + self.assertEqual(entry["answers"][-1]["text"], words) + build = load(self.round_file("build.json")) + self.assertTrue(os.path.isfile(build["path"])) + self.assertEqual(build["ref"], self.repo.good) + with open(self.round_file("commands/run_tasks.sh"), encoding="utf-8") as fh: + script = fh.read() + for part in (self.round_file("trial-tasks.txt"), "--repeat 3", "--cap 0.5", self.repo.good, build["path"]): + self.assertIn(part, script) + self.assertEqual(self.marker("runner.marker"), []) + + def test_a_budget_under_the_price_is_refused(self) -> None: + self.kept() + r = self.select(["--size", "guards", "--budget-usd", "0.05", "--text", "five cents"]) + self.assertNotEqual(r.code, 0) + self.assertIn("$0.09", r.err) + self.assertFalse(os.path.exists(self.round_file("trial-tasks.txt"))) + + def test_a_budget_over_the_round_s_is_refused(self) -> None: + self.kept() + r = self.select(["--size", "targets", "--budget-usd", "11", "--text", "eleven"]) + self.assertNotEqual(r.code, 0) + self.assertIn("$10.00", r.err) + + def test_canaries_come_from_a_fixed_seed(self) -> None: + self.kept() + first = self.select([]).out + self.assertEqual(self.select([]).out, first) + + def test_without_touched_there_are_no_guards(self) -> None: + commands = self.commands() + commands["replay"] = None + self.start(commands, ["run_tasks"]) + self.candidate("good", self.repo.good, self.repo.test_ref) + self.check("good") + run(["decide", "--loop", self.loop, "--stop", "1"]) + run(["answer", "--loop", self.loop, "--stop", "1", "--keep", "good", "--text", "keep good"]) + r = self.select([]) + self.assertEqual(r.code, 0, r.out + r.err) + self.assertIn("guards: none", r.out) + self.assertIn("no replay", r.out) + + +class StopTwo(Flow): + def test_the_page_gives_three_options_with_their_cost(self) -> None: + job = self.chosen() + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job]) + self.assertEqual(r.code, 0, r.out + r.err) + trial_record = load(self.round_file("trial.json")) + self.assertAlmostEqual(trial_record["cost_usd"], 0.09) + self.assertAlmostEqual(trial_record["budget_left_usd"], 9.91) + self.assertEqual(trial_record["targets"]["before"], {"trials": 2, "passed": 0}) + self.assertEqual(trial_record["targets"]["after"], {"trials": 6, "passed": 5}) + self.assertEqual(trial_record["guards"], {"trials": 3, "passed": 3}) + options = {o["name"]: o for o in trial_record["options"]} + self.assertEqual(set(options), {"stop", "rerun", "full"}) + self.assertAlmostEqual(options["stop"]["usd"], 0.0) + self.assertAlmostEqual(options["rerun"]["usd"], 0.09) + self.assertAlmostEqual(options["full"]["usd"], 0.06) + page = self.page("decision-2.md") + for text in ("$9.91", "$0.09", "$0.06", "5 of 6", "agent-"): + self.assertIn(text, page) + self.assertNotIn("{{", page) + for name in ("run_tasks-rerun.sh", "run_tasks-full.sh"): + self.assertTrue(os.path.isfile(self.round_file(os.path.join("commands", name)))) + with open(self.round_file("full-tasks.txt"), encoding="utf-8") as fh: + self.assertEqual(len(fh.read().split()), 6) + self.assertEqual(self.marker("runner.marker"), []) + + def test_the_answer_of_stop_two(self) -> None: + job = self.chosen() + run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job]) + words = "A full pass, the targets look fixed." + r = run(["answer", "--loop", self.loop, "--stop", "2", "--choice", "full", "--text", words]) + self.assertEqual(r.code, 0, r.out + r.err) + entry = load(os.path.join(self.loop, "ledger.json"))["rounds"][0] + self.assertEqual(entry["answers"][-1], {"about": "stop 2: what to run next", "date": entry["answers"][-1]["date"], + "text": words, "choice": "full"}) + + def test_compare_reads_the_selection_and_the_predictions(self) -> None: + job = self.chosen() + r = run(["compare", "--loop", self.loop, "--new", job, "--against", "run"]) + self.assertEqual(r.code, 0, r.out + r.err) + result = load(self.round_file("result.json")) + self.assertEqual(result["targets"]["after"], {"trials": 6, "passed": 5}) + self.assertEqual(result["predictions"][0]["result"], {"trials": 6, "passed": 5, "tasks": 2}) From 508a5ba923ebc26a3fd09503cb13da9953f67cd7 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:35:35 +0100 Subject: [PATCH 10/92] feat(run-to-fix): check candidates and write both decision pages candidate registers a fix of a cause and the trials it predicts. check runs the cause's test before and after the fix, the agent's suite against its failures on the base ref, and the replay on both refs, each in a worktree that is removed when the check ends; a paid command is written for a person to start. decide writes the pages of stop 1 and stop 2, answer stores the person's words, and select prices the trial run, builds it and writes its run command. --- .../skills/run-to-fix/assets/decision-1.md | 20 ++ .../skills/run-to-fix/assets/decision-2.md | 19 ++ .claude/skills/run-to-fix/scripts/checking.py | 224 ++++++++++++++++++ .claude/skills/run-to-fix/scripts/checkout.py | 70 ++++++ .claude/skills/run-to-fix/scripts/deciding.py | 204 ++++++++++++++++ .claude/skills/run-to-fix/scripts/loop.py | 48 +++- .../skills/run-to-fix/scripts/selecting.py | 135 +++++++++++ 7 files changed, 717 insertions(+), 3 deletions(-) create mode 100644 .claude/skills/run-to-fix/assets/decision-1.md create mode 100644 .claude/skills/run-to-fix/assets/decision-2.md create mode 100644 .claude/skills/run-to-fix/scripts/checking.py create mode 100644 .claude/skills/run-to-fix/scripts/checkout.py create mode 100644 .claude/skills/run-to-fix/scripts/deciding.py create mode 100644 .claude/skills/run-to-fix/scripts/selecting.py diff --git a/.claude/skills/run-to-fix/assets/decision-1.md b/.claude/skills/run-to-fix/assets/decision-1.md new file mode 100644 index 00000000..daf0b433 --- /dev/null +++ b/.claude/skills/run-to-fix/assets/decision-1.md @@ -0,0 +1,20 @@ +# Stop 1 of {{ROUND}}: the candidates to keep + +{{SUMMARY}} + +| candidate | cause | trials the cause explains | test before → after the fix | suite failures added | replay before → after | false successes added | diff | +|---|---|---|---|---|---|---|---| +{{ROWS}} + +Replay counts are of the {{TRIALS}} recorded trials: done and passed / done and failed / not done and passed / not +done and failed. A false success is a failed trial that the agent reports as a success. + +## Checks that did not run + +{{NOT_RUN}} + +## Answer + +``` +python3 loop.py answer --loop {{LOOP}} --stop 1 --keep IDS --text "the person's words" +``` diff --git a/.claude/skills/run-to-fix/assets/decision-2.md b/.claude/skills/run-to-fix/assets/decision-2.md new file mode 100644 index 00000000..b8726af9 --- /dev/null +++ b/.claude/skills/run-to-fix/assets/decision-2.md @@ -0,0 +1,19 @@ +# Stop 2 of {{ROUND}}: what to run next + +Build: {{BUILD}}. Trial run: {{TRIAL_JOB}}, {{TRIALS}} trials, {{SPENT}}. Budget left: {{LEFT}} of {{BUDGET}}. + +| tasks | trials passed before | trials passed in the trial run | test | +|---|---|---|---| +{{RESULTS}} + +| option | what runs | price | command | +|---|---|---|---| +{{OPTIONS}} + +Prices use the trial run's cost per trial, {{PER_TRIAL}}. + +## Answer + +``` +python3 loop.py answer --loop {{LOOP}} --stop 2 --choice stop|rerun|full --text "the person's words" +``` diff --git a/.claude/skills/run-to-fix/scripts/checking.py b/.claude/skills/run-to-fix/scripts/checking.py new file mode 100644 index 00000000..74612ed6 --- /dev/null +++ b/.claude/skills/run-to-fix/scripts/checking.py @@ -0,0 +1,224 @@ +"""Stage 3, `loop.py candidate` and `loop.py check`: a candidate's test, the agent's suite and the replay.""" +from __future__ import annotations + +import argparse +import csv +import os +import re +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple + +from adapter import Adapter +from checkout import diff_size, resolve, worktree, write_paid_on_ref +from classifying import read_rows +from reading import adapter_of +from rounds import LoopError, Ran, now, open_round, read_input, read_json, run_command, write_json +from trials import TrialRow + +CANDIDATE_ID = re.compile(r"^[a-z0-9][a-z0-9_-]{0,39}$") +RULE_LINE = re.compile(r"(?m)^Rule:\s*(\S+)\s*$") +TABLE = ("done_pass", "done_fail", "not_done_pass", "not_done_fail") + + +def cause_trials(round_dir: str, cause: str) -> Tuple[str, List[str], List[str]]: + """The rule a cause names and the failing trials and tasks it counts.""" + path = os.path.join(round_dir, "causes", f"{cause}.md") + if not os.path.isfile(path): + raise LoopError(f"{path} is missing: write the cause before its candidates") + with open(path, encoding="utf-8") as fh: + m = RULE_LINE.search(fh.read()) + if not m: + raise LoopError(f"{path} needs a line 'Rule: ' naming the group of groups.json or the found class that " + "counts its trials") + rule = m.group(1) + with open(os.path.join(round_dir, "trials.csv"), encoding="utf-8", newline="") as fh: + table = list(csv.DictReader(fh)) + groups = {g["id"]: g for g in read_json(os.path.join(round_dir, "groups.json"))["groups"]} + if rule in groups: + names = set(groups[rule]["trial_names"]) + elif table and f"found:{rule}" in table[0]: + names = {r["trial"] for r in table if r[f"found:{rule}"] == "1" and r["class"] != "pass"} + else: + raise LoopError(f"{path} names the rule {rule}, which is neither a group of groups.json nor a found class " + "of trials.csv; run loop.py classify after adding a found class") + if not names: + raise LoopError(f"the rule {rule} of {path} counts no failing trial") + tasks = sorted({r["task"] for r in table if r["trial"] in names}) + return rule, sorted(names), tasks + + +def cmd_candidate(args: argparse.Namespace) -> None: + adapter = adapter_of(args.loop) + round_dir = open_round(args.loop) + read_input(round_dir) + if not CANDIDATE_ID.match(args.id): + raise LoopError(f"a candidate id is lower-case letters, digits, - and _, got {args.id!r}") + if (args.test is None) != (args.test_ref is None): + raise LoopError("--test and --test-ref go together: the test file and the commit that adds it before the fix") + if args.test is not None and (os.path.isabs(args.test) or ".." in re.split(r"[\\/]", args.test)): + raise LoopError(f"--test is a path inside the checkout, got {args.test!r}") + rule, trials, tasks = cause_trials(round_dir, args.cause) + ref = resolve(adapter.checkout, args.ref) + test_ref = resolve(adapter.checkout, args.test_ref) if args.test_ref else None + record = {"schema": "run-to-fix.candidate/v1", "id": args.id, "created": now(), "cause": args.cause, "rule": rule, + "ref": ref, "test": args.test, "test_ref": test_ref, "prediction": {"trials": trials, "tasks": tasks}} + write_json(os.path.join(round_dir, "candidates", args.id, "candidate.json"), record) + print(f"candidate {args.id}: {ref[:12]} for the cause {args.cause}, which counts {len(trials)} trials " + f"on {len(tasks)} tasks ({rule})") + + +@dataclass(frozen=True) +class Outcome: + ran: Optional[Ran] + reason: Optional[str] + + +def on_ref(adapter: Adapter, loop_dir: str, round_dir: str, name: str, sha: str, values: Dict[str, str], + log: str, timeout_s: float) -> Outcome: + """Run an adapter command in a worktree of `sha`, or say why it did not run.""" + line = adapter.commands[name] + if line is None: + return Outcome(None, f"commands.{name} is null: this check did not run") + if name in adapter.paid: + path = os.path.join(round_dir, "commands", f"{name}-{sha[:12]}.sh") + write_paid_on_ref(path, adapter.checkout, loop_dir, sha, line, values) + return Outcome(None, f"commands.{name} costs money: written to {path}, not started") + with worktree(adapter.checkout, loop_dir, sha) as tree: + ran = run_command(line, dict(values, worktree=tree), tree, log, timeout_s) + if ran.code is None: + raise LoopError(f"commands.{name} on {sha[:12]} did not finish in time; the log is {ran.log}") + return Outcome(ran, None) + + +def ensure_ok(name: str, outcome: Outcome) -> None: + if outcome.ran is not None and outcome.ran.code != 0: + raise LoopError(f"commands.{name} exited with code {outcome.ran.code}; the log is {outcome.ran.log}:\n" + f"{outcome.ran.tail}") + + +def check_test(adapter: Adapter, args: argparse.Namespace, round_dir: str, cdir: str, cand: dict) -> dict: + if cand["test"] is None: + return {"ran": False, "reason": "the candidate names no test"} + out = {} + for when, sha in (("before", cand["test_ref"]), ("after", cand["ref"])): + o = on_ref(adapter, args.loop, round_dir, "test_one", sha, {"test": cand["test"]}, + os.path.join(cdir, f"test-{when}.log"), args.command_timeout_s) + if o.ran is None: + return {"ran": False, "reason": o.reason} + out[when] = {"ref": sha, "code": o.ran.code} + return {"ran": True, "reason": None, "test": cand["test"], **out, + "fails_before": out["before"]["code"] != 0, "passes_after": out["after"]["code"] == 0} + + +def suite_failures(adapter: Adapter, name: str, outcome: Outcome) -> List[str]: + assert outcome.ran is not None and adapter.test_failed_pattern is not None + with open(outcome.ran.log, encoding="utf-8", errors="replace") as fh: + lines = fh.read().splitlines()[1:] + found = sorted({m.group(1) for m in (adapter.test_failed_pattern.search(ln) for ln in lines) if m}) + if outcome.ran.code != 0 and not found: + raise LoopError(f"commands.test on {name} exited with code {outcome.ran.code} and printed no failed test that " + f"test_failed_pattern takes: the suite is broken. The log is {outcome.ran.log}:\n{outcome.ran.tail}") + return found + + +def check_suite(adapter: Adapter, args: argparse.Namespace, round_dir: str, cdir: str, base: str, ref: str) -> dict: + cache = os.path.join(round_dir, "base", "suite.json") + if os.path.isfile(cache) and read_json(cache)["ref"] == base: + base_failures = read_json(cache)["failures"] + else: + o = on_ref(adapter, args.loop, round_dir, "test", base, {}, os.path.join(round_dir, "base", "suite.log"), + args.command_timeout_s) + if o.ran is None: + return {"ran": False, "reason": o.reason} + base_failures = suite_failures(adapter, "base_ref", o) + write_json(cache, {"ref": base, "failures": base_failures}) + o = on_ref(adapter, args.loop, round_dir, "test", ref, {}, os.path.join(cdir, "suite.log"), args.command_timeout_s) + if o.ran is None: + return {"ran": False, "reason": o.reason} + failures = suite_failures(adapter, ref[:12], o) + return {"ran": True, "reason": None, "base_ref": base, "base_failures": base_failures, "failures": failures, + "added": sorted(set(failures) - set(base_failures)), "removed": sorted(set(base_failures) - set(failures))} + + +def replay_labels(path: str, rows: List[TrialRow]) -> dict: + data = read_json(path) + trials = data.get("trials") if isinstance(data, dict) else None + if not isinstance(trials, dict): + raise LoopError(f"{path} must be {{\"ref\": ..., \"trials\": {{\"\": {{\"label\": ...}}}}}}") + missing = sorted(r.trial for r in rows if not isinstance(trials.get(r.trial), dict) + or not isinstance(trials[r.trial].get("label"), str)) + if missing: + raise LoopError(f"{path} has no label for {len(missing)} recorded trials: {', '.join(missing[:10])}") + return data + + +def cross(data: dict, rows: List[TrialRow], success: List[str]) -> Dict[str, int]: + table = dict.fromkeys(TABLE, 0) + for r in rows: + done = data["trials"][r.trial]["label"] in success + table[("done_" if done else "not_done_") + ("pass" if r.passed else "fail")] += 1 + return table + + +def false_successes(data: dict, rows: List[TrialRow], success: List[str]) -> List[str]: + return sorted(r.trial for r in rows if not r.passed and data["trials"][r.trial]["label"] in success) + + +def check_replay(adapter: Adapter, args: argparse.Namespace, round_dir: str, cdir: str, base: str, ref: str, + job_dir: str, rows: List[TrialRow]) -> dict: + labels = {} + for when, sha, out in (("before", base, os.path.join(round_dir, "base", "replay.json")), + ("after", ref, os.path.join(cdir, "replay.json"))): + if when == "before" and os.path.isfile(out) and read_json(out).get("_loop_ref") == base: + labels[when] = replay_labels(out, rows) + continue + o = on_ref(adapter, args.loop, round_dir, "replay", sha, {"ref": sha, "job_dir": job_dir, "out_json": out}, + os.path.join(os.path.dirname(out), "replay.log"), args.command_timeout_s) + if o.ran is None: + if when == "before": + on_ref(adapter, args.loop, round_dir, "replay", ref, {"ref": ref, "job_dir": job_dir, "out_json": + os.path.join(cdir, "replay.json")}, os.path.join(cdir, "replay.log"), args.command_timeout_s) + return {"ran": False, "reason": o.reason} + ensure_ok("replay", o) + data = replay_labels(out, rows) + if when == "before": + write_json(out, dict(data, _loop_ref=base)) + labels[when] = data + success = adapter.replay_success_labels + before, after = false_successes(labels["before"], rows, success), false_successes(labels["after"], rows, success) + touched_given = any("touched" in v for v in labels["after"]["trials"].values()) + return {"ran": True, "reason": None, "base_ref": base, "ref": ref, + "counts": "recorded trials: done and passed / done and failed / not done and passed / not done and failed", + "before": cross(labels["before"], rows, success), "after": cross(labels["after"], rows, success), + "false_success_before": len(before), "false_success_after": len(after), + "false_success_added": len(after) - len(before), "false_success_new": sorted(set(after) - set(before)), + "kept_recorded": {w: len(labels[w].get("kept_recorded") or []) for w in ("before", "after")}, + "touched": sorted(n for n, v in labels["after"]["trials"].items() if v.get("touched") is True) + if touched_given else None} + + +def cmd_check(args: argparse.Namespace) -> None: + adapter = adapter_of(args.loop) + round_dir = open_round(args.loop) + record = read_input(round_dir) + cdir = os.path.join(round_dir, "candidates", args.candidate) + path = os.path.join(cdir, "candidate.json") + if not os.path.isfile(path): + raise LoopError(f"{path} is missing; register the candidate with loop.py candidate") + cand = read_json(path) + job_dir = record["run"]["job_dir"] + rows = read_rows(job_dir, float(record["pass_threshold"])) + base = resolve(adapter.checkout, adapter.base_ref) + checks = { + "schema": "run-to-fix.checks/v1", "candidate": cand["id"], "created": now(), "ref": cand["ref"], + "test_ref": cand["test_ref"], "base_ref": base, + "test": check_test(adapter, args, round_dir, cdir, cand), + "suite": check_suite(adapter, args, round_dir, cdir, base, cand["ref"]), + "replay": check_replay(adapter, args, round_dir, cdir, base, cand["ref"], job_dir, rows), + "diff": diff_size(adapter.checkout, cand["test_ref"] or base, cand["ref"]), + } + write_json(os.path.join(cdir, "checks.json"), checks) + print(f"wrote {cdir}/checks.json") + for name in ("test", "suite", "replay"): + part = checks[name] + print(f" {name}: " + ("ran" if part["ran"] else f"did not run: {part['reason']}")) diff --git a/.claude/skills/run-to-fix/scripts/checkout.py b/.claude/skills/run-to-fix/scripts/checkout.py new file mode 100644 index 00000000..23ba916f --- /dev/null +++ b/.claude/skills/run-to-fix/scripts/checkout.py @@ -0,0 +1,70 @@ +"""The agent's git checkout: refs, diffs, and detached worktrees under the loop folder.""" +from __future__ import annotations + +import contextlib +import os +import shlex +import subprocess +from typing import Dict, Iterator, List + +from rounds import LoopError, fill + + +def git(checkout: str, args: List[str]) -> str: + p = subprocess.run(["git", "-C", checkout] + args, capture_output=True, text=True, stdin=subprocess.DEVNULL) + if p.returncode != 0: + raise LoopError(f"git {' '.join(args)} in {checkout} failed: {(p.stderr or p.stdout).strip()}") + return p.stdout.strip() + + +def resolve(checkout: str, ref: str) -> str: + """The full commit id of `ref` in the checkout.""" + p = subprocess.run(["git", "-C", checkout, "rev-parse", "--verify", "--quiet", f"{ref}^{{commit}}"], + capture_output=True, text=True, stdin=subprocess.DEVNULL) + if p.returncode != 0 or not p.stdout.strip(): + raise LoopError(f"{ref!r} is not a commit of {checkout}") + return p.stdout.strip() + + +def is_ancestor(checkout: str, older: str, newer: str) -> bool: + p = subprocess.run(["git", "-C", checkout, "merge-base", "--is-ancestor", older, newer], + capture_output=True, text=True, stdin=subprocess.DEVNULL) + if p.returncode not in (0, 1): + raise LoopError(f"git merge-base in {checkout} failed: {p.stderr.strip()}") + return p.returncode == 0 + + +def diff_size(checkout: str, old: str, new: str) -> Dict[str, int]: + """Files and lines a commit range changes, from `git diff --numstat` (a binary file counts no lines).""" + files, added, removed = 0, 0, 0 + for line in git(checkout, ["diff", "--numstat", old, new]).splitlines(): + a, r, _ = line.split("\t", 2) + files += 1 + added += int(a) if a.isdigit() else 0 + removed += int(r) if r.isdigit() else 0 + return {"files": files, "added": added, "removed": removed} + + +@contextlib.contextmanager +def worktree(checkout: str, loop_dir: str, sha: str) -> Iterator[str]: + """A detached worktree of `sha` under /worktrees/, removed when the block ends, on failure too.""" + base = os.path.join(loop_dir, "worktrees") + os.makedirs(base, exist_ok=True) + path = os.path.join(base, f"{sha[:12]}-{os.getpid()}") + git(checkout, ["worktree", "add", "--detach", path, sha]) + try: + yield path + finally: + git(checkout, ["worktree", "remove", "--force", path]) + + +def write_paid_on_ref(path: str, checkout: str, loop_dir: str, sha: str, line: str, values: Dict[str, str]) -> None: + """A paid command that needs a worktree, written as a script that makes the worktree, runs it, and removes it.""" + tree = os.path.join(loop_dir, "worktrees", f"{sha[:12]}-paid") + os.makedirs(os.path.dirname(path), exist_ok=True) + q = shlex.quote + with open(path, "w", encoding="utf-8", newline="\n") as fh: + fh.write("#!/usr/bin/env bash\nset -euo pipefail\n") + fh.write(f"git -C {q(checkout)} worktree add --detach {q(tree)} {q(sha)}\n") + fh.write(f"trap 'git -C {q(checkout)} worktree remove --force {q(tree)}' EXIT\n") + fh.write(f"cd {q(tree)}\n{fill(line, dict(values, worktree=tree))}\n") diff --git a/.claude/skills/run-to-fix/scripts/deciding.py b/.claude/skills/run-to-fix/scripts/deciding.py new file mode 100644 index 00000000..835df220 --- /dev/null +++ b/.claude/skills/run-to-fix/scripts/deciding.py @@ -0,0 +1,204 @@ +"""The two stops: `loop.py decide` writes a decision page, `loop.py answer` records the person's answer.""" +from __future__ import annotations + +import argparse +import os +import re +from typing import Dict, List, Optional + +from classifying import read_rows +from comparing import TARGET_TRIALS, passes, targets_of, usd +from reading import adapter_of, health +from rounds import LoopError, ledger_round, load_ledger, now, open_round, read_input, read_json, save_ledger, write_json, write_paid +from trials import TrialRow + +ASSETS = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "assets") +CHOICES = ("stop", "rerun", "full") + + +def fill_page(template: str, values: Dict[str, str], out: str) -> None: + with open(os.path.join(ASSETS, template), encoding="utf-8") as fh: + page = fh.read() + for key, value in values.items(): + page = page.replace("{{" + key + "}}", value) + left = sorted(set(re.findall(r"\{\{[A-Z_]+\}\}", page))) + if left: + raise LoopError(f"{template}: placeholders not filled: {left}") + with open(out, "w", encoding="utf-8", newline="\n") as fh: + fh.write(page) + + +def cell(text: str) -> str: + return text.replace("|", "\\|").replace("\n", " ") + + +def table_text(t: Dict[str, int]) -> str: + return f"{t['done_pass']} / {t['done_fail']} / {t['not_done_pass']} / {t['not_done_fail']}" + + +def candidate_row(cand: dict, checks: Optional[dict]) -> List[str]: + pred = cand["prediction"] + explains = f"{len(pred['trials'])} trials on {len(pred['tasks'])} tasks ({cand['rule']})" + if checks is None: + return [cand["id"], cand["cause"], explains, "not checked", "not checked", "not checked", "not checked", ""] + test, suite, replay, diff = checks["test"], checks["suite"], checks["replay"], checks["diff"] + if not test["ran"]: + test_text = "did not run" + else: + test_text = {(True, True): "fails, then passes", (False, True): "passes before the fix too", + (True, False): "still fails after the fix", (False, False): "passes before, fails after"}[ + (test["fails_before"], test["passes_after"])] + suite_text = ("did not run" if not suite["ran"] else + ", ".join(suite["added"]) if suite["added"] else "none") + if replay["ran"]: + replay_text = f"{table_text(replay['before'])} → {table_text(replay['after'])}" + false_text = f"{replay['false_success_added']:+d}" + ( + f" (new: {', '.join(replay['false_success_new'])})" if replay["false_success_new"] else "") + else: + replay_text, false_text = "did not run", "did not run" + diff_text = f"{diff['files']} files, +{diff['added']} −{diff['removed']}" + return [cand["id"], cand["cause"], explains, test_text, suite_text, replay_text, false_text, diff_text] + + +def decide_one(args: argparse.Namespace, round_dir: str, record: dict) -> None: + base = os.path.join(round_dir, "candidates") + ids = sorted(d for d in os.listdir(base) if os.path.isfile(os.path.join(base, d, "candidate.json"))) \ + if os.path.isdir(base) else [] + if not ids: + raise LoopError("no candidate is registered: add one with loop.py candidate, then loop.py check") + rows, not_run = [], [] + for cid in ids: + cand = read_json(os.path.join(base, cid, "candidate.json")) + checks_path = os.path.join(base, cid, "checks.json") + checks = read_json(checks_path) if os.path.isfile(checks_path) else None + rows.append("| " + " | ".join(cell(c) for c in candidate_row(cand, checks)) + " |") + if checks is None: + not_run.append(f"- {cid}: not checked; run loop.py check --candidate {cid}") + continue + for name in ("test", "suite", "replay"): + if not checks[name]["ran"]: + not_run.append(f"- {cid}, {name}: {checks[name]['reason']}") + trials = len(read_rows(record["run"]["job_dir"], float(record["pass_threshold"]))) + fill_page("decision-1.md", { + "ROUND": os.path.basename(round_dir), "LOOP": args.loop, "TRIALS": str(trials), + "SUMMARY": f"{len(ids)} candidates, checked against the run {record['run']['job_dir']}.", + "ROWS": "\n".join(rows), "NOT_RUN": "\n".join(not_run) or "Every check ran.", + }, os.path.join(round_dir, "decision-1.md")) + print(f"wrote {round_dir}/decision-1.md: {len(ids)} candidates, {len(not_run)} checks that did not run") + + +def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: + adapter = adapter_of(args.loop) + ledger = load_ledger(args.loop) + entry = ledger_round(ledger, os.path.basename(round_dir)) + selection = entry.get("selection") + if not selection: + raise LoopError("no trial run was selected: run loop.py select with --size first") + threshold = float(record["pass_threshold"]) + checked = health("trial", args.trial_job, threshold) + if checked.refused: + raise LoopError("the trial run is not analysable:\n " + "\n ".join(checked.refused)) + if checked.agents != [adapter.agent]: + raise LoopError(f"adapter.json is for the agent {adapter.agent!r}, and the trial run names " + + ", ".join(repr(a) for a in checked.agents)) + job_dir = os.path.abspath(args.trial_job) + rows: List[TrialRow] = read_rows(job_dir, threshold) + before = read_rows(record["run"]["job_dir"], threshold) + targets = targets_of(selection, before, rows) + of = lambda tasks: passes([r for r in rows if r.task in set(tasks)]) + costs = [r.cost_usd for r in rows] + spent = sum(c for c in costs if c is not None) + per_trial = spent / len(rows) if rows else None + left = float(record["budget_usd"]) - spent + rerun_tasks = selection["targets"] + selection["guards"] + full_tasks = sorted({r.task for r in before}) + options = [{"name": "stop", "what": "nothing", "usd": 0.0, "command": None}] + for name, tasks, repeat in (("rerun", rerun_tasks, TARGET_TRIALS), ("full", full_tasks, 1)): + units = os.path.join(round_dir, f"{name}-tasks.txt") + with open(units, "w", encoding="utf-8", newline="\n") as fh: + fh.write("".join(t + "\n" for t in tasks)) + command = None + line = adapter.commands["run_tasks"] + if line is not None: + build = read_json(os.path.join(round_dir, "build.json")) + command = write_paid(round_dir, f"run_tasks-{name}", line, { + "units_file": units, "repeat": str(repeat), "jobs_dir": os.path.join(round_dir, "jobs"), + "budget_usd": f"{max(left, 0.0):.2f}", "build": build["path"] or "", "ref": selection["ref"]}, + adapter.checkout) + what = (f"the {len(tasks)} targets and guards again, {repeat} trials each" if name == "rerun" + else f"a full pass of the {len(tasks)} tasks of the run, {repeat} trial each") + options.append({"name": name, "what": what, "usd": len(tasks) * repeat * per_trial if per_trial is not None else None, + "command": command}) + build = read_json(os.path.join(round_dir, "build.json")) + trial_record = { + "schema": "run-to-fix.trial/v1", "created": now(), "job_dir": job_dir, "trials": len(rows), + "cost_usd": spent, "cost_complete": all(c is not None for c in costs), "cost_per_trial_usd": per_trial, + "budget_usd": float(record["budget_usd"]), "budget_left_usd": left, "targets": targets, + "guards": of(selection["guards"]), "canaries": of(selection["canaries"]), "options": options, + "build": build, + } + write_json(os.path.join(round_dir, "trial.json"), trial_record) + share = lambda x: f"{x['passed']} of {x['trials']}" if x else "none" + results = [f"| targets ({targets['tasks']}) | {share(targets['before'])} | {share(targets['after'])} | " + f"{targets['verdict'] or 'none'}, p = {targets['p']:.3g} |" if targets["p"] is not None else + "| targets | none | none | none |"] + for name in ("guards", "canaries"): + results.append(f"| {name} ({len(selection[name])}) | | {share(trial_record[name])} | |") + option_rows = [f"| {o['name']} | {o['what']} | {usd(o['usd']) if o['usd'] is not None else 'unknown'} | " + f"{cell(o['command']) if o['command'] else 'none'} |" for o in options] + fill_page("decision-2.md", { + "ROUND": os.path.basename(round_dir), "LOOP": args.loop, + "BUILD": f"{build['path']} from {build['ref'][:12]}" if build["path"] else f"{build['reason']} ({build['ref'][:12]})", + "TRIAL_JOB": job_dir, "TRIALS": str(len(rows)), + "SPENT": usd(spent) + ("" if trial_record["cost_complete"] else " at least: some trials report no cost"), + "LEFT": usd(left), "BUDGET": usd(float(record["budget_usd"])), "RESULTS": "\n".join(results), + "OPTIONS": "\n".join(option_rows), "PER_TRIAL": usd(per_trial) if per_trial is not None else "unknown", + }, os.path.join(round_dir, "decision-2.md")) + print(f"wrote {round_dir}: trial.json, decision-2.md; budget left {usd(left)}") + for o in options: + print(f" {o['name']}: {o['what']}, " + (usd(o["usd"]) if o["usd"] is not None else "price unknown")) + + +def cmd_decide(args: argparse.Namespace) -> None: + round_dir = open_round(args.loop) + record = read_input(round_dir) + if args.stop == 1: + decide_one(args, round_dir, record) + elif args.trial_job is None: + raise LoopError("decide --stop 2 needs --trial-job: the job folder of the trial run") + else: + decide_two(args, round_dir, record) + + +def cmd_answer(args: argparse.Namespace) -> None: + round_dir = open_round(args.loop) + page = os.path.join(round_dir, f"decision-{args.stop}.md") + if not os.path.isfile(page): + raise LoopError(f"{page} is missing: write it with loop.py decide --stop {args.stop} before the answer") + if not args.text.strip(): + raise LoopError("--text holds the person's words and cannot be empty") + ledger = load_ledger(args.loop) + entry = ledger_round(ledger, os.path.basename(round_dir)) + if args.stop == 1: + if args.keep is None: + raise LoopError("answer --stop 1 needs --keep: candidate ids separated by commas, or none") + keep = [] if args.keep == "none" else [k.strip() for k in args.keep.split(",") if k.strip()] + missing = [k for k in keep if not os.path.isfile(os.path.join(round_dir, "candidates", k, "checks.json"))] + if missing: + raise LoopError(f"these candidates are not checked: {', '.join(missing)}") + entry["answers"].append({"about": "stop 1: the candidates to keep", "date": now(), "text": args.text, + "keep": keep}) + entry["kept"] = keep + entry["predictions"] = [] + for k in keep: + cand = read_json(os.path.join(round_dir, "candidates", k, "candidate.json")) + entry["predictions"].append({"candidate": k, "cause": cand["cause"], "rule": cand["rule"], + "tasks": cand["prediction"]["tasks"], "expect": "pass"}) + else: + if args.choice not in CHOICES: + raise LoopError(f"answer --stop 2 needs --choice, one of {', '.join(CHOICES)}") + entry["answers"].append({"about": "stop 2: what to run next", "date": now(), "text": args.text, + "choice": args.choice}) + entry["next"] = args.choice + save_ledger(args.loop, ledger) + print(f"recorded the answer of stop {args.stop} in ledger.json") diff --git a/.claude/skills/run-to-fix/scripts/loop.py b/.claude/skills/run-to-fix/scripts/loop.py index a5c6a461..efe7e5c5 100644 --- a/.claude/skills/run-to-fix/scripts/loop.py +++ b/.claude/skills/run-to-fix/scripts/loop.py @@ -3,6 +3,11 @@ read stage 1: health and metrics of the run and its baseline; refuses a run that is not analysable classify stage 2: one class and signature per trial, groups and their warnings + candidate stage 3: register a fix candidate of a cause + check stage 3: the candidate's test, the agent's suite and the replay, each in a worktree + decide stop 1 or stop 2: write the decision page + answer record the person's answer of a stop + select stage 4: the tasks of the trial run and their price; with --size, the build and the run command compare stage 5: a new run against the round's run or baseline, the verdict, the stop rule Every file is written under the loop folder; see references/files.md. @@ -14,11 +19,14 @@ from typing import List from adapter import AdapterError +from checking import cmd_candidate, cmd_check from classifying import cmd_classify from comparing import cmd_compare +from deciding import cmd_answer, cmd_decide from found import RuleError from reading import cmd_read from rounds import LoopError +from selecting import cmd_select from trials import TrialError @@ -33,11 +41,45 @@ def main(argv: List[str]) -> int: rd.add_argument("--pass-threshold", type=float, default=1.0, help="reward at or above this is a pass (default 1)") rd.add_argument("--continue-after-stop", help="the person's words to start a round after a stop rule said stop") rd.set_defaults(func=cmd_read) - cl = sub.add_parser("classify", help="stage 2: classes, signatures and groups of the open round") + timeout = argparse.ArgumentParser(add_help=False) + timeout.add_argument("--command-timeout-s", type=float, default=3600.0, + help="an adapter command still running after this many seconds is stopped (default 3600)") + cl = sub.add_parser("classify", parents=[timeout], help="stage 2: classes, signatures and groups of the open round") cl.add_argument("--loop", required=True) - cl.add_argument("--command-timeout-s", type=float, default=3600.0, - help="an adapter command still running after this many seconds is stopped (default 3600)") cl.set_defaults(func=cmd_classify) + ca = sub.add_parser("candidate", help="stage 3: register a fix candidate") + ca.add_argument("--loop", required=True) + ca.add_argument("--id", required=True, help="the candidate's id, as in candidates//") + ca.add_argument("--cause", required=True, help="the cause it fixes, as in causes/.md") + ca.add_argument("--ref", required=True, help="the commit of the fix in the agent's checkout") + ca.add_argument("--test", help="the test file of the cause, relative to the checkout") + ca.add_argument("--test-ref", help="the commit that adds the test, before the fix") + ca.set_defaults(func=cmd_candidate) + ch = sub.add_parser("check", parents=[timeout], help="stage 3: check a candidate") + ch.add_argument("--loop", required=True) + ch.add_argument("--candidate", required=True) + ch.set_defaults(func=cmd_check) + de = sub.add_parser("decide", help="write the page of stop 1 or stop 2") + de.add_argument("--loop", required=True) + de.add_argument("--stop", type=int, required=True, choices=(1, 2)) + de.add_argument("--trial-job", help="stop 2: the job folder of the trial run") + de.set_defaults(func=cmd_decide) + an = sub.add_parser("answer", help="record the person's answer of a stop") + an.add_argument("--loop", required=True) + an.add_argument("--stop", type=int, required=True, choices=(1, 2)) + an.add_argument("--text", required=True, help="the person's words") + an.add_argument("--keep", help="stop 1: candidate ids separated by commas, or none") + an.add_argument("--choice", help="stop 2: stop, rerun or full") + an.set_defaults(func=cmd_answer) + se = sub.add_parser("select", parents=[timeout], help="stage 4: propose and choose the trial run") + se.add_argument("--loop", required=True) + se.add_argument("--ref", required=True, help="the commit to try; it contains every kept candidate") + se.add_argument("--canaries", type=int, required=True, help="how many passing tasks to draw as canaries") + se.add_argument("--size", choices=("targets", "guards", "canaries"), + help="the person's choice: targets, targets and guards, or all three") + se.add_argument("--budget-usd", type=float, help="the budget the person named for the trial run") + se.add_argument("--text", help="the person's words") + se.set_defaults(func=cmd_select) cp = sub.add_parser("compare", help="stage 5: compare a new run and close the round") cp.add_argument("--loop", required=True) cp.add_argument("--new", required=True, help="the job folder of the new run, as for read --run") diff --git a/.claude/skills/run-to-fix/scripts/selecting.py b/.claude/skills/run-to-fix/scripts/selecting.py new file mode 100644 index 00000000..aee91388 --- /dev/null +++ b/.claude/skills/run-to-fix/scripts/selecting.py @@ -0,0 +1,135 @@ +"""Stage 4, `loop.py select`: the tasks of the trial run and their price, the build, the written run command.""" +from __future__ import annotations + +import argparse +import hashlib +import os +import random +from typing import Dict, List, Optional, Tuple + +from adapter import Adapter +from checkout import is_ancestor, resolve, worktree, write_paid_on_ref +from classifying import read_rows +from comparing import TARGET_TRIALS, usd +from reading import adapter_of +from rounds import (LoopError, ledger_round, load_ledger, now, open_round, read_input, read_json, run_command, + save_ledger, write_json, write_paid) +from trials import solved_tasks + +#: canaries are drawn with this seed, so the same round proposes the same canaries +SEED = 20261117 +SIZES = ("targets", "guards", "canaries") + + +def kept_candidates(round_dir: str, entry: dict) -> List[dict]: + if entry.get("kept") is None: + raise LoopError("stop 1 has no answer yet: record it with loop.py answer --stop 1") + if not entry["kept"]: + raise LoopError("the answer of stop 1 kept no candidate, so there is nothing to try") + return [dict(read_json(os.path.join(round_dir, "candidates", cid, "checks.json")), id=cid) for cid in entry["kept"]] + + +def guards_of(kept: List[dict], rows_by_trial: Dict[str, Tuple[str, bool]], targets: List[str]) -> Tuple[List[str], Optional[str]]: + touched: Optional[set] = None + for checks in kept: + replay = checks["replay"] + if replay["ran"] and replay["touched"] is not None: + touched = (touched or set()) | set(replay["touched"]) + if touched is None: + return [], "no replay reports which trials ran the changed code" + tasks = {rows_by_trial[t][0] for t in touched if t in rows_by_trial and rows_by_trial[t][1]} + return sorted(tasks - set(targets)), None + + +def build_on(adapter: Adapter, loop_dir: str, round_dir: str, sha: str, timeout_s: float) -> dict: + line = adapter.commands["build"] + if line is None: + return {"ref": sha, "path": None, "sha256": None, "reason": "commands.build is null: the run installs the checkout"} + if "build" in adapter.paid: + path = os.path.join(round_dir, "commands", f"build-{sha[:12]}.sh") + write_paid_on_ref(path, adapter.checkout, loop_dir, sha, line, {}) + return {"ref": sha, "path": None, "sha256": None, + "reason": f"commands.build costs money: written to {path}, not started"} + log = os.path.join(round_dir, "logs", "build.log") + with worktree(adapter.checkout, loop_dir, sha) as tree: + ran = run_command(line, {"worktree": tree}, tree, log, timeout_s) + if ran.code != 0: + raise LoopError(f"commands.build exited with code {ran.code}; the log is {log}:\n{ran.tail}") + with open(log, encoding="utf-8", errors="replace") as fh: + lines = [ln.strip() for ln in fh.read().splitlines()[1:] if ln.strip()] + built = lines[-1] if lines else "" + inside = os.path.abspath(built).startswith(os.path.abspath(tree) + os.sep) + if not built or not os.path.isabs(built) or inside or not os.path.exists(built): + raise LoopError(f"commands.build must print, as its last line, the absolute path of what it built, outside " + f"{{worktree}}; it printed {built!r}") + digest = None + if os.path.isfile(built): + with open(built, "rb") as fh: + digest = hashlib.sha256(fh.read()).hexdigest() + return {"ref": sha, "path": built, "sha256": digest, "reason": None} + + +def cmd_select(args: argparse.Namespace) -> None: + adapter = adapter_of(args.loop) + round_dir = open_round(args.loop) + record = read_input(round_dir) + ledger = load_ledger(args.loop) + entry = ledger_round(ledger, os.path.basename(round_dir)) + kept = kept_candidates(round_dir, entry) + ref = resolve(adapter.checkout, args.ref) + for checks in kept: + if not is_ancestor(adapter.checkout, checks["ref"], ref): + raise LoopError(f"{args.ref} does not contain the kept candidate {checks['id']} ({checks['ref'][:12]}): " + "merge the kept candidates into one commit of the work branch and select that") + + rows = read_rows(record["run"]["job_dir"], float(record["pass_threshold"])) + solved, _ = solved_tasks(rows) + targets = sorted({t for p in entry["predictions"] for t in p["tasks"]}) + guards, no_guards = guards_of(kept, {r.trial: (r.task, r.passed) for r in rows}, targets) + pool = sorted(t for t, ok in solved.items() if ok and t not in targets and t not in guards) + canaries = sorted(random.Random(SEED).sample(pool, min(args.canaries, len(pool)))) + cost = read_json(os.path.join(round_dir, "metrics.json")) + per_trial = next(r for r in cost["runs"] if r["label"] == "run")["cost_mean"] + sizes = {"targets": targets, "guards": targets + guards, "canaries": targets + guards + canaries} + names = {"targets": "targets", "guards": "targets and guards", "canaries": "targets, guards and canaries"} + price = {k: (len(v) * TARGET_TRIALS * per_trial if per_trial is not None else None) for k, v in sizes.items()} + + print(f"targets ({len(targets)} tasks of the kept causes): {', '.join(targets)}") + print(f"guards: {', '.join(guards)}" if guards else f"guards: none ({no_guards or 'no passing task ran the changed code'})") + print(f"canaries ({len(canaries)} passing tasks, seed {SEED}): {', '.join(canaries) or 'none'}") + for k in SIZES: + n = len(sizes[k]) + print(f"{names[k]}: {n} tasks, {n * TARGET_TRIALS} trials, " + + (usd(price[k]) if price[k] is not None else "price unknown: the run reports no cost")) + if args.size is None: + print("choose a size with --size targets|guards|canaries --budget-usd B --text \"the person's words\"") + return + + if args.budget_usd is None or not args.text: + raise LoopError("--size needs --budget-usd and --text: the budget and the person's words") + left = float(record["budget_usd"]) + if args.budget_usd > left: + raise LoopError(f"the budget of the trial run, {usd(args.budget_usd)}, is over the round's budget, {usd(left)}") + if price[args.size] is not None and price[args.size] > args.budget_usd: + raise LoopError(f"{names[args.size]} cost {usd(price[args.size])}, over the budget of {usd(args.budget_usd)}") + tasks = sizes[args.size] + units = os.path.join(round_dir, "trial-tasks.txt") + with open(units, "w", encoding="utf-8", newline="\n") as fh: + fh.write("".join(t + "\n" for t in tasks)) + build = build_on(adapter, args.loop, round_dir, ref, args.command_timeout_s) + write_json(os.path.join(round_dir, "build.json"), build) + entry["selection"] = {"ref": ref, "size": args.size, "targets": targets, + "guards": guards if args.size != "targets" else [], + "canaries": canaries if args.size == "canaries" else [], "repeat": TARGET_TRIALS, + "seed": SEED, "price_usd": price[args.size], "budget_usd": args.budget_usd, + "guards_reason": no_guards} + entry["answers"].append({"about": "the trial run: its size and budget", "date": now(), "text": args.text}) + line = adapter.commands["run_tasks"] + if line is None: + written = "commands.run_tasks is null: start the run of trial-tasks.txt by other means" + else: + values = {"units_file": units, "repeat": str(TARGET_TRIALS), "jobs_dir": os.path.join(round_dir, "jobs"), + "budget_usd": str(args.budget_usd), "build": build["path"] or "", "ref": ref} + written = "start it: " + write_paid(round_dir, "run_tasks", line, values, adapter.checkout) + save_ledger(args.loop, ledger) + print(f"wrote {round_dir}: trial-tasks.txt ({len(tasks)} tasks), build.json; {written}") From 4c0aa8e6216730b69a15d9c0ed31dc27be58b761 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:39:34 +0100 Subject: [PATCH 11/92] test(run-to-fix): count the trial run in the money a round has spent --- .claude/skills/run-to-fix/tests/test_decide.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index 08833907..d9db75b9 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -209,3 +209,15 @@ def test_compare_reads_the_selection_and_the_predictions(self) -> None: result = load(self.round_file("result.json")) self.assertEqual(result["targets"]["after"], {"trials": 6, "passed": 5}) self.assertEqual(result["predictions"][0]["result"], {"trials": 6, "passed": 5, "tasks": 2}) + + def test_the_stop_rule_counts_the_trial_run_and_the_new_run(self) -> None: + job = self.chosen() + run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job]) + run(["answer", "--loop", self.loop, "--stop", "2", "--choice", "full", "--text", "a full pass"]) + full = write_job(self.jobs, "full", "alpha", "harbor", + [trial(f"{t}__f", f"bench/{t}", 1.0) for t in ("p1", "p2", "p3", "f1", "f2", "f3")]) + r = run(["compare", "--loop", self.loop, "--new", full, "--against", "run"]) + self.assertEqual(r.code, 0, r.out + r.err) + rule = load(self.round_file("result.json"))["stop_rule"] + self.assertAlmostEqual(rule["spent_usd"], 0.09 + 0.06) + self.assertAlmostEqual(rule["left_usd"], 10 - 0.15) From b2070e77a9a325c819a56630f8790adf47d0ced0 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:43:12 +0100 Subject: [PATCH 12/92] fix(run-to-fix): count the trial run in the money a round has spent --- .claude/skills/run-to-fix/scripts/comparing.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/scripts/comparing.py b/.claude/skills/run-to-fix/scripts/comparing.py index 28062f97..79193e40 100644 --- a/.claude/skills/run-to-fix/scripts/comparing.py +++ b/.claude/skills/run-to-fix/scripts/comparing.py @@ -57,6 +57,22 @@ def predictions_of(predictions: List[dict], new: List[TrialRow]) -> List[dict]: return out +def spent_in_round(round_dir: str, record: dict, new_dir: str, new_cost: Optional[float]) -> Optional[float]: + """What the round's runs cost: its trial run, when there was one, and the new run unless it is the round's own + run. None when a run reports no cost.""" + total = 0.0 + trial_path = os.path.join(round_dir, "trial.json") + if os.path.isfile(trial_path): + trial_run = read_json(trial_path) + if trial_run["job_dir"] != new_dir: + total += trial_run["cost_usd"] + if new_dir != record["run"]["job_dir"]: + if new_cost is None: + return None + total += new_cost + return total + + def stop_rule(result_verdict: str, groups: List[dict], budget: float, spent: Optional[float], cost_per_trial: Optional[float]) -> dict: big = [g for g in groups if g["trials"] >= GROUP_MIN_TRIALS] @@ -112,7 +128,7 @@ def cmd_compare(args: argparse.Namespace) -> None: entry = ledger_round(ledger, os.path.basename(round_dir)) classes, signatures = classes_of(new_rows, adapter.limits) groups = failure_groups(new_rows, classes, signatures, None) - spent = 0.0 if new_dir == record["run"]["job_dir"] else runs["new"]["cost_total"] + spent = spent_in_round(round_dir, record, new_dir, runs["new"]["cost_total"]) rule = stop_rule(result_verdict, groups, float(record["budget_usd"]), spent, runs["new"]["cost_mean"]) result = { "schema": "run-to-fix.result/v1", "created": now(), "round": os.path.basename(round_dir), From 045b6a7e3c92c7fa53725e72e2f205b87b1f56b3 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 19:43:12 +0100 Subject: [PATCH 13/92] docs(run-to-fix): say that the run of tasks is always written for a person --- .claude/skills/run-to-fix/references/adapter.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/references/adapter.md b/.claude/skills/run-to-fix/references/adapter.md index 2002a56d..20d669b9 100644 --- a/.claude/skills/run-to-fix/references/adapter.md +++ b/.claude/skills/run-to-fix/references/adapter.md @@ -75,7 +75,8 @@ means. 3. A command named in `paid` is never started by the loop. `loop.py` writes it, with every placeholder filled, to `round-NN/commands/.sh` and stops. A person starts it, and the loop goes on from the job folder it produced. The written line is complete as text: the task list, the repeats, the job folder and the budget are all - in it. + in it. `commands.run_tasks` is always written this way, whether `paid` names it or not, because a run of tasks + calls the model. 4. A null command switches its check off. The decision page then says that the check did not run, and which evidence is missing because of it. 5. The loop writes the output of every command to a file under `round-NN/`. A command that has not From efd58daec994b9370c0ce6f48cd77069261f6512 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:46:05 +0100 Subject: [PATCH 14/92] test(run-to-fix): count every trial run of a round in money and in targets --- .../skills/run-to-fix/tests/test_decide.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index d9db75b9..0c4925a7 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -221,3 +221,64 @@ def test_the_stop_rule_counts_the_trial_run_and_the_new_run(self) -> None: rule = load(self.round_file("result.json"))["stop_rule"] self.assertAlmostEqual(rule["spent_usd"], 0.09 + 0.06) self.assertAlmostEqual(rule["left_usd"], 10 - 0.15) + + +class Reruns(Flow): + """A rerun of stop 2 is a second trial run of the round: its money and its trials add to the first.""" + + def rerun_job(self) -> str: + trials = [trial(f"{task}-{k}__u", f"bench/{task}", 1.0) for task in ("f1", "f2", "p1") for k in range(3)] + return write_job(self.jobs, "rerun", "alpha", "harbor", trials) + + def after_rerun(self) -> str: + first = self.chosen() + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", first]) + self.assertEqual(r.code, 0, r.out + r.err) + run(["answer", "--loop", self.loop, "--stop", "2", "--choice", "rerun", "--text", "run them again"]) + second = self.rerun_job() + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", second]) + self.assertEqual(r.code, 0, r.out + r.err) + return second + + def test_the_second_decision_counts_both_trial_runs(self) -> None: + self.after_rerun() + record = load(self.round_file("trial.json")) + self.assertAlmostEqual(record["cost_usd"], 0.18) + self.assertAlmostEqual(record["budget_left_usd"], 9.82) + self.assertEqual(record["targets"]["after"], {"trials": 12, "passed": 11}) + self.assertEqual(record["guards"], {"trials": 6, "passed": 6}) + runs = load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["trial_runs"] + self.assertEqual([os.path.basename(t["job_dir"]) for t in runs], ["trial", "rerun"]) + self.assertIn("$9.82", self.page("decision-2.md")) + self.assertIn("11 of 12", self.page("decision-2.md")) + with open(self.round_file("commands/run_tasks-full.sh"), encoding="utf-8") as fh: + self.assertIn("--cap 9.82", fh.read()) + + def test_deciding_twice_on_one_trial_run_counts_it_once(self) -> None: + first = self.chosen() + for _ in range(2): + run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", first]) + self.assertEqual(len(load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["trial_runs"]), 1) + self.assertAlmostEqual(load(self.round_file("trial.json"))["cost_usd"], 0.09) + + def test_select_after_a_trial_run_checks_the_budget_left(self) -> None: + first = self.chosen() + run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", first]) + r = self.select(["--size", "targets", "--budget-usd", "9.95", "--text", "again"]) + self.assertNotEqual(r.code, 0) + self.assertIn("$9.91", r.err) + + def test_compare_counts_every_trial_run(self) -> None: + self.after_rerun() + full = write_job(self.jobs, "full", "alpha", "harbor", + [trial(f"{t}__f", f"bench/{t}", 1.0) for t in ("p1", "p2", "p3", "f1", "f2", "f3")]) + r = run(["compare", "--loop", self.loop, "--new", full, "--against", "run"]) + self.assertEqual(r.code, 0, r.out + r.err) + rule = load(self.round_file("result.json"))["stop_rule"] + self.assertAlmostEqual(rule["spent_usd"], 0.09 + 0.09 + 0.06) + + def test_compare_of_a_trial_run_counts_it_once(self) -> None: + second = self.after_rerun() + r = run(["compare", "--loop", self.loop, "--new", second, "--against", "run"]) + self.assertEqual(r.code, 0, r.out + r.err) + self.assertAlmostEqual(load(self.round_file("result.json"))["stop_rule"]["spent_usd"], 0.18) From b039633eca9d929108371b83b0daa7ed7c73c9f2 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:51:19 +0100 Subject: [PATCH 15/92] fix(run-to-fix): count every trial run of a round decide --stop 2 adds each trial job to the round's list in the ledger. The money spent in select, decide and compare comes from that list, and the exact test of the targets reads the trials of every trial run on the same code. --- .../skills/run-to-fix/assets/decision-2.md | 3 +- .../skills/run-to-fix/scripts/comparing.py | 23 +++++++-------- .claude/skills/run-to-fix/scripts/deciding.py | 28 +++++++++++++------ .claude/skills/run-to-fix/scripts/rounds.py | 3 +- .../skills/run-to-fix/scripts/selecting.py | 7 +++-- 5 files changed, 40 insertions(+), 24 deletions(-) diff --git a/.claude/skills/run-to-fix/assets/decision-2.md b/.claude/skills/run-to-fix/assets/decision-2.md index b8726af9..7d4d32c2 100644 --- a/.claude/skills/run-to-fix/assets/decision-2.md +++ b/.claude/skills/run-to-fix/assets/decision-2.md @@ -1,6 +1,7 @@ # Stop 2 of {{ROUND}}: what to run next -Build: {{BUILD}}. Trial run: {{TRIAL_JOB}}, {{TRIALS}} trials, {{SPENT}}. Budget left: {{LEFT}} of {{BUDGET}}. +Build: {{BUILD}}. Trial runs on it: {{TRIAL_JOB}}, {{TRIALS}} trials. Spent in this round: {{SPENT}}. Budget left: +{{LEFT}} of {{BUDGET}}. | tasks | trials passed before | trials passed in the trial run | test | |---|---|---|---| diff --git a/.claude/skills/run-to-fix/scripts/comparing.py b/.claude/skills/run-to-fix/scripts/comparing.py index 79193e40..17ef0482 100644 --- a/.claude/skills/run-to-fix/scripts/comparing.py +++ b/.claude/skills/run-to-fix/scripts/comparing.py @@ -57,16 +57,17 @@ def predictions_of(predictions: List[dict], new: List[TrialRow]) -> List[dict]: return out -def spent_in_round(round_dir: str, record: dict, new_dir: str, new_cost: Optional[float]) -> Optional[float]: - """What the round's runs cost: its trial run, when there was one, and the new run unless it is the round's own - run. None when a run reports no cost.""" - total = 0.0 - trial_path = os.path.join(round_dir, "trial.json") - if os.path.isfile(trial_path): - trial_run = read_json(trial_path) - if trial_run["job_dir"] != new_dir: - total += trial_run["cost_usd"] - if new_dir != record["run"]["job_dir"]: +def trial_spent(entry: dict) -> float: + """What the round's trial runs cost, from the list of them in the ledger.""" + return sum(t["cost_usd"] for t in entry.get("trial_runs") or []) + + +def spent_in_round(entry: dict, record: dict, new_dir: str, new_cost: Optional[float]) -> Optional[float]: + """What the round's runs cost: every trial run, and the new run unless it is one of them or the round's own + run. None when the new run reports no cost.""" + total = trial_spent(entry) + known = {t["job_dir"] for t in entry.get("trial_runs") or []} | {record["run"]["job_dir"]} + if new_dir not in known: if new_cost is None: return None total += new_cost @@ -128,7 +129,7 @@ def cmd_compare(args: argparse.Namespace) -> None: entry = ledger_round(ledger, os.path.basename(round_dir)) classes, signatures = classes_of(new_rows, adapter.limits) groups = failure_groups(new_rows, classes, signatures, None) - spent = spent_in_round(round_dir, record, new_dir, runs["new"]["cost_total"]) + spent = spent_in_round(entry, record, new_dir, runs["new"]["cost_total"]) rule = stop_rule(result_verdict, groups, float(record["budget_usd"]), spent, runs["new"]["cost_mean"]) result = { "schema": "run-to-fix.result/v1", "created": now(), "round": os.path.basename(round_dir), diff --git a/.claude/skills/run-to-fix/scripts/deciding.py b/.claude/skills/run-to-fix/scripts/deciding.py index 835df220..df513668 100644 --- a/.claude/skills/run-to-fix/scripts/deciding.py +++ b/.claude/skills/run-to-fix/scripts/deciding.py @@ -7,7 +7,7 @@ from typing import Dict, List, Optional from classifying import read_rows -from comparing import TARGET_TRIALS, passes, targets_of, usd +from comparing import TARGET_TRIALS, passes, targets_of, trial_spent, usd from reading import adapter_of, health from rounds import LoopError, ledger_round, load_ledger, now, open_round, read_input, read_json, save_ledger, write_json, write_paid from trials import TrialRow @@ -102,13 +102,23 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: raise LoopError(f"adapter.json is for the agent {adapter.agent!r}, and the trial run names " + ", ".join(repr(a) for a in checked.agents)) job_dir = os.path.abspath(args.trial_job) - rows: List[TrialRow] = read_rows(job_dir, threshold) + this_run: List[TrialRow] = read_rows(job_dir, threshold) + costs = [r.cost_usd for r in this_run] + runs = [t for t in entry.setdefault("trial_runs", []) if t["job_dir"] != job_dir] + runs.append({"job_dir": job_dir, "ref": selection["ref"], "trials": len(this_run), + "cost_usd": sum(c for c in costs if c is not None), "cost_complete": all(c is not None for c in costs), + "added": now()}) + entry["trial_runs"] = runs + save_ledger(args.loop, ledger) + # every trial run of this round on the same code adds its trials to the exact test of the targets + same = [t for t in runs if t["ref"] == selection["ref"]] + rows: List[TrialRow] = [r for t in same for r in read_rows(t["job_dir"], threshold)] before = read_rows(record["run"]["job_dir"], threshold) targets = targets_of(selection, before, rows) of = lambda tasks: passes([r for r in rows if r.task in set(tasks)]) - costs = [r.cost_usd for r in rows] - spent = sum(c for c in costs if c is not None) - per_trial = spent / len(rows) if rows else None + spent = trial_spent(entry) + cost_on_ref = sum(t["cost_usd"] for t in same) + per_trial = cost_on_ref / len(rows) if rows else None left = float(record["budget_usd"]) - spent rerun_tasks = selection["targets"] + selection["guards"] full_tasks = sorted({r.task for r in before}) @@ -131,8 +141,10 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: "command": command}) build = read_json(os.path.join(round_dir, "build.json")) trial_record = { - "schema": "run-to-fix.trial/v1", "created": now(), "job_dir": job_dir, "trials": len(rows), - "cost_usd": spent, "cost_complete": all(c is not None for c in costs), "cost_per_trial_usd": per_trial, + "schema": "run-to-fix.trial/v1", "created": now(), "job_dir": job_dir, + "counts": "cost_usd is every trial run of the round; trials and the results are the trial runs on this ref", + "trial_runs": runs, "trials": len(rows), "cost_usd": spent, + "cost_complete": all(t["cost_complete"] for t in runs), "cost_per_trial_usd": per_trial, "budget_usd": float(record["budget_usd"]), "budget_left_usd": left, "targets": targets, "guards": of(selection["guards"]), "canaries": of(selection["canaries"]), "options": options, "build": build, @@ -149,7 +161,7 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: fill_page("decision-2.md", { "ROUND": os.path.basename(round_dir), "LOOP": args.loop, "BUILD": f"{build['path']} from {build['ref'][:12]}" if build["path"] else f"{build['reason']} ({build['ref'][:12]})", - "TRIAL_JOB": job_dir, "TRIALS": str(len(rows)), + "TRIAL_JOB": ", ".join(t["job_dir"] for t in same), "TRIALS": str(len(rows)), "SPENT": usd(spent) + ("" if trial_record["cost_complete"] else " at least: some trials report no cost"), "LEFT": usd(left), "BUDGET": usd(float(record["budget_usd"])), "RESULTS": "\n".join(results), "OPTIONS": "\n".join(option_rows), "PER_TRIAL": usd(per_trial) if per_trial is not None else "unknown", diff --git a/.claude/skills/run-to-fix/scripts/rounds.py b/.claude/skills/run-to-fix/scripts/rounds.py index be16d497..bec8e6aa 100644 --- a/.claude/skills/run-to-fix/scripts/rounds.py +++ b/.claude/skills/run-to-fix/scripts/rounds.py @@ -176,7 +176,8 @@ def ledger_round(ledger: dict, name: str) -> dict: for entry in ledger["rounds"]: if entry["round"] == name: return entry - entry = {"round": name, "started": now(), "answers": [], "selection": None, "predictions": [], "result": None} + entry = {"round": name, "started": now(), "answers": [], "selection": None, "predictions": [], "trial_runs": [], + "result": None} ledger["rounds"].append(entry) return entry diff --git a/.claude/skills/run-to-fix/scripts/selecting.py b/.claude/skills/run-to-fix/scripts/selecting.py index aee91388..88f99fcd 100644 --- a/.claude/skills/run-to-fix/scripts/selecting.py +++ b/.claude/skills/run-to-fix/scripts/selecting.py @@ -10,7 +10,7 @@ from adapter import Adapter from checkout import is_ancestor, resolve, worktree, write_paid_on_ref from classifying import read_rows -from comparing import TARGET_TRIALS, usd +from comparing import TARGET_TRIALS, trial_spent, usd from reading import adapter_of from rounds import (LoopError, ledger_round, load_ledger, now, open_round, read_input, read_json, run_command, save_ledger, write_json, write_paid) @@ -107,9 +107,10 @@ def cmd_select(args: argparse.Namespace) -> None: if args.budget_usd is None or not args.text: raise LoopError("--size needs --budget-usd and --text: the budget and the person's words") - left = float(record["budget_usd"]) + left = float(record["budget_usd"]) - trial_spent(entry) if args.budget_usd > left: - raise LoopError(f"the budget of the trial run, {usd(args.budget_usd)}, is over the round's budget, {usd(left)}") + raise LoopError(f"the budget of the trial run, {usd(args.budget_usd)}, is over what the round has left, " + f"{usd(left)} of {usd(float(record['budget_usd']))}") if price[args.size] is not None and price[args.size] > args.budget_usd: raise LoopError(f"{names[args.size]} cost {usd(price[args.size])}, over the budget of {usd(args.budget_usd)}") tasks = sizes[args.size] From e5b6cac46f6f2f759022dc72e62c60fa7f72e1df Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Tue, 17 Nov 2026 23:57:16 +0100 Subject: [PATCH 16/92] docs(run-to-fix): write the procedure, the classes and the files of the loop --- .claude/skills/run-to-fix/SKILL.md | 178 ++++++++++++++++++ .../skills/run-to-fix/references/classes.md | 86 +++++++++ .claude/skills/run-to-fix/references/files.md | 93 +++++++++ 3 files changed, 357 insertions(+) create mode 100644 .claude/skills/run-to-fix/SKILL.md create mode 100644 .claude/skills/run-to-fix/references/classes.md create mode 100644 .claude/skills/run-to-fix/references/files.md diff --git a/.claude/skills/run-to-fix/SKILL.md b/.claude/skills/run-to-fix/SKILL.md new file mode 100644 index 00000000..d1cd76ac --- /dev/null +++ b/.claude/skills/run-to-fix/SKILL.md @@ -0,0 +1,178 @@ +--- +name: run-to-fix +description: Turn a finished benchmark run into checked fix candidates for the agent, as a loop with two stops for a person - classify every trial, find causes backed by counts, prove each cause with a test, check fix candidates on the recorded trials, try the kept fixes on chosen tasks, and compare the new run with exact tests and a stop rule written in advance. Works for any agent and both run paths of Nullius (a Harbor recipe or the standard), with everything that belongs to one agent in an adapter file. Use whenever the user has a finished run and wants to find the agent's errors and fix them, run another round of fixes, decide which fix to keep, or check whether a fix helped. For a one-off analysis or comparison page without fixes, use run-analysis. +--- + +# Run to fix + +Takes a finished run of an agent and turns it into fix candidates that have been checked before any money is spent +on them, then into a new run that says whether the fixes helped. One pass through the five stages below is a round. +A round stops twice and waits for a person: which candidates to keep, and what to run next at what cost. The person +names every budget and starts every command that costs money. + +The work splits as in `run-analysis`. The bundled script counts and keeps the records, all in one folder. It reads +only the files both run paths write. Explaining is done by you: reading trials, naming +causes, writing tests and fixes in the agent's code. + +## What it needs + +- The `run-analysis` skill in a folder beside this one. `loop.py` calls its `runs.py` for `health`, `analyze` and + `render`, and stops with a message naming the missing file when it is absent. +- Python 3 with the standard library, and git. The tests of this skill ran on Python 3.12. +- A loop folder the person names, outside the repository or in a folder git ignores. `loop.py` writes only there. +- An adapter, `adapter.json` in the loop folder: one file per agent with its checkout, its commands and the limits + of the run (`references/adapter.md`). For an agent of the Nullius catalogue, draft it from + `catalogue/agents//agent.json` (`versions..upstream.repository` and `tag`) and the run's settings, and + ask the person to confirm it. Ask the person for every value you cannot read from a file. +- The run: one job folder with one folder per trial. A baseline run to compare it with is optional. + +`L` below is `scripts/loop.py` beside this skill. Every command takes `--loop `. `references/files.md` +lists every file the loop writes and reads. + +## Stage 1. Read the run + +```bash +python3 L read --loop DIR --run JOB [--baseline JOB] --budget-usd B [--pass-threshold 1] +``` + +The person names `B`, the budget of the round. `read` runs `health` and `analyze` of `run-analysis`, writes +`input.json`, `health.txt` and `metrics.json`, and refuses a run that `health` calls not analysable (no trials, one +shared error, an incomplete or unfinished job) or whose trials name another agent than the adapter. A refusal is the +answer: give its cause, as `run-analysis` describes for a broken run, and stop. After a round whose stop rule said +stop, `read` also refuses until the person's words are passed with `--continue-after-stop "..."`. + +## Stage 2. Find causes + +```bash +python3 L classify --loop DIR +``` + +`classify` gives each trial one of seven classes and a signature, groups the failing trials by signature, and +attaches warnings (`references/classes.md`). It writes `trials.csv` and `groups.json`, and runs `commands.classes` of +the adapter for the agent's own classes when the adapter has it. + +Then read two or three trials of each group, largest first, as `run-analysis/references/investigation.md` describes. +When you see a pattern the seven classes do not separate, write it as a found class in `found-classes.json` and run +`classify` again, so that the script counts it on every trial, the passing ones included. Write each cause to +`round-NN/causes/.md`: the claim, a line `Rule: ` that counts its trials, the trials and +their count, the code lines at fault at `base_ref`, the layer (model, agent, harness or benchmark), and what you did +not check. A group with the warning `fails_on_baseline` points at the benchmark; read one task's verifier and +expected answer before you write that the benchmark is defective. + +## Stage 3. Prove and fix + +For each cause in the agent layer: + +1. On `source.work_branch` of the checkout, commit a test on a recorded trial that fails on the code of `base_ref`. + Then commit one or more fixes, each its own candidate; a candidate is a commit whose history holds the test. +2. Register and check each candidate: + +```bash +python3 L candidate --loop DIR --id ID --cause CAUSE --ref FIX --test PATH --test-ref TEST_COMMIT +python3 L check --loop DIR --candidate ID +python3 L decide --loop DIR --stop 1 +``` + +`check` runs, each in a detached worktree of the commit that is removed afterwards: the test before and after the fix, +`commands.test` against the failures the suite has on `base_ref`, and `commands.replay` on every recorded trial at +`base_ref` and at the fix. The replay's labels crossed with pass and fail give the table done and passed / done and +failed / not done and passed / not done and failed, and the count of false successes, failed trials the agent reports +as a success. A command named in `paid` is written to `round-NN/commands/` and not started. + +**Stop 1.** Show the person `decision-1.md`: one row per candidate with its cause, test, table before and after, false +successes added and diff size, and the checks that did not run. Ask which candidates to keep, and record the answer in +the person's words: + +```bash +python3 L answer --loop DIR --stop 1 --keep ID[,ID] --text "the person's words" +``` + +## Stage 4. Build and try + +Merge the kept candidates into one commit of the work branch. Then: + +```bash +python3 L select --loop DIR --ref COMMIT --canaries N +``` + +prints the trial run in three sizes with their price from the cost per trial of the run that was read: the targets +(the tasks the kept causes predict), plus the guards (passing tasks whose recorded trial ran the changed code, when +the replay reports `touched`), plus the canaries (passing tasks drawn with a fixed seed). Every task gets 3 trials. The +person picks a size and names its budget, which cannot exceed what the round has left: + +```bash +python3 L select --loop DIR --ref COMMIT --canaries N --size targets|guards|canaries --budget-usd B --text "..." +``` + +This writes `trial-tasks.txt`, runs `commands.build` and writes `build.json`, and writes the run to +`round-NN/commands/run_tasks.sh`. The person starts it. When its job folder exists: + +```bash +python3 L decide --loop DIR --stop 2 --trial-job JOB +``` + +**Stop 2.** Show the person `decision-2.md`: the build, the targets' trials before and in every trial run of this +round on that build with Fisher's exact test, the guards and canaries, the money spent and left, and three options +with their price: stop, the targets and guards again, a full pass. The commands of the last two are written. Record +the choice: + +```bash +python3 L answer --loop DIR --stop 2 --choice stop|rerun|full --text "the person's words" +``` + +A rerun is another trial run of the round: run `decide --stop 2` on its job folder, and its trials and money add to +the earlier ones. + +## Stage 5. Compare and record + +```bash +python3 L compare --loop DIR --new JOB --against run +``` + +`compare` reads the new run against the round's run (`--against baseline` compares with the round's baseline) and +writes `result.json`: the pass rates with their standard error; the tasks both runs share, won and lost; the verdict +"improved" or "regressed" when an exact two-sided binomial test on won and lost gives p under 0.05, "noise" otherwise; +the targets' trials with Fisher's exact test; each prediction of stage 3 beside its result; and the stop rule. It +closes the round and writes `report.html` with `render` of `run-analysis`; fill its prose slots as that skill +describes (`runs.py render --slots`, then `runs.py check`). + +The stop rule is fixed before a round starts. No new round when the verdict is noise and no group of failing trials +of the new run has 3 trials or more, or when the budget left is under the price of the cheapest option, a trial run +of the largest such group's tasks. + +## When the adapter lacks a command + +| null command | what the loop does | what is weaker | +|---|---|---| +| `test_one` | the test of a cause does not run | nothing shows that the cause is real or that the fix removes it | +| `test` | the agent's suite does not run | a fix may break other code unseen | +| `replay` | no table before and after, no guards | false successes and the effect on passing trials are unknown until a paid run | +| `build` | the run installs the checkout | the build tried is not recorded by digest | +| `classes` | no extra columns | the seven classes stay the only ones | +| `run_tasks` | no command is written | the person starts the run of `trial-tasks.txt` by other means | +| a null limit | its class is switched off | those trials fall into a later class, usually `verifier_failed` | + +Every decision page names the checks that did not run. + +## Writing rules + +These follow `run-analysis`, with what the loop adds. + +- **Every count says what it counts**: trials, tasks or model calls, and of which run. +- **A cause is a claim with a count.** It names the rule that counts its trials and the trials it explains. A + plausible mechanism without a count is a hypothesis; say so. +- **Respect the noise.** Quote the verdict and its p value with any gain. Do not call a round an improvement on a + noise verdict. +- **Separate the layers.** Model, agent, harness or benchmark. A claim that a benchmark is defective needs the check + its warning names. +- **The person decides.** Never start a written command, pick a size or keep a candidate for the person, and store + each answer in the person's words. +- **Nothing sensitive.** No keys, tokens, host addresses or personal identifiers from logs. + +## Limits + +- `read` decides that a run is not analysable from the opening words of the verdicts `runs.py health` prints. A change + of that wording in `run-analysis` changes what `read` refuses. +- The classes read only shared files. What one agent writes reaches the loop through a found class or + `commands.classes`. +- A trial run gives every task the same number of trials, because `run_tasks` takes one `{repeat}`. diff --git a/.claude/skills/run-to-fix/references/classes.md b/.claude/skills/run-to-fix/references/classes.md new file mode 100644 index 00000000..123a9372 --- /dev/null +++ b/.claude/skills/run-to-fix/references/classes.md @@ -0,0 +1,86 @@ +# Classes, signatures, groups and warnings + +`loop.py classify` gives every trial of the run one class. The rules read only what both run paths of Nullius write +for any agent: the trial's `result.json`, its ATIF trajectory `agent/trajectory.json`, its `verifier/` folder, and +the three numbers of `limits` in the adapter. They read no agent log, no file that only one agent writes, and no +setting of one agent. + +## The seven classes + +The first rule that holds gives the class. + +| class | rule | switched off by | +|---|---|---| +| `pass` | the primary reward reaches the pass threshold (1 unless `read --pass-threshold` gives another) | never | +| `time_limit` | `exception_info.exception_type` is `AgentTimeoutError`, or the agent phase (`agent_execution`) lasted `limits.time_limit_s` or longer | a null `time_limit_s` switches off the rule on the agent phase; the timeout exception still counts | +| `exception` | `exception_info` is present | never | +| `no_score` | no reward and no exception | never | +| `step_limit` | the trial made `limits.step_limit` model calls or more | a null `step_limit` | +| `output_limit` | one model reply has `limits.output_token_limit` completion tokens or more | a null `output_token_limit` | +| `verifier_failed` | the verifier ran and the reward is under the threshold | never | + +With every limit null, each trial is `pass`, `time_limit`, `exception`, `no_score` or `verifier_failed`. +`groups.json` lists the rules a null limit switched off under `rules_off`. + +The primary reward and the count of model calls follow the rules of `run-analysis`. The reward is +`verifier_result.rewards.reward`, else the first numeric reward; model calls are the per-call usage records +(`extra.raw_usage` or steps with `metrics`, whichever is more), else the agent's steps. The largest reply is the +largest `metrics.completion_tokens` of a step or `output_tokens` of a usage record. `classify` stops when its count +of trials or passes, or of the pairs with the baseline, differs from the count in `metrics.json`. + +## Signatures and groups + +A signature is the class plus a detail with numbers and long hexadecimal ids masked as `#`: + +| class | detail | +|---|---| +| `time_limit`, `exception` | the exception type and the last line of its message | +| `verifier_failed` | the name of the first failed test in `verifier/ctrf.json`, else the first line of `verifier/test-stdout.txt` with FAIL, ERROR, Error or assert, else `reason` or `error` of `verifier/score.json`; empty when none of these exists | +| the others | none | + +A group is the set of failing trials with one signature. Groups are numbered `g01`, `g02`, ... from the largest. + +## Warnings + +A warning is attached to a group, with its count, and points a person to a check. + +| warning | when | +|---|---| +| `frequent` | the group has at least 3 trials and at least 5% of the failing trials | +| `fails_on_baseline` | every task of the group also fails on the baseline; the benchmark is a suspect | +| `fails_in_every_repeat` | some tasks of the group have several trials and fail in all of them; only a run with repeats gets it | + +A claim that a benchmark is defective needs the check the warning names: read the verifier and the expected answer +of one task. + +## Found classes + +While reading trials, the assistant may add a rule that the script then counts on every trial, the passing ones +included. The rules live in `/found-classes.json`; the skill ships with none. + +```json +{ + "schema": "run-to-fix.found-classes/v1", + "rules": [ + {"id": "reply_cut", "meaning": "A model reply ended at the length limit.", + "file": "agent/trajectory.json", "json_path": "steps.*.extra.finish_reason", "value": "length"}, + {"id": "stopped_at_cap", "meaning": "The agent wrote that it stopped at its step cap.", + "file": "agent/output.json", "regex": "\"status\":\\s*\"step_cap\""} + ] +} +``` + +1. `id` is lower-case letters, digits and `_`. `meaning` is one sentence. `file` is a path inside the trial folder. +2. A rule has either `regex` (searched in the file's text) or `json_path` with `value`. A `json_path` segment is a + key, a list index, or `*` for every element; the rule matches when any value reached equals `value`. +3. A found class may read a file that only one agent writes. +4. `trials.csv` gets a column `found:` per rule: `1`, `0` (no match or no file), or `unreadable` (a JSON file + that does not parse). `groups.json` counts each rule's trials, failing trials and unreadable files. + +A cause names the rule that counts its trials (`references/files.md`, causes). + +## The agent's own classes + +`commands.classes` of the adapter writes classes and flags per trial. Each becomes a column `agent_class:` or +`agent_flag:` of `trials.csv`, beside the seven classes. When the command is null, `groups.json` says the check +did not run. diff --git a/.claude/skills/run-to-fix/references/files.md b/.claude/skills/run-to-fix/references/files.md new file mode 100644 index 00000000..92cf952f --- /dev/null +++ b/.claude/skills/run-to-fix/references/files.md @@ -0,0 +1,93 @@ +# Files of the loop + +A person names the loop folder. `loop.py` writes only there, plus worktree records in the agent checkout's git +folder while a check runs. Tests and fixes are commits that the assistant makes on `source.work_branch`. + +## The loop folder + +| file | written by | holds | +|---|---|---| +| `adapter.json` | the assistant, confirmed by the person | the agent's checkout, commands and limits (`references/adapter.md`) | +| `found-classes.json` | the assistant | rules counted on every trial (`references/classes.md`) | +| `ledger.json` | `read`, `answer`, `select`, `decide --stop 2`, `compare` | one entry per round, below | +| `worktrees/` | `check`, `select` | a detached worktree per check, removed when the check ends; empty between commands | +| `round-NN/` | every command | the files of one round, below | + +`read` starts `round-01`, and a new round after a round with `result.json`. Every other command works on the last +round that has no `result.json`. + +## A round + +| file | stage | written by | holds | +|---|---|---|---| +| `input.json` | 1 | `read` | the run and baseline job folders, the adapter's SHA-256, the budget, the pass threshold, the `runs.py` used; `refused` with its reason when the run is not analysable | +| `health.txt` | 1 | `read` | `runs.py health` of the run and the baseline | +| `metrics.json` | 1 | `read` | `runs.py analyze` of the baseline and the run, labels `baseline` and `run` | +| `trials.csv` | 2 | `classify` | one row per trial: `trial`, `task`, `class`, `signature`, `reward`, `pair`, `cost_usd`, `model_calls`, `agent_s`, `max_completion_tokens`, `exception_type`, then `found:`, `agent_class:`, `agent_flag:` | +| `groups.json` | 2 | `classify` | class counts, the rules switched off, the groups with their warnings, the counts of found and agent classes | +| `agent-classes.json` | 2 | `commands.classes` | the agent's own classes and flags per trial | +| `causes/.md` | 2 | the assistant | one cause, below | +| `candidates//candidate.json` | 3 | `candidate` | the cause, the rule, the fix commit, the test and its commit, the trials and tasks the cause predicts | +| `candidates//checks.json` | 3 | `check` | the test before and after the fix, the suite against `base_ref`, the replay table before and after, the diff size | +| `candidates//*.log`, `replay.json` | 3 | `check` | the output of each command | +| `base/suite.json`, `base/replay.json` | 3 | `check` | the suite's failures and the replay on `base_ref`, made once per round | +| `decision-1.md` | stop 1 | `decide --stop 1` | one row per candidate and the checks that did not run | +| `commands/-.sh`, `commands/run_tasks*.sh` | 3, 4 | `check`, `select`, `decide --stop 2` | commands a person starts | +| `trial-tasks.txt` | 4 | `select --size` | the tasks of the trial run, one per line, the `{units_file}` of `run_tasks` | +| `build.json` | 4 | `select --size` | the commit tried, the path `commands.build` printed and its SHA-256 | +| `jobs/` | 4 | the person's run | the job folders that `run_tasks` writes | +| `trial.json` | stop 2 | `decide --stop 2` | the round's trial runs, the results of targets, guards and canaries, the money spent and left, the options | +| `rerun-tasks.txt`, `full-tasks.txt` | stop 2 | `decide --stop 2` | the tasks of the two options that run something | +| `decision-2.md` | stop 2 | `decide --stop 2` | the build, the results, the budget left and three options with their price | +| `compare-metrics.json` | 5 | `compare` | `runs.py analyze` of the run compared against (`before`) and the new run (`new`) | +| `result.json` | 5 | `compare` | pass rates with their standard error, pairs, the exact test and the verdict, the targets, the predictions beside their results, the groups of the new run, the stop rule | +| `report.html` | 5 | `compare`, then the assistant | the page of `runs.py render`, with its prose slots filled by the assistant | + +## A cause + +`causes/.md` is one claim. It has these parts, and a line `Rule: ` that names the group of `groups.json` or +the found class that counts its trials; `candidate` refuses a cause without one. + +``` +# The agent reports success after a write it never checked + +Rule: g02 + +Trials: 7 (list them). Tasks: 7. +Code: the lines at fault, as path:line at base_ref. +Layer: agent (model, agent, harness or benchmark). +Not checked: what the files cannot tell. +``` + +## The ledger + +```json +{ + "schema": "run-to-fix.ledger/v1", + "rounds": [{ + "round": "round-01", "started": "", + "answers": [{"about": "stop 1: the candidates to keep", "date": "", "text": ""}], + "kept": [""], + "predictions": [{"candidate": "...", "cause": "...", "rule": "...", "tasks": ["..."], "expect": "pass"}], + "selection": {"ref": "...", "size": "guards", "targets": [], "guards": [], "canaries": [], "repeat": 3, + "seed": 20261117, "price_usd": 0.0, "budget_usd": 0.0, "guards_reason": null}, + "trial_runs": [{"job_dir": "...", "ref": "...", "trials": 0, "cost_usd": 0.0, "cost_complete": true}], + "next": "rerun", + "result": {"verdict": "noise", "p": 0.68, "won": 10, "lost": 13, "stop": true, "reasons": ["..."]} + }] +} +``` + +Every answer of the person is stored with its date and in the person's words: the candidates kept at stop 1, the +size and budget of the trial run, the choice at stop 2, and the words that start a round after a stop rule said +stop. The money a round has spent is the sum of its `trial_runs` plus the new run that `compare` reads. + +## Files of a run that the loop reads + +| file | used for | +|---|---| +| `/result.json` | task, trial name, reward, exception, agent phase, cost, agent name | +| `/agent/trajectory.json` | model calls and the largest reply | +| `/verifier/ctrf.json`, `test-stdout.txt`, `score.json` | the detail of a `verifier_failed` signature | +| any file a found class names | the found class's column | +| everything `runs.py` reads | `health`, `analyze` and `render` (`run-analysis/references/files.md`) | From 196483940131670dec0fafb29699c099e91279e4 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:13:32 +0100 Subject: [PATCH 17/92] test(run-to-fix): mask quoted strings in exception signatures --- .../skills/run-to-fix/tests/test_classify.py | 78 +++++++++++++++++++ 1 file changed, 78 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_classify.py b/.claude/skills/run-to-fix/tests/test_classify.py index 1517d12d..22c21afe 100644 --- a/.claude/skills/run-to-fix/tests/test_classify.py +++ b/.claude/skills/run-to-fix/tests/test_classify.py @@ -347,3 +347,81 @@ def test_a_rule_with_two_patterns_is_refused(self) -> None: r = run(["classify", "--loop", self.loop]) self.assertNotEqual(r.code, 0) self.assertIn("both", r.err) + + +#: the last lines of two exception messages of one real run; they differ in the model text quoted after unit= +REPEATED_STREAM = [ + r"JiuwenSwarm failed: FrameworkError: [181001] model call failed, reason: LLM repeated stream output detected: " + r"field=reasoning_content, unit='OK.\n\nWriting.\n\nLet me go.\n\n', repeat_count=6", + r"JiuwenSwarm failed: FrameworkError: [181001] model call failed, reason: LLM repeated stream output detected: " + r"field=reasoning_content, unit='\n\nHmm.', repeat_count=27", +] + + +class QuotedStrings(LoopCase): + def classify_exceptions(self, messages: List[str]) -> Dict[str, Dict[str, str]]: + trials = [] + for i, message in enumerate(messages): + t = trial(f"e{i}__a", f"bench/e{i}", None) + t.exception = ("NonZeroAgentExitCodeError", message) + trials.append(t) + trials.append(trial("p__a", "bench/p", 1.0)) + run_dir = write_job(self.jobs, "run", "alpha", "harbor", trials) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(run_dir, None) + self.classify_ok() + return self.by_trial() + + def test_two_repeated_streams_share_one_signature(self) -> None: + rows = self.classify_exceptions(REPEATED_STREAM) + self.assertEqual(rows["e0__a"]["signature"], rows["e1__a"]["signature"]) + self.assertTrue(rows["e0__a"]["signature"].endswith("unit='#', repeat_count=#"), rows["e0__a"]["signature"]) + group = load(self.round_file("groups.json"))["groups"][0] + self.assertEqual(group["trials"], 2) + self.assertEqual(group["example"], "NonZeroAgentExitCodeError: " + REPEATED_STREAM[0]) + + def test_the_mask_runs_before_the_cut(self) -> None: + rows = self.classify_exceptions([f"reason: unit='{'x' * 400}', repeat_count=3"]) + self.assertEqual(rows["e0__a"]["signature"], "exception: NonZeroAgentExitCodeError: reason: unit='#', repeat_count=#") + + def test_a_quote_that_is_not_closed_masks_to_the_end(self) -> None: + rows = self.classify_exceptions(["reason: unit='OK. Writing. Let me go"]) + self.assertEqual(rows["e0__a"]["signature"], "exception: NonZeroAgentExitCodeError: reason: unit='#") + + def test_an_apostrophe_inside_a_word_masks_nothing(self) -> None: + rows = self.classify_exceptions(["the agent's reply isn't valid at step 12"]) + self.assertEqual(rows["e0__a"]["signature"], + "exception: NonZeroAgentExitCodeError: the agent's reply isn't valid at step #") + + def test_double_quotes_are_masked(self) -> None: + rows = self.classify_exceptions(['tool "browser_click" failed with "element not found"']) + self.assertEqual(rows["e0__a"]["signature"], 'exception: NonZeroAgentExitCodeError: tool "#" failed with "#"') + + def test_verifier_and_timeout_details_keep_their_quotes(self) -> None: + failed = trial("f__a", "bench/f", 0.0) + failed.verifier_text = "FAIL: Kevin Zhao's email does not have the 'Clients' label" + late = trial("t__a", "bench/t", None) + late.exception = ("AgentTimeoutError", "the agent did not answer 'ping' in time") + run_dir = write_job(self.jobs, "run", "alpha", "harbor", [failed, late, trial("p__a", "bench/p", 1.0)]) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(run_dir, None) + self.classify_ok() + rows = self.by_trial() + self.assertEqual(rows["f__a"]["signature"], + "verifier_failed: FAIL: Kevin Zhao's email does not have the 'Clients' label") + self.assertEqual(rows["t__a"]["signature"], "time_limit: AgentTimeoutError: the agent did not answer 'ping' in time") + + +class BaselineWarningText(LoopCase): + def test_a_small_group_only_says_the_task_fails_on_both_runs(self) -> None: + run_dir = write_job(self.jobs, "run", "alpha", "harbor", + [shared_failure("a__r", "bench/a"), trial("p__r", "bench/p", 1.0)]) + base_dir = write_job(self.jobs, "base", "alpha", "harbor", + [shared_failure("a__b", "bench/a"), trial("p__b", "bench/p", 1.0)]) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(run_dir, base_dir) + self.classify_ok() + warning = load(self.round_file("groups.json"))["groups"][0]["warnings"][0] + self.assertEqual(warning["kind"], "fails_on_baseline") + self.assertIn("fails on both runs", warning["text"]) + self.assertNotIn("benchmark", warning["text"]) From ae90436419b78a87f72795745b8a84f0aaa8ba55 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:18:44 +0100 Subject: [PATCH 18/92] feat(run-to-fix): mask quoted strings in exception signatures Two exceptions that differ only in the model text they quote now share a signature. The mask runs before the detail is cut, a quote that is not closed masks to the end, and an apostrophe inside a word masks nothing. Each group keeps the detail of its first trial as written in example. The baseline warning names the benchmark only for groups of 3 tasks or more. --- .../skills/run-to-fix/scripts/classifying.py | 23 ++++---- .claude/skills/run-to-fix/scripts/trials.py | 57 +++++++++++++++---- 2 files changed, 59 insertions(+), 21 deletions(-) diff --git a/.claude/skills/run-to-fix/scripts/classifying.py b/.claude/skills/run-to-fix/scripts/classifying.py index 395b1261..d4d5eff0 100644 --- a/.claude/skills/run-to-fix/scripts/classifying.py +++ b/.claude/skills/run-to-fix/scripts/classifying.py @@ -11,10 +11,11 @@ from found import UNREADABLE, Rule, apply, load_rules from reading import adapter_of from rounds import LoopError, open_round, read_input, read_json, run_command, write_json, write_paid -from trials import CLASSES, TrialRow, classify, read_trial, rules_off, signature, solved_tasks, trial_dirs +from trials import CLASSES, Signature, TrialRow, classify, read_trial, rules_off, signature, solved_tasks, trial_dirs FREQUENT_MIN_TRIALS = 3 FREQUENT_MIN_SHARE = 0.05 +BENCHMARK_SUSPECT_TASKS = 3 BASE_COLUMNS = ["trial", "task", "class", "signature", "reward", "pair", "cost_usd", "model_calls", "agent_s", "max_completion_tokens", "exception_type"] @@ -23,7 +24,7 @@ def read_rows(job_dir: str, threshold: float) -> List[TrialRow]: return [read_trial(d, threshold) for d in trial_dirs(job_dir)] -def classes_of(rows: List[TrialRow], limits: Limits) -> Tuple[Dict[str, str], Dict[str, str]]: +def classes_of(rows: List[TrialRow], limits: Limits) -> Tuple[Dict[str, str], Dict[str, Signature]]: """The class and the signature of each trial, by trial name.""" classes, signatures = {}, {} for row in rows: @@ -96,10 +97,11 @@ def warnings_of(group_rows: List[TrialRow], failures: int, by_task: Dict[str, Li "text": f"{n} of {failures} failing trials share this signature. Read two or three of them."}) tasks = sorted({r.task for r in group_rows}) if base_solved is not None and all(t in base_solved and not base_solved[t] for t in tasks): - out.append({"kind": "fails_on_baseline", "tasks": len(tasks), - "text": f"All {len(tasks)} tasks of this group also fail on the baseline, so the benchmark is a " - "suspect. Check one task's verifier and expected answer before any claim that the " - "benchmark is defective."}) + text = (f"All {len(tasks)} tasks of this group also fail on the baseline, so the benchmark is a suspect. " + "Check one task's verifier and expected answer before any claim that the benchmark is defective." + if len(tasks) >= BENCHMARK_SUSPECT_TASKS else + ("The task fails on both runs." if len(tasks) == 1 else f"The {len(tasks)} tasks fail on both runs.")) + out.append({"kind": "fails_on_baseline", "tasks": len(tasks), "text": text}) repeated = [t for t in tasks if len(by_task[t]) > 1] every = [t for t in repeated if not any(r.passed for r in by_task[t])] if every: @@ -131,7 +133,7 @@ def cmd_classify(args: argparse.Namespace) -> None: table = [] found_counts = {r.id: collections.Counter() for r in rules} for row in rows: - cls, sig = classes[row.trial], signatures[row.trial] + cls, sig = classes[row.trial], signatures[row.trial].text cells = {"trial": row.trial, "task": row.task, "class": cls, "signature": sig, "reward": "" if row.reward is None else row.reward, "pair": pair_of(row.task, run_solved, base_solved), "cost_usd": "" if row.cost_usd is None else row.cost_usd, @@ -184,17 +186,18 @@ def cmd_classify(args: argparse.Namespace) -> None: print(f" check not run: {agent['reason']}") -def failure_groups(rows: List[TrialRow], classes: Dict[str, str], signatures: Dict[str, str], +def failure_groups(rows: List[TrialRow], classes: Dict[str, str], signatures: Dict[str, Signature], base_solved: Optional[Dict[str, bool]]) -> List[dict]: """The failing trials grouped by signature, largest first, each with its warnings.""" _, by_task = solved_tasks(rows) failing = [r for r in rows if classes[r.trial] != "pass"] by_sig: Dict[str, List[TrialRow]] = collections.defaultdict(list) for r in failing: - by_sig[signatures[r.trial]].append(r) + by_sig[signatures[r.trial].text].append(r) ordered = sorted(by_sig.items(), key=lambda kv: (-len(kv[1]), kv[0])) return [{ - "id": f"g{i + 1:02d}", "class": classes[members[0].trial], "signature": sig, "trials": len(members), + "id": f"g{i + 1:02d}", "class": classes[members[0].trial], "signature": sig, + "example": signatures[members[0].trial].example, "trials": len(members), "tasks": len({m.task for m in members}), "trial_names": sorted(m.trial for m in members), "task_names": sorted({m.task for m in members}), "warnings": warnings_of(members, len(failing), by_task, base_solved), diff --git a/.claude/skills/run-to-fix/scripts/trials.py b/.claude/skills/run-to-fix/scripts/trials.py index 03883ff8..4049b02d 100644 --- a/.claude/skills/run-to-fix/scripts/trials.py +++ b/.claude/skills/run-to-fix/scripts/trials.py @@ -16,6 +16,10 @@ FAILING_LINE = re.compile(r"\b(FAIL(?:ED|URE)?|ERROR|Error|AssertionError|assert)\b") SCORE_TEXT_KEYS = ("reason", "error") DETAIL_CHARS = 200 +EXAMPLE_CHARS = 500 +#: an opening quote: a quote mark that does not follow a letter or digit, so the apostrophe of "agent's" opens nothing +QUOTE_OPEN = re.compile(r"(? List[str]: return off -def mask(text: str) -> str: - return re.sub(r"\s+", " ", NUMBER.sub("#", text)).strip()[:DETAIL_CHARS] +def mask_quotes(text: str) -> str: + """Each quoted string as a quote, # and the closing quote; a quote that is not closed masks to the end.""" + out, i = [], 0 + while True: + m = QUOTE_OPEN.search(text, i) + if m is None: + out.append(text[i:]) + return "".join(out) + q = m.group(1) + out.append(text[i:m.start()] + q + "#") + close = re.compile(re.escape(q) + r"(?!\w)").search(text, m.end()) + if close is None: + return "".join(out) + out.append(q) + i = close.end() + + +def _collapse(text: str) -> str: + return re.sub(r"\s+", " ", text).strip() + + +def mask(text: str, quotes: bool) -> str: + """The detail of a signature: quoted strings when `quotes`, then numbers and long ids, masked; then cut.""" + return _collapse(NUMBER.sub("#", mask_quotes(text) if quotes else text))[:DETAIL_CHARS] def _last_line(message: str) -> str: @@ -166,7 +192,7 @@ def _exception_detail(row: TrialRow) -> str: prefix = f"{row.exception_type}: " if line.startswith(prefix): line = line[len(prefix):] - return mask(f"{row.exception_type}: {line}" if line else str(row.exception_type)) + return f"{row.exception_type}: {line}" if line else str(row.exception_type) def _ctrf_failure(path: str) -> Optional[str]: @@ -185,30 +211,39 @@ def _verifier_detail(trial_dir: str) -> str: if os.path.isfile(ctrf): name = _ctrf_failure(ctrf) if name: - return mask(name) + return name stdout = os.path.join(verifier, "test-stdout.txt") if os.path.isfile(stdout): with open(stdout, encoding="utf-8", errors="replace") as fh: for line in fh: if FAILING_LINE.search(line): - return mask(line) + return line score = os.path.join(verifier, "score.json") if os.path.isfile(score): data = read_json(score) for key in SCORE_TEXT_KEYS: value = data.get(key) if isinstance(data, dict) else None if isinstance(value, str) and value.strip(): - return mask(_last_line(value)) + return _last_line(value) return "" -def signature(row: TrialRow, cls: str) -> str: - detail = "" +@dataclass(frozen=True) +class Signature: + text: str + example: str + + +def signature(row: TrialRow, cls: str) -> Signature: + """The class and its masked detail; `example` is the detail as written, with secrets masked.""" + raw = "" if row.exception_type is not None and cls in ("time_limit", "exception"): - detail = _exception_detail(row) + raw = _exception_detail(row) elif cls == "verifier_failed": - detail = _verifier_detail(row.dir) - return f"{cls}: {detail}" if detail else cls + raw = _verifier_detail(row.dir) + detail = mask(raw, cls == "exception") + return Signature(text=f"{cls}: {detail}" if detail else cls, + example=_collapse(SECRETS.sub("[redacted]", raw))[:EXAMPLE_CHARS]) def trial_dirs(job_dir: str) -> List[str]: From b5deae48f89e20292aecb16f38d4c413ae1637fd Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:18:44 +0100 Subject: [PATCH 19/92] docs(run-to-fix): describe the quote mask, the group example and the baseline warning --- .claude/skills/run-to-fix/references/classes.md | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/.claude/skills/run-to-fix/references/classes.md b/.claude/skills/run-to-fix/references/classes.md index 123a9372..9d9e46bb 100644 --- a/.claude/skills/run-to-fix/references/classes.md +++ b/.claude/skills/run-to-fix/references/classes.md @@ -34,11 +34,18 @@ A signature is the class plus a detail with numbers and long hexadecimal ids mas | class | detail | |---|---| -| `time_limit`, `exception` | the exception type and the last line of its message | +| `exception` | the exception type and the last line of its message, with each quoted string masked as well | +| `time_limit` | the exception type and the last line of its message, when there is an exception | | `verifier_failed` | the name of the first failed test in `verifier/ctrf.json`, else the first line of `verifier/test-stdout.txt` with FAIL, ERROR, Error or assert, else `reason` or `error` of `verifier/score.json`; empty when none of these exists | | the others | none | +In an `exception` detail, a quoted string becomes `'#'` or `"#"` before the detail is cut to 200 characters, so two +messages that differ only in the model text they quote share a signature. A quote with no closing quote masks to the +end of the detail. An apostrophe inside a word, as in `agent's`, opens no quote. Verifier details keep their quoted +values, because those separate one assertion from another. + A group is the set of failing trials with one signature. Groups are numbered `g01`, `g02`, ... from the largest. +`example` holds the detail of the group's first trial as written, with only key-shaped strings masked. ## Warnings @@ -47,7 +54,7 @@ A warning is attached to a group, with its count, and points a person to a check | warning | when | |---|---| | `frequent` | the group has at least 3 trials and at least 5% of the failing trials | -| `fails_on_baseline` | every task of the group also fails on the baseline; the benchmark is a suspect | +| `fails_on_baseline` | every task of the group also fails on the baseline; for a group of 3 tasks or more the text names the benchmark as a suspect, for a smaller group it says only that the tasks fail on both runs | | `fails_in_every_repeat` | some tasks of the group have several trials and fail in all of them; only a run with repeats gets it | A claim that a benchmark is defective needs the check the warning names: read the verifier and the expected answer From 78448417c0c0a37b1c554bba9cc07d5525f34c10 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Wed, 18 Nov 2026 00:18:54 +0100 Subject: [PATCH 20/92] docs(run-to-fix): state the effect of the quote mask in its own sentence --- .claude/skills/run-to-fix/references/classes.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.claude/skills/run-to-fix/references/classes.md b/.claude/skills/run-to-fix/references/classes.md index 9d9e46bb..a6b271dd 100644 --- a/.claude/skills/run-to-fix/references/classes.md +++ b/.claude/skills/run-to-fix/references/classes.md @@ -39,8 +39,8 @@ A signature is the class plus a detail with numbers and long hexadecimal ids mas | `verifier_failed` | the name of the first failed test in `verifier/ctrf.json`, else the first line of `verifier/test-stdout.txt` with FAIL, ERROR, Error or assert, else `reason` or `error` of `verifier/score.json`; empty when none of these exists | | the others | none | -In an `exception` detail, a quoted string becomes `'#'` or `"#"` before the detail is cut to 200 characters, so two -messages that differ only in the model text they quote share a signature. A quote with no closing quote masks to the +In an `exception` detail, a quoted string becomes `'#'` or `"#"` before the detail is cut to 200 characters. Two +messages that differ only in the model text they quote then share a signature. A quote with no closing quote masks to the end of the detail. An apostrophe inside a word, as in `agent's`, opens no quote. Verifier details keep their quoted values, because those separate one assertion from another. From 960cb28f2e69fdc9c5972981b6d4bd3d18b47470 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:22:49 +0100 Subject: [PATCH 21/92] test(run-to-fix): name the runs of a round page by their job folders --- .../skills/run-to-fix/tests/test_compare.py | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_compare.py b/.claude/skills/run-to-fix/tests/test_compare.py index e041e54a..e703a7d6 100644 --- a/.claude/skills/run-to-fix/tests/test_compare.py +++ b/.claude/skills/run-to-fix/tests/test_compare.py @@ -218,3 +218,35 @@ def test_the_next_read_after_a_stop_needs_the_person(self) -> None: self.assertEqual(answers[0]["about"], "continue after the stop rule of round-01") self.assertEqual(answers[0]["text"], words) self.assertRegex(answers[0]["date"], r"^\d{4}-\d{2}-\d{2}T") + + +class PageLabels(LoopCase): + """The page of a round names the two runs by their job folders; result.json keeps its keys.""" + + def compare_jobs(self, before_root: str, before_name: str, new_root: str, new_name: str) -> Tuple[dict, str]: + before, new = paired(5, 2, 1, 1) + os.makedirs(before_root, exist_ok=True) + os.makedirs(new_root, exist_ok=True) + before_dir = write_job(before_root, before_name, "alpha", "harbor", before) + new_dir = write_job(new_root, new_name, "alpha", "harbor", new) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(before_dir, None) + r = run(["compare", "--loop", self.loop, "--new", new_dir, "--against", "run"]) + self.assertEqual(r.code, 0, r.out + r.err) + with open(self.round_file("report.html"), encoding="utf-8") as fh: + return load(self.round_file("compare-metrics.json")), fh.read() + + def test_the_labels_are_the_job_names(self) -> None: + metrics, page = self.compare_jobs(self.jobs, "agent-old", self.jobs, "agent-new") + self.assertEqual([r["label"] for r in metrics["runs"]], ["agent-old", "agent-new"]) + self.assertIn("agent-old", page) + self.assertIn("agent-new", page) + self.assertNotIn("before", page) + result = load(self.round_file("result.json")) + self.assertEqual(set(result["pass_rate"]), {"before", "new"}) + self.assertEqual((result["pairs"]["won"], result["pairs"]["lost"]), (2, 1)) + + def test_two_jobs_of_one_name_keep_before_and_new(self) -> None: + metrics, page = self.compare_jobs(os.path.join(self.jobs, "a"), "trial", os.path.join(self.jobs, "b"), "trial") + self.assertEqual([r["label"] for r in metrics["runs"]], ["before", "new"]) + self.assertIn("before", page) From 275f7c6906a24786aabfc370ac8fbe16b6b4e1cc Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:27:26 +0100 Subject: [PATCH 22/92] fix(run-to-fix): name the runs of a round page by their job folders --- .claude/skills/run-to-fix/scripts/comparing.py | 16 ++++++++++++---- .claude/skills/run-to-fix/scripts/reading.py | 3 ++- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/.claude/skills/run-to-fix/scripts/comparing.py b/.claude/skills/run-to-fix/scripts/comparing.py index 17ef0482..351d6444 100644 --- a/.claude/skills/run-to-fix/scripts/comparing.py +++ b/.claude/skills/run-to-fix/scripts/comparing.py @@ -3,7 +3,7 @@ import argparse import os -from typing import Dict, List, Optional +from typing import Dict, List, Optional, Tuple from classifying import classes_of, cross_check, failure_groups, read_rows from reading import adapter_of, analyze, health @@ -23,6 +23,13 @@ def usd(value: float) -> str: return f"-${-value:.2f}" if value < 0 else f"${value:.2f}" +def page_labels(before_job: str, new_job: str) -> Tuple[str, str]: + """The names of the two runs on the round page: their job folders, or before and new when the names are equal.""" + if before_job and new_job and before_job != new_job: + return before_job, new_job + return "before", "new" + + def rate_of(run: dict) -> dict: return {"rate": run["pass_rate"], "standard_error": run["standard_error"], "tasks": run["tasks"], "trials": run["trials"], "trials_passed": run["trials_passed"]} @@ -114,12 +121,13 @@ def cmd_compare(args: argparse.Namespace) -> None: + ", ".join(repr(a) for a in checked.agents)) metrics_path = os.path.join(round_dir, "compare-metrics.json") - print(analyze([("before", against["job_dir"]), ("new", args.new)], threshold, metrics_path)) + labels = page_labels(os.path.basename(against["job_dir"]), checked.job) + print(analyze([(labels[0], against["job_dir"]), (labels[1], args.new)], threshold, metrics_path)) metrics = read_json(metrics_path) - runs: Dict[str, dict] = {r["label"]: r for r in metrics["runs"]} + runs: Dict[str, dict] = dict(zip(("before", "new"), metrics["runs"])) new_dir = runs["new"]["dir"] before_rows, new_rows = read_rows(against["job_dir"], threshold), read_rows(new_dir, threshold) - cross_check(metrics_path, [("before", before_rows), ("new", new_rows)]) + cross_check(metrics_path, [(labels[0], before_rows), (labels[1], new_rows)]) pair = metrics["pairs"][0] won, lost = pair["second_only"], pair["first_only"] diff --git a/.claude/skills/run-to-fix/scripts/reading.py b/.claude/skills/run-to-fix/scripts/reading.py index 413f72f6..0e14ceb9 100644 --- a/.claude/skills/run-to-fix/scripts/reading.py +++ b/.claude/skills/run-to-fix/scripts/reading.py @@ -39,6 +39,7 @@ def full_spec(spec: str) -> str: @dataclass(frozen=True) class Health: text: str + job: str agents: List[str] refused: List[str] @@ -61,7 +62,7 @@ def health(label: str, spec: str, threshold: float) -> Health: refused.append(f"{label}: {v}") refused += [f"{label}: e.g. {example}" for message, example in (job.get("error_examples") or {}).items() if example and message in v] - return Health(text=text, agents=list(job.get("agents") or []), refused=refused) + return Health(text=text, job=str(job.get("job") or ""), agents=list(job.get("agents") or []), refused=refused) def analyze(runs: List[Tuple[str, str]], threshold: float, out: str) -> str: From 9785255c8708d0bbb9c044b30e46c177a46d8047 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Wed, 18 Nov 2026 03:27:26 +0100 Subject: [PATCH 23/92] docs(run-to-fix): say how the runs of compare-metrics.json are labelled --- .claude/skills/run-to-fix/references/files.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/references/files.md b/.claude/skills/run-to-fix/references/files.md index 92cf952f..3ea7966b 100644 --- a/.claude/skills/run-to-fix/references/files.md +++ b/.claude/skills/run-to-fix/references/files.md @@ -39,7 +39,7 @@ round that has no `result.json`. | `trial.json` | stop 2 | `decide --stop 2` | the round's trial runs, the results of targets, guards and canaries, the money spent and left, the options | | `rerun-tasks.txt`, `full-tasks.txt` | stop 2 | `decide --stop 2` | the tasks of the two options that run something | | `decision-2.md` | stop 2 | `decide --stop 2` | the build, the results, the budget left and three options with their price | -| `compare-metrics.json` | 5 | `compare` | `runs.py analyze` of the run compared against (`before`) and the new run (`new`) | +| `compare-metrics.json` | 5 | `compare` | `runs.py analyze` of the run compared against and the new run, labelled by their job folders, or `before` and `new` when the two names are equal | | `result.json` | 5 | `compare` | pass rates with their standard error, pairs, the exact test and the verdict, the targets, the predictions beside their results, the groups of the new run, the stop rule | | `report.html` | 5 | `compare`, then the assistant | the page of `runs.py render`, with its prose slots filled by the assistant | From fc0bc9355e8fbceb37cf2de5058b0d20bf49a36d Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:35:08 +0100 Subject: [PATCH 24/92] test(run-to-fix): price each task by its recorded cost, and two small fixes The made-up trials get their own costs, failed ones more than passing ones, and the expected prices follow the price of each task. New tests cover the cheapest group of the stop rule, a task without a cost, a trial job named as PATH::JOB, and a result that cannot be written. --- .claude/skills/run-to-fix/tests/agentrepo.py | 6 ++ .claude/skills/run-to-fix/tests/jobs.py | 7 +- .../skills/run-to-fix/tests/test_compare.py | 63 +++++++++++++++++- .../skills/run-to-fix/tests/test_decide.py | 65 ++++++++++++------- 4 files changed, 115 insertions(+), 26 deletions(-) diff --git a/.claude/skills/run-to-fix/tests/agentrepo.py b/.claude/skills/run-to-fix/tests/agentrepo.py index ce0b7f0a..7efcd6b2 100644 --- a/.claude/skills/run-to-fix/tests/agentrepo.py +++ b/.claude/skills/run-to-fix/tests/agentrepo.py @@ -87,6 +87,10 @@ def make_repo(root: str) -> Repo: return Repo(path=path, base=base, test_ref=test_ref, good=good, bad=bad) +#: what each recorded trial cost; the failed ones cost more, as in real runs +RECORDED_COST = {"p1__r": 0.01, "p2__r": 0.01, "p3__r": 0.01, "f1__r": 0.05, "f2__r": 0.07, "f3__r": 0.03} + + def recorded_trials() -> List[Trial]: out = [trial("p1__r", "bench/p1", 1.0), trial("p2__r", "bench/p2", 1.0), trial("p3__r", "bench/p3", 1.0)] for name in ("f1", "f2"): @@ -94,6 +98,8 @@ def recorded_trials() -> List[Trial]: t.files = {"verifier/ctrf.json": '{"results": {"tests": [{"name": "test_state", "status": "failed"}]}}'} out.append(t) out.append(trial("f3__r", "bench/f3", 0.0)) + for t in out: + t.cost = RECORDED_COST[t.name] return out diff --git a/.claude/skills/run-to-fix/tests/jobs.py b/.claude/skills/run-to-fix/tests/jobs.py index 51c1ef66..030a1ba5 100644 --- a/.claude/skills/run-to-fix/tests/jobs.py +++ b/.claude/skills/run-to-fix/tests/jobs.py @@ -30,12 +30,13 @@ class Trial: max_output: int verifier_text: str files: Dict[str, str] + cost: Optional[float] def trial(name: str, task: str, reward: Optional[float]) -> Trial: - """A trial of 5 model calls, 60 s, 100 tokens per reply, no exception, no extra files.""" + """A trial of 5 model calls, 60 s, 100 tokens per reply, 0.01 USD, no exception, no extra files.""" return Trial(name=name, task=task, reward=reward, exception=None, agent_s=60.0, calls=5, max_output=100, - verifier_text="", files={}) + verifier_text="", files={}, cost=0.01) def iso(t: dt.datetime) -> str: @@ -73,7 +74,7 @@ def write_trial(job_dir: str, t: Trial, agent: str, model: str, runner: str, ind "environment_setup": {"started_at": iso(start), "finished_at": iso(agent_start)}, "agent_execution": {"started_at": iso(agent_start), "finished_at": iso(agent_end)}, "agent_result": {"n_input_tokens": 1000 * t.calls, "n_output_tokens": 100 * t.calls, "n_cache_tokens": 0, - "cost_usd": 0.01}, + "cost_usd": t.cost}, } if t.reward is not None: res["verifier"] = {"started_at": iso(agent_end), "finished_at": iso(end)} diff --git a/.claude/skills/run-to-fix/tests/test_compare.py b/.claude/skills/run-to-fix/tests/test_compare.py index e703a7d6..265d6b64 100644 --- a/.claude/skills/run-to-fix/tests/test_compare.py +++ b/.claude/skills/run-to-fix/tests/test_compare.py @@ -2,7 +2,7 @@ from __future__ import annotations import os -from typing import List, Tuple +from typing import List, Optional, Tuple from jobs import (NO_LIMITS, LoopCase, Trial, adapter, load, run, trial, write_job, write_json) @@ -250,3 +250,64 @@ def test_two_jobs_of_one_name_keep_before_and_new(self) -> None: metrics, page = self.compare_jobs(os.path.join(self.jobs, "a"), "trial", os.path.join(self.jobs, "b"), "trial") self.assertEqual([r["label"] for r in metrics["runs"]], ["before", "new"]) self.assertIn("before", page) + + +def priced(name: str, task: str, cost: Optional[float], test: str) -> Trial: + t = trial(name, task, 0.0) + t.cost = cost + t.files = {"verifier/ctrf.json": '{"results": {"tests": [{"name": "%s", "status": "failed"}]}}' % test} + return t + + +class Prices(LoopCase): + """The price of an option is the recorded cost of each of its tasks, times the trials each task gets.""" + + def compare_groups(self, new_failures: List[Trial]) -> Tuple[dict, str]: + tasks = sorted({t.task for t in new_failures}) + before = [trial(f"{t.split('/')[-1]}__b", t, 1.0) for t in tasks] + [trial(f"p{i}__b", f"bench/p{i}", 1.0) for i in range(5)] + new = list(new_failures) + [trial(f"p{i}__n", f"bench/p{i}", 1.0) for i in range(5)] + before_dir = write_job(self.jobs, "before", "alpha", "harbor", before) + new_dir = write_job(self.jobs, "new", "alpha", "harbor", new) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(before_dir, None) + r = run(["compare", "--loop", self.loop, "--new", new_dir, "--against", "run"]) + self.assertEqual(r.code, 0, r.out + r.err) + return load(self.round_file("result.json"))["stop_rule"], r.out + + def test_the_cheapest_option_is_the_lowest_priced_group(self) -> None: + costly = [priced(f"a{i}__n", f"bench/a{i}", 0.10, "test_a") for i in range(4)] + cheap = [priced(f"b{i}__n", f"bench/b{i}", 0.02, "test_b") for i in range(3)] + rule, _ = self.compare_groups(costly + cheap) + self.assertEqual(rule["groups_of_3_or_more"], 2) + option = rule["cheapest_option"] + self.assertEqual(option["group"], "g02") + self.assertAlmostEqual(option["usd"], 3 * 0.02 * 3) + self.assertEqual(option["unpriced_tasks"], 0) + + def test_a_task_without_a_cost_takes_the_mean_of_the_others(self) -> None: + group = [priced("a0__n", "bench/a0", 0.02, "test_a"), priced("a1__n", "bench/a1", 0.04, "test_a"), + priced("a2__n", "bench/a2", None, "test_a")] + rule, out = self.compare_groups(group) + option = rule["cheapest_option"] + self.assertAlmostEqual(option["usd"], (0.02 + 0.04 + 0.03) * 3) + self.assertEqual(option["unpriced_tasks"], 1) + self.assertIn("1 task priced at the mean of the others", out) + + def test_no_cost_at_all_leaves_the_price_unknown(self) -> None: + group = [priced(f"a{i}__n", f"bench/a{i}", None, "test_a") for i in range(3)] + rule, _ = self.compare_groups(group) + self.assertIsNone(rule["cheapest_option"]["usd"]) + self.assertTrue(any("budget" in n for n in rule["not_checked"])) + + +class ResultBeforeLedger(LoopCase): + def test_a_result_that_cannot_be_written_leaves_the_ledger_open(self) -> None: + before, new = paired(5, 1, 1, 1) + before_dir = write_job(self.jobs, "before", "alpha", "harbor", before) + new_dir = write_job(self.jobs, "new", "alpha", "harbor", new) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(before_dir, None) + os.makedirs(self.round_file("result.json.tmp")) + r = run(["compare", "--loop", self.loop, "--new", new_dir, "--against", "run"]) + self.assertNotEqual(r.code, 0) + self.assertIsNone(load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["result"]) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index 0c4925a7..910bbc13 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -35,9 +35,11 @@ def select(self, extra: List[str]) -> Result: def trial_job(self) -> str: trials = [] - for task, passes in (("f1", 3), ("f2", 2), ("p1", 3)): + for task, passes, cost in (("f1", 3, 0.04), ("f2", 2, 0.06), ("p1", 3, 0.01)): for k in range(3): - trials.append(trial(f"{task}-{k}__t", f"bench/{task}", 1.0 if k < passes else 0.0)) + t = trial(f"{task}-{k}__t", f"bench/{task}", 1.0 if k < passes else 0.0) + t.cost = cost + trials.append(t) return write_job(self.jobs, "trial", "alpha", "harbor", trials) def chosen(self) -> str: @@ -101,9 +103,9 @@ def test_the_proposal_prices_each_size(self) -> None: self.kept() r = self.select([]) self.assertEqual(r.code, 0, r.out + r.err) - self.assertIn("targets: 2 tasks, 6 trials, $0.06", r.out) - self.assertIn("targets and guards: 3 tasks, 9 trials, $0.09", r.out) - self.assertIn("targets, guards and canaries: 4 tasks, 12 trials, $0.12", r.out) + self.assertIn("targets: 2 tasks, 6 trials, $0.36", r.out) + self.assertIn("targets and guards: 3 tasks, 9 trials, $0.39", r.out) + self.assertIn("targets, guards and canaries: 4 tasks, 12 trials, $0.42", r.out) self.assertFalse(os.path.exists(self.round_file("trial-tasks.txt"))) def test_a_ref_without_a_kept_candidate_is_refused(self) -> None: @@ -123,7 +125,7 @@ def test_choosing_a_size_writes_the_run_and_starts_nothing(self) -> None: sel = entry["selection"] self.assertEqual((sel["targets"], sel["guards"], sel["canaries"]), (["bench/f1", "bench/f2"], ["bench/p1"], [])) self.assertEqual((sel["repeat"], sel["budget_usd"], sel["ref"]), (3, 0.5, self.repo.good)) - self.assertAlmostEqual(sel["price_usd"], 0.09) + self.assertAlmostEqual(sel["price_usd"], 0.39) self.assertEqual(entry["answers"][-1]["text"], words) build = load(self.round_file("build.json")) self.assertTrue(os.path.isfile(build["path"])) @@ -138,7 +140,7 @@ def test_a_budget_under_the_price_is_refused(self) -> None: self.kept() r = self.select(["--size", "guards", "--budget-usd", "0.05", "--text", "five cents"]) self.assertNotEqual(r.code, 0) - self.assertIn("$0.09", r.err) + self.assertIn("$0.39", r.err) self.assertFalse(os.path.exists(self.round_file("trial-tasks.txt"))) def test_a_budget_over_the_round_s_is_refused(self) -> None: @@ -172,18 +174,18 @@ def test_the_page_gives_three_options_with_their_cost(self) -> None: r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job]) self.assertEqual(r.code, 0, r.out + r.err) trial_record = load(self.round_file("trial.json")) - self.assertAlmostEqual(trial_record["cost_usd"], 0.09) - self.assertAlmostEqual(trial_record["budget_left_usd"], 9.91) + self.assertAlmostEqual(trial_record["cost_usd"], 0.33) + self.assertAlmostEqual(trial_record["budget_left_usd"], 9.67) self.assertEqual(trial_record["targets"]["before"], {"trials": 2, "passed": 0}) self.assertEqual(trial_record["targets"]["after"], {"trials": 6, "passed": 5}) self.assertEqual(trial_record["guards"], {"trials": 3, "passed": 3}) options = {o["name"]: o for o in trial_record["options"]} self.assertEqual(set(options), {"stop", "rerun", "full"}) self.assertAlmostEqual(options["stop"]["usd"], 0.0) - self.assertAlmostEqual(options["rerun"]["usd"], 0.09) - self.assertAlmostEqual(options["full"]["usd"], 0.06) + self.assertAlmostEqual(options["rerun"]["usd"], 0.33) + self.assertAlmostEqual(options["full"]["usd"], 0.16) page = self.page("decision-2.md") - for text in ("$9.91", "$0.09", "$0.06", "5 of 6", "agent-"): + for text in ("$9.67", "$0.33", "$0.16", "5 of 6", "agent-"): self.assertIn(text, page) self.assertNotIn("{{", page) for name in ("run_tasks-rerun.sh", "run_tasks-full.sh"): @@ -219,8 +221,8 @@ def test_the_stop_rule_counts_the_trial_run_and_the_new_run(self) -> None: r = run(["compare", "--loop", self.loop, "--new", full, "--against", "run"]) self.assertEqual(r.code, 0, r.out + r.err) rule = load(self.round_file("result.json"))["stop_rule"] - self.assertAlmostEqual(rule["spent_usd"], 0.09 + 0.06) - self.assertAlmostEqual(rule["left_usd"], 10 - 0.15) + self.assertAlmostEqual(rule["spent_usd"], 0.33 + 0.06) + self.assertAlmostEqual(rule["left_usd"], 10 - 0.39) class Reruns(Flow): @@ -243,30 +245,33 @@ def after_rerun(self) -> str: def test_the_second_decision_counts_both_trial_runs(self) -> None: self.after_rerun() record = load(self.round_file("trial.json")) - self.assertAlmostEqual(record["cost_usd"], 0.18) - self.assertAlmostEqual(record["budget_left_usd"], 9.82) + self.assertAlmostEqual(record["cost_usd"], 0.42) + self.assertAlmostEqual(record["budget_left_usd"], 9.58) + options = {o["name"]: o for o in record["options"]} + self.assertAlmostEqual(options["rerun"]["usd"], (0.025 + 0.035 + 0.01) * 3) + self.assertAlmostEqual(options["full"]["usd"], 0.025 + 0.035 + 0.01 + 0.01 + 0.01 + 0.03) self.assertEqual(record["targets"]["after"], {"trials": 12, "passed": 11}) self.assertEqual(record["guards"], {"trials": 6, "passed": 6}) runs = load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["trial_runs"] self.assertEqual([os.path.basename(t["job_dir"]) for t in runs], ["trial", "rerun"]) - self.assertIn("$9.82", self.page("decision-2.md")) + self.assertIn("$9.58", self.page("decision-2.md")) self.assertIn("11 of 12", self.page("decision-2.md")) with open(self.round_file("commands/run_tasks-full.sh"), encoding="utf-8") as fh: - self.assertIn("--cap 9.82", fh.read()) + self.assertIn("--cap 9.58", fh.read()) def test_deciding_twice_on_one_trial_run_counts_it_once(self) -> None: first = self.chosen() for _ in range(2): run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", first]) self.assertEqual(len(load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["trial_runs"]), 1) - self.assertAlmostEqual(load(self.round_file("trial.json"))["cost_usd"], 0.09) + self.assertAlmostEqual(load(self.round_file("trial.json"))["cost_usd"], 0.33) def test_select_after_a_trial_run_checks_the_budget_left(self) -> None: first = self.chosen() run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", first]) r = self.select(["--size", "targets", "--budget-usd", "9.95", "--text", "again"]) self.assertNotEqual(r.code, 0) - self.assertIn("$9.91", r.err) + self.assertIn("$9.67", r.err) def test_compare_counts_every_trial_run(self) -> None: self.after_rerun() @@ -275,10 +280,26 @@ def test_compare_counts_every_trial_run(self) -> None: r = run(["compare", "--loop", self.loop, "--new", full, "--against", "run"]) self.assertEqual(r.code, 0, r.out + r.err) rule = load(self.round_file("result.json"))["stop_rule"] - self.assertAlmostEqual(rule["spent_usd"], 0.09 + 0.09 + 0.06) + self.assertAlmostEqual(rule["spent_usd"], 0.33 + 0.09 + 0.06) def test_compare_of_a_trial_run_counts_it_once(self) -> None: second = self.after_rerun() r = run(["compare", "--loop", self.loop, "--new", second, "--against", "run"]) self.assertEqual(r.code, 0, r.out + r.err) - self.assertAlmostEqual(load(self.round_file("result.json"))["stop_rule"]["spent_usd"], 0.18) + self.assertAlmostEqual(load(self.round_file("result.json"))["stop_rule"]["spent_usd"], 0.42) + + +class TrialJobSpec(Flow): + def test_a_trial_job_named_inside_a_folder_of_several_jobs(self) -> None: + self.chosen() + holder = os.path.join(self.jobs, "holder") + os.makedirs(holder) + self.jobs, outer = holder, self.jobs + job = self.trial_job() + write_job(holder, "other", "alpha", "harbor", [trial("x__o", "bench/x", 1.0)]) + self.jobs = outer + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", f"{holder}::trial"]) + self.assertEqual(r.code, 0, r.out + r.err) + record = load(self.round_file("trial.json")) + self.assertEqual(record["job_dir"], job) + self.assertEqual(record["trials"], 9) From 548fc8f316566b10159db0d67887c0a96d774023 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:46:59 +0100 Subject: [PATCH 25/92] fix(run-to-fix): price each task of a run by its recorded cost A run of chosen tasks costs the sum over its tasks of the mean recorded cost of each task's trials, times the trials it gets. The cost comes from this round's trial runs on the same commit when they hold the task, else from the run that was read; compare prices from the new run. A task with no cost takes the mean of the others, and the line counts such tasks. select, decide --stop 2 and the stop rule share one function, and the stop rule names its lowest-priced group. decide --stop 2 reads the job folder that health found, so PATH::JOB works. compare writes result.json before the ledger entry. stats.py loses standard_error, which had no caller. --- .../skills/run-to-fix/assets/decision-2.md | 3 +- .../skills/run-to-fix/scripts/comparing.py | 30 +++++----- .claude/skills/run-to-fix/scripts/deciding.py | 23 ++++---- .claude/skills/run-to-fix/scripts/pricing.py | 56 +++++++++++++++++++ .claude/skills/run-to-fix/scripts/reading.py | 3 +- .../skills/run-to-fix/scripts/selecting.py | 22 ++++---- .claude/skills/run-to-fix/scripts/stats.py | 5 -- 7 files changed, 100 insertions(+), 42 deletions(-) create mode 100644 .claude/skills/run-to-fix/scripts/pricing.py diff --git a/.claude/skills/run-to-fix/assets/decision-2.md b/.claude/skills/run-to-fix/assets/decision-2.md index 7d4d32c2..74932c33 100644 --- a/.claude/skills/run-to-fix/assets/decision-2.md +++ b/.claude/skills/run-to-fix/assets/decision-2.md @@ -11,7 +11,8 @@ Build: {{BUILD}}. Trial runs on it: {{TRIAL_JOB}}, {{TRIALS}} trials. Spent in t |---|---|---|---| {{OPTIONS}} -Prices use the trial run's cost per trial, {{PER_TRIAL}}. +Each task is priced at the mean cost of its trials in this round's trial runs on this build, else in the run that +was read, times the trials it gets. A task with no recorded cost takes the mean of the others. ## Answer diff --git a/.claude/skills/run-to-fix/scripts/comparing.py b/.claude/skills/run-to-fix/scripts/comparing.py index 351d6444..11835d88 100644 --- a/.claude/skills/run-to-fix/scripts/comparing.py +++ b/.claude/skills/run-to-fix/scripts/comparing.py @@ -9,18 +9,15 @@ from reading import adapter_of, analyze, health from rounds import (LoopError, call_runs, ledger_round, load_ledger, now, open_round, read_input, read_json, save_ledger, write_json) +from pricing import describe, price_of, task_costs, usd from stats import binomial_two_sided, fisher_two_sided, verdict from trials import TrialRow TARGET_TRIALS = 3 GROUP_MIN_TRIALS = 3 STOP_RULE = ("No new round when the verdict is noise and no group of failing trials of the new run has 3 trials " - "or more, or when the budget left is under the price of the cheapest option: a trial run of the tasks " - "of the largest such group, 3 trials each, at the new run's mean cost per trial.") - - -def usd(value: float) -> str: - return f"-${-value:.2f}" if value < 0 else f"${value:.2f}" + "or more, or when the budget left is under the price of the cheapest option: the lowest-priced trial run " + "of the tasks of one such group, 3 trials each, each task at the mean cost of its trials in the new run.") def page_labels(before_job: str, new_job: str) -> Tuple[str, str]: @@ -82,20 +79,23 @@ def spent_in_round(entry: dict, record: dict, new_dir: str, new_cost: Optional[f def stop_rule(result_verdict: str, groups: List[dict], budget: float, spent: Optional[float], - cost_per_trial: Optional[float]) -> dict: + costs: Dict[str, float]) -> dict: big = [g for g in groups if g["trials"] >= GROUP_MIN_TRIALS] reasons, not_checked = [], [] if result_verdict == "noise" and not big: reasons.append(f"the verdict is noise and no group of the new run has {GROUP_MIN_TRIALS} failing trials or more") cheapest = None if big: - price = big[0]["tasks"] * TARGET_TRIALS * cost_per_trial if cost_per_trial is not None else None - cheapest = {"what": f"a trial run of the {big[0]['tasks']} tasks of {big[0]['id']}, {TARGET_TRIALS} trials each", - "usd": price} + priced = [(g, price_of(g["task_names"], TARGET_TRIALS, [costs])) for g in big] + known = [(g, p) for g, p in priced if p.usd is not None] + group, price = min(known, key=lambda gp: gp[1].usd) if known else priced[0] + cheapest = {"group": group["id"], "what": f"a trial run of the {price.tasks} tasks of {group['id']}, " + f"{TARGET_TRIALS} trials each", "usd": price.usd, "trials": price.trials, + "unpriced_tasks": price.unpriced_tasks, "text": describe(price)} left = budget - spent if spent is not None else None if cheapest is not None: if cheapest["usd"] is None or left is None: - not_checked.append("the budget: the new run reports no cost") + not_checked.append("the budget: no trial of the groups' tasks, or of the new run, reports a cost") elif left < cheapest["usd"]: reasons.append(f"the budget left, {usd(left)}, is under the price of the cheapest option, " f"{usd(cheapest['usd'])}") @@ -138,7 +138,7 @@ def cmd_compare(args: argparse.Namespace) -> None: classes, signatures = classes_of(new_rows, adapter.limits) groups = failure_groups(new_rows, classes, signatures, None) spent = spent_in_round(entry, record, new_dir, runs["new"]["cost_total"]) - rule = stop_rule(result_verdict, groups, float(record["budget_usd"]), spent, runs["new"]["cost_mean"]) + rule = stop_rule(result_verdict, groups, float(record["budget_usd"]), spent, task_costs(new_rows)) result = { "schema": "run-to-fix.result/v1", "created": now(), "round": os.path.basename(round_dir), "counts": "tasks for pairs; a task is solved when at least half of its trials pass. Trials for targets.", @@ -149,15 +149,15 @@ def cmd_compare(args: argparse.Namespace) -> None: "test": {"name": "exact two-sided binomial test on won and lost tasks", "p": p}, "verdict": result_verdict, "targets": targets_of(entry.get("selection"), before_rows, new_rows), "predictions": predictions_of(entry.get("predictions") or [], new_rows), - "groups": [{k: g[k] for k in ("id", "class", "signature", "trials", "tasks")} for g in groups], + "groups": [{k: g[k] for k in ("id", "class", "signature", "trials", "tasks", "task_names")} for g in groups], "stop_rule": rule, } report = os.path.join(round_dir, "report.html") call_runs(["render", metrics_path, "--out", report]) + write_json(os.path.join(round_dir, "result.json"), result) entry["result"] = {"verdict": result_verdict, "p": p, "won": won, "lost": lost, "stop": rule["stop"], "reasons": rule["reasons"], "file": os.path.join(os.path.basename(round_dir), "result.json")} save_ledger(args.loop, ledger) - write_json(os.path.join(round_dir, "result.json"), result) for label in ("before", "new"): r = result["pass_rate"][label] @@ -168,6 +168,8 @@ def cmd_compare(args: argparse.Namespace) -> None: f"neither {pr['neither']}") print(f"verdict: {result_verdict} (exact two-sided binomial test on won and lost, p = {p:.3g})") print(f"targets: {result['targets']['text']}") + if rule["cheapest_option"] is not None: + print(f"cheapest option: {rule['cheapest_option']['what']}, {rule['cheapest_option']['text']}") print("stop rule: " + ("stop: " + "; ".join(rule["reasons"]) if rule["stop"] else "go on") + ("" if not rule["not_checked"] else "; not checked: " + "; ".join(rule["not_checked"]))) print(f"wrote {round_dir}: compare-metrics.json, result.json, report.html (its prose slots are open: fill them " diff --git a/.claude/skills/run-to-fix/scripts/deciding.py b/.claude/skills/run-to-fix/scripts/deciding.py index df513668..6b6c95b7 100644 --- a/.claude/skills/run-to-fix/scripts/deciding.py +++ b/.claude/skills/run-to-fix/scripts/deciding.py @@ -7,7 +7,8 @@ from typing import Dict, List, Optional from classifying import read_rows -from comparing import TARGET_TRIALS, passes, targets_of, trial_spent, usd +from comparing import TARGET_TRIALS, passes, targets_of, trial_spent +from pricing import describe, price_of, task_costs, usd from reading import adapter_of, health from rounds import LoopError, ledger_round, load_ledger, now, open_round, read_input, read_json, save_ledger, write_json, write_paid from trials import TrialRow @@ -101,7 +102,7 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: if checked.agents != [adapter.agent]: raise LoopError(f"adapter.json is for the agent {adapter.agent!r}, and the trial run names " + ", ".join(repr(a) for a in checked.agents)) - job_dir = os.path.abspath(args.trial_job) + job_dir = checked.dir this_run: List[TrialRow] = read_rows(job_dir, threshold) costs = [r.cost_usd for r in this_run] runs = [t for t in entry.setdefault("trial_runs", []) if t["job_dir"] != job_dir] @@ -117,12 +118,11 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: targets = targets_of(selection, before, rows) of = lambda tasks: passes([r for r in rows if r.task in set(tasks)]) spent = trial_spent(entry) - cost_on_ref = sum(t["cost_usd"] for t in same) - per_trial = cost_on_ref / len(rows) if rows else None + sources = [task_costs(rows), task_costs(before)] left = float(record["budget_usd"]) - spent rerun_tasks = selection["targets"] + selection["guards"] full_tasks = sorted({r.task for r in before}) - options = [{"name": "stop", "what": "nothing", "usd": 0.0, "command": None}] + options = [{"name": "stop", "what": "nothing", "usd": 0.0, "unpriced_tasks": 0, "text": usd(0.0), "command": None}] for name, tasks, repeat in (("rerun", rerun_tasks, TARGET_TRIALS), ("full", full_tasks, 1)): units = os.path.join(round_dir, f"{name}-tasks.txt") with open(units, "w", encoding="utf-8", newline="\n") as fh: @@ -137,14 +137,15 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: adapter.checkout) what = (f"the {len(tasks)} targets and guards again, {repeat} trials each" if name == "rerun" else f"a full pass of the {len(tasks)} tasks of the run, {repeat} trial each") - options.append({"name": name, "what": what, "usd": len(tasks) * repeat * per_trial if per_trial is not None else None, - "command": command}) + price = price_of(tasks, repeat, sources) + options.append({"name": name, "what": what, "usd": price.usd, "unpriced_tasks": price.unpriced_tasks, + "text": describe(price), "command": command}) build = read_json(os.path.join(round_dir, "build.json")) trial_record = { "schema": "run-to-fix.trial/v1", "created": now(), "job_dir": job_dir, "counts": "cost_usd is every trial run of the round; trials and the results are the trial runs on this ref", "trial_runs": runs, "trials": len(rows), "cost_usd": spent, - "cost_complete": all(t["cost_complete"] for t in runs), "cost_per_trial_usd": per_trial, + "cost_complete": all(t["cost_complete"] for t in runs), "budget_usd": float(record["budget_usd"]), "budget_left_usd": left, "targets": targets, "guards": of(selection["guards"]), "canaries": of(selection["canaries"]), "options": options, "build": build, @@ -156,7 +157,7 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: "| targets | none | none | none |"] for name in ("guards", "canaries"): results.append(f"| {name} ({len(selection[name])}) | | {share(trial_record[name])} | |") - option_rows = [f"| {o['name']} | {o['what']} | {usd(o['usd']) if o['usd'] is not None else 'unknown'} | " + option_rows = [f"| {o['name']} | {o['what']} | {o['text']} | " f"{cell(o['command']) if o['command'] else 'none'} |" for o in options] fill_page("decision-2.md", { "ROUND": os.path.basename(round_dir), "LOOP": args.loop, @@ -164,11 +165,11 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: "TRIAL_JOB": ", ".join(t["job_dir"] for t in same), "TRIALS": str(len(rows)), "SPENT": usd(spent) + ("" if trial_record["cost_complete"] else " at least: some trials report no cost"), "LEFT": usd(left), "BUDGET": usd(float(record["budget_usd"])), "RESULTS": "\n".join(results), - "OPTIONS": "\n".join(option_rows), "PER_TRIAL": usd(per_trial) if per_trial is not None else "unknown", + "OPTIONS": "\n".join(option_rows), }, os.path.join(round_dir, "decision-2.md")) print(f"wrote {round_dir}: trial.json, decision-2.md; budget left {usd(left)}") for o in options: - print(f" {o['name']}: {o['what']}, " + (usd(o["usd"]) if o["usd"] is not None else "price unknown")) + print(f" {o['name']}: {o['what']}, {o['text']}") def cmd_decide(args: argparse.Namespace) -> None: diff --git a/.claude/skills/run-to-fix/scripts/pricing.py b/.claude/skills/run-to-fix/scripts/pricing.py new file mode 100644 index 00000000..89e5f26d --- /dev/null +++ b/.claude/skills/run-to-fix/scripts/pricing.py @@ -0,0 +1,56 @@ +"""The price of a run of chosen tasks, from what each task's trials cost when they were recorded.""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Dict, List, Optional + +from trials import TrialRow + + +@dataclass(frozen=True) +class Price: + usd: Optional[float] + tasks: int + trials: int + unpriced_tasks: int + + +def task_costs(rows: List[TrialRow]) -> Dict[str, float]: + """The mean cost of each task's trials, over the trials that report a cost.""" + by_task: Dict[str, List[float]] = {} + for r in rows: + if r.cost_usd is not None: + by_task.setdefault(r.task, []).append(r.cost_usd) + return {t: sum(v) / len(v) for t, v in by_task.items()} + + +def price_of(tasks: List[str], trials_each: int, sources: List[Dict[str, float]]) -> Price: + """Each task at its cost in the first source that holds it, times its trials. A task no source holds takes the + mean of the priced tasks; with none priced, the price is unknown.""" + known: Dict[str, float] = {} + for t in tasks: + found = next((s[t] for s in sources if t in s), None) + if found is not None: + known[t] = found + n = len(tasks) + if not tasks: + return Price(usd=0.0, tasks=0, trials=0, unpriced_tasks=0) + if not known: + return Price(usd=None, tasks=n, trials=n * trials_each, unpriced_tasks=n) + mean = sum(known.values()) / len(known) + return Price(usd=sum(known.get(t, mean) for t in tasks) * trials_each, tasks=n, trials=n * trials_each, + unpriced_tasks=n - len(known)) + + +def usd(value: float) -> str: + return f"-${-value:.2f}" if value < 0 else f"${value:.2f}" + + +def describe(price: Price) -> str: + if price.usd is None: + return "price unknown: no trial of these tasks reports a cost" + text = usd(price.usd) + if price.unpriced_tasks: + noun = "task" if price.unpriced_tasks == 1 else "tasks" + text += f", {price.unpriced_tasks} {noun} priced at the mean of the others" + return text diff --git a/.claude/skills/run-to-fix/scripts/reading.py b/.claude/skills/run-to-fix/scripts/reading.py index 0e14ceb9..368c228d 100644 --- a/.claude/skills/run-to-fix/scripts/reading.py +++ b/.claude/skills/run-to-fix/scripts/reading.py @@ -40,6 +40,7 @@ def full_spec(spec: str) -> str: class Health: text: str job: str + dir: str agents: List[str] refused: List[str] @@ -62,7 +63,7 @@ def health(label: str, spec: str, threshold: float) -> Health: refused.append(f"{label}: {v}") refused += [f"{label}: e.g. {example}" for message, example in (job.get("error_examples") or {}).items() if example and message in v] - return Health(text=text, job=str(job.get("job") or ""), agents=list(job.get("agents") or []), refused=refused) + return Health(text=text, job=str(job.get("job") or ""), dir=str(job.get("dir") or ""), agents=list(job.get("agents") or []), refused=refused) def analyze(runs: List[Tuple[str, str]], threshold: float, out: str) -> str: diff --git a/.claude/skills/run-to-fix/scripts/selecting.py b/.claude/skills/run-to-fix/scripts/selecting.py index 88f99fcd..e77565e5 100644 --- a/.claude/skills/run-to-fix/scripts/selecting.py +++ b/.claude/skills/run-to-fix/scripts/selecting.py @@ -10,7 +10,8 @@ from adapter import Adapter from checkout import is_ancestor, resolve, worktree, write_paid_on_ref from classifying import read_rows -from comparing import TARGET_TRIALS, trial_spent, usd +from comparing import TARGET_TRIALS, trial_spent +from pricing import describe, price_of, task_costs, usd from reading import adapter_of from rounds import (LoopError, ledger_round, load_ledger, now, open_round, read_input, read_json, run_command, save_ledger, write_json, write_paid) @@ -88,19 +89,18 @@ def cmd_select(args: argparse.Namespace) -> None: guards, no_guards = guards_of(kept, {r.trial: (r.task, r.passed) for r in rows}, targets) pool = sorted(t for t, ok in solved.items() if ok and t not in targets and t not in guards) canaries = sorted(random.Random(SEED).sample(pool, min(args.canaries, len(pool)))) - cost = read_json(os.path.join(round_dir, "metrics.json")) - per_trial = next(r for r in cost["runs"] if r["label"] == "run")["cost_mean"] sizes = {"targets": targets, "guards": targets + guards, "canaries": targets + guards + canaries} names = {"targets": "targets", "guards": "targets and guards", "canaries": "targets, guards and canaries"} - price = {k: (len(v) * TARGET_TRIALS * per_trial if per_trial is not None else None) for k, v in sizes.items()} + threshold = float(record["pass_threshold"]) + on_ref = [r for t in entry.get("trial_runs") or [] if t["ref"] == ref for r in read_rows(t["job_dir"], threshold)] + sources = [task_costs(on_ref), task_costs(rows)] + price = {k: price_of(v, TARGET_TRIALS, sources) for k, v in sizes.items()} print(f"targets ({len(targets)} tasks of the kept causes): {', '.join(targets)}") print(f"guards: {', '.join(guards)}" if guards else f"guards: none ({no_guards or 'no passing task ran the changed code'})") print(f"canaries ({len(canaries)} passing tasks, seed {SEED}): {', '.join(canaries) or 'none'}") for k in SIZES: - n = len(sizes[k]) - print(f"{names[k]}: {n} tasks, {n * TARGET_TRIALS} trials, " - + (usd(price[k]) if price[k] is not None else "price unknown: the run reports no cost")) + print(f"{names[k]}: {price[k].tasks} tasks, {price[k].trials} trials, {describe(price[k])}") if args.size is None: print("choose a size with --size targets|guards|canaries --budget-usd B --text \"the person's words\"") return @@ -111,8 +111,9 @@ def cmd_select(args: argparse.Namespace) -> None: if args.budget_usd > left: raise LoopError(f"the budget of the trial run, {usd(args.budget_usd)}, is over what the round has left, " f"{usd(left)} of {usd(float(record['budget_usd']))}") - if price[args.size] is not None and price[args.size] > args.budget_usd: - raise LoopError(f"{names[args.size]} cost {usd(price[args.size])}, over the budget of {usd(args.budget_usd)}") + chosen = price[args.size] + if chosen.usd is not None and chosen.usd > args.budget_usd: + raise LoopError(f"{names[args.size]} cost {describe(chosen)}, over the budget of {usd(args.budget_usd)}") tasks = sizes[args.size] units = os.path.join(round_dir, "trial-tasks.txt") with open(units, "w", encoding="utf-8", newline="\n") as fh: @@ -122,7 +123,8 @@ def cmd_select(args: argparse.Namespace) -> None: entry["selection"] = {"ref": ref, "size": args.size, "targets": targets, "guards": guards if args.size != "targets" else [], "canaries": canaries if args.size == "canaries" else [], "repeat": TARGET_TRIALS, - "seed": SEED, "price_usd": price[args.size], "budget_usd": args.budget_usd, + "seed": SEED, "price_usd": chosen.usd, "unpriced_tasks": chosen.unpriced_tasks, + "budget_usd": args.budget_usd, "guards_reason": no_guards} entry["answers"].append({"about": "the trial run: its size and budget", "date": now(), "text": args.text}) line = adapter.commands["run_tasks"] diff --git a/.claude/skills/run-to-fix/scripts/stats.py b/.claude/skills/run-to-fix/scripts/stats.py index d81c8f2e..3b7c6dfc 100644 --- a/.claude/skills/run-to-fix/scripts/stats.py +++ b/.claude/skills/run-to-fix/scripts/stats.py @@ -2,17 +2,12 @@ from __future__ import annotations import math -from typing import Optional ALPHA = 0.05 #: probabilities within this relative distance of the observed table's count as equally extreme TIE = 1e-7 -def standard_error(rate: float, n: int) -> Optional[float]: - return math.sqrt(rate * (1 - rate) / n) if n else None - - def binomial_two_sided(won: int, lost: int) -> float: """p of an exact two-sided binomial test of won against lost, with even odds.""" n = won + lost From 25d9f958f33a8c1eda68047e845c3289a04dea32 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:46:59 +0100 Subject: [PATCH 26/92] test(run-to-fix): pin which braces in a command line are placeholders --- .claude/skills/run-to-fix/tests/test_read.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_read.py b/.claude/skills/run-to-fix/tests/test_read.py index f27819af..062fbbe2 100644 --- a/.claude/skills/run-to-fix/tests/test_read.py +++ b/.claude/skills/run-to-fix/tests/test_read.py @@ -166,3 +166,19 @@ def test_names_the_missing_skill(self) -> None: self.assertNotEqual(p.returncode, 0) self.assertIn("run-analysis", p.stderr) self.assertIn("runs.py", p.stderr) + + +class ShellBraces(LoopCase): + """A {word} of lower-case letters is a placeholder; a shell variable in upper case reaches the shell.""" + + def test_an_upper_case_shell_variable_is_left_to_the_shell(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + self.set_adapter(adapter("alpha", NO_LIMITS, {"classes": "make-classes ${HOME} {job_dir} {out_json}"})) + self.read_ok(run_dir, None) + + def test_a_lower_case_shell_variable_is_refused(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + self.set_adapter(adapter("alpha", NO_LIMITS, {"classes": "make-classes ${home} {job_dir} {out_json}"})) + r = self.read(run_dir, None) + self.assertNotEqual(r.code, 0) + self.assertIn("{home}", r.err) From 895aaef0f144e75a682b60ad313b6d9d54835cff Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:47:36 +0100 Subject: [PATCH 27/92] docs(run-to-fix): describe the price of a run, the shell the loop needs, and braces in commands --- .claude/skills/run-to-fix/SKILL.md | 19 ++++++++++++------- .../skills/run-to-fix/references/adapter.md | 5 ++++- .claude/skills/run-to-fix/references/files.md | 9 +++++++++ 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/.claude/skills/run-to-fix/SKILL.md b/.claude/skills/run-to-fix/SKILL.md index d1cd76ac..34914582 100644 --- a/.claude/skills/run-to-fix/SKILL.md +++ b/.claude/skills/run-to-fix/SKILL.md @@ -18,7 +18,8 @@ causes, writing tests and fixes in the agent's code. - The `run-analysis` skill in a folder beside this one. `loop.py` calls its `runs.py` for `health`, `analyze` and `render`, and stops with a message naming the missing file when it is absent. -- Python 3 with the standard library, and git. The tests of this skill ran on Python 3.12. +- Python 3 with the standard library, git, and bash on a POSIX system: Linux or WSL. The tests of this skill ran + on Python 3.12. - A loop folder the person names, outside the repository or in a folder git ignores. `loop.py` writes only there. - An adapter, `adapter.json` in the loop folder: one file per agent with its checkout, its commands and the limits of the run (`references/adapter.md`). For an agent of the Nullius catalogue, draft it from @@ -95,10 +96,13 @@ Merge the kept candidates into one commit of the work branch. Then: python3 L select --loop DIR --ref COMMIT --canaries N ``` -prints the trial run in three sizes with their price from the cost per trial of the run that was read: the targets -(the tasks the kept causes predict), plus the guards (passing tasks whose recorded trial ran the changed code, when -the replay reports `touched`), plus the canaries (passing tasks drawn with a fixed seed). Every task gets 3 trials. The -person picks a size and names its budget, which cannot exceed what the round has left: +prints the trial run in three sizes with their price: the targets (the tasks the kept causes predict), plus the +guards (passing tasks whose recorded trial ran the changed code, when the replay reports `touched`), plus the canaries +(passing tasks drawn with a fixed seed). Every task gets 3 trials. A price is the sum over the tasks of the mean +recorded cost of each task's trials, times its trials. The cost comes from this round's trial runs on the same commit +when they hold the task, else from the run that was read. A task with no recorded cost takes the mean of the others, +and the printed line counts such tasks. The person picks a size and names its budget, which cannot exceed what the +round has left: ```bash python3 L select --loop DIR --ref COMMIT --canaries N --size targets|guards|canaries --budget-usd B --text "..." @@ -137,8 +141,9 @@ closes the round and writes `report.html` with `render` of `run-analysis`; fill describes (`runs.py render --slots`, then `runs.py check`). The stop rule is fixed before a round starts. No new round when the verdict is noise and no group of failing trials -of the new run has 3 trials or more, or when the budget left is under the price of the cheapest option, a trial run -of the largest such group's tasks. +of the new run has 3 trials or more, or when the budget left is under the price of the cheapest option. That option +is the lowest-priced trial run of one such group's tasks, 3 trials each, each task at the mean cost of its trials in +the new run; `result.json` names the group. ## When the adapter lacks a command diff --git a/.claude/skills/run-to-fix/references/adapter.md b/.claude/skills/run-to-fix/references/adapter.md index 20d669b9..67dc6387 100644 --- a/.claude/skills/run-to-fix/references/adapter.md +++ b/.claude/skills/run-to-fix/references/adapter.md @@ -49,7 +49,10 @@ Every key is present, and a value the agent does not have is null. `read` refuse | `{build}` | `run_tasks` | the path `commands.build` printed; empty when `commands.build` is null | A command is one line of POSIX shell. The loop replaces each placeholder with its value quoted for the shell. -Write each placeholder without quotes around it. When the adapter is read, the loop refuses a placeholder that a +Write each placeholder without quotes around it. A `{word}` of lower-case letters and `_` is always read as a +placeholder. A command line therefore holds no such word other than its placeholders: a shell variable written +`${home}` is refused as an unknown placeholder. Write `$home` instead, or a variable in upper case such as `${HOME}`; +both reach the shell unchanged. When the adapter is read, the loop refuses a placeholder that a command does not take, and a command that leaves out one it needs: | command | needs | diff --git a/.claude/skills/run-to-fix/references/files.md b/.claude/skills/run-to-fix/references/files.md index 3ea7966b..1082701c 100644 --- a/.claude/skills/run-to-fix/references/files.md +++ b/.claude/skills/run-to-fix/references/files.md @@ -82,6 +82,15 @@ Every answer of the person is stored with its date and in the person's words: th size and budget of the trial run, the choice at stop 2, and the words that start a round after a stop rule said stop. The money a round has spent is the sum of its `trial_runs` plus the new run that `compare` reads. +## Prices + +A price of a run of chosen tasks is the sum over the tasks of the mean recorded cost of each task's trials, times +the trials the task gets. `select` and `decide --stop 2` take that cost from this round's trial runs on the same +commit when they hold the task, else from the run that was read; the stop rule of `compare` takes it from the new +run. A task with no recorded cost takes the mean of the priced tasks of the same run, and the price says how many +tasks it priced that way. With no priced task, the price is unknown. `selection.price_usd` of the ledger, the options +of `trial.json` and `cheapest_option` of `result.json` hold such prices. + ## Files of a run that the loop reads | file | used for | From 0476f1fc6dd781b176ba9a7f8742738379e4d15a Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:01:58 +0100 Subject: [PATCH 28/92] test(run-to-fix): refuse a loop folder that git would list as untracked --- .claude/skills/run-to-fix/tests/test_read.py | 39 ++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_read.py b/.claude/skills/run-to-fix/tests/test_read.py index 062fbbe2..72966c77 100644 --- a/.claude/skills/run-to-fix/tests/test_read.py +++ b/.claude/skills/run-to-fix/tests/test_read.py @@ -182,3 +182,42 @@ def test_a_lower_case_shell_variable_is_refused(self) -> None: r = self.read(run_dir, None) self.assertNotEqual(r.code, 0) self.assertIn("{home}", r.err) + + +class LoopFolderInARepository(LoopCase): + """read refuses a loop folder that git would list as untracked, before it writes anything.""" + + def setUp(self) -> None: + super().setUp() + self.repo = os.path.join(self.tmp, "repo") + os.makedirs(self.repo) + subprocess.run(["git", "init", "-q", self.repo], check=True, capture_output=True) + self.run_dir = healthy(self.jobs, "run", "alpha") + + def loop_at(self, path: str) -> None: + os.makedirs(path, exist_ok=True) + self.loop = path + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + + def test_a_folder_git_does_not_ignore_is_refused(self) -> None: + self.loop_at(os.path.join(self.repo, "loops", "agent")) + r = self.read(self.run_dir, None) + self.assertNotEqual(r.code, 0) + self.assertIn(os.path.realpath(self.repo), r.err) + self.assertIn("outside the repository", r.err) + self.assertIn(".gitignore", r.err) + self.assertEqual(sorted(os.listdir(self.loop)), ["adapter.json"]) + + def test_a_folder_git_ignores_is_read(self) -> None: + self.loop_at(os.path.join(self.repo, "loops", "agent")) + with open(os.path.join(self.repo, ".gitignore"), "w", encoding="utf-8") as fh: + fh.write("loops/\n") + self.read_ok(self.run_dir, None) + self.assertTrue(os.path.isfile(self.round_file("input.json"))) + + def test_the_repository_root_is_refused(self) -> None: + self.loop_at(self.repo) + r = self.read(self.run_dir, None) + self.assertNotEqual(r.code, 0) + self.assertIn(os.path.realpath(self.repo), r.err) + self.assertFalse(os.path.exists(os.path.join(self.repo, "round-01"))) From f8acef24c0be65055fc81b7d12ac0ec839ec88c0 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:17:44 +0100 Subject: [PATCH 29/92] fix(run-to-fix): refuse a loop folder that git would list as untracked read asks git, before it writes anything, whether the loop folder lies in a work tree and whether that repository ignores it. A folder outside every repository, or an ignored one, is read; any other is refused with the repository root and the two ways out. Any other answer of git stops read with git's output. --- .claude/skills/run-to-fix/scripts/reading.py | 33 ++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.claude/skills/run-to-fix/scripts/reading.py b/.claude/skills/run-to-fix/scripts/reading.py index 368c228d..6b410a25 100644 --- a/.claude/skills/run-to-fix/scripts/reading.py +++ b/.claude/skills/run-to-fix/scripts/reading.py @@ -4,6 +4,7 @@ import argparse import json import os +import subprocess from dataclasses import dataclass from typing import List, Optional, Tuple @@ -16,6 +17,37 @@ "the job never wrote a finish time") +def git_in(cwd: str, args: List[str]) -> "subprocess.CompletedProcess[str]": + try: + return subprocess.run(["git", "-C", cwd] + args, capture_output=True, text=True, stdin=subprocess.DEVNULL) + except FileNotFoundError as exc: + raise LoopError("git is needed to check the loop folder, and it is not on PATH") from exc + + +def guard_loop_folder(loop_dir: str) -> None: + """Refuse a loop folder inside a git work tree that git does not ignore: its round files would show as + untracked, and round pages hold host paths.""" + path = os.path.realpath(loop_dir) + top = git_in(path, ["rev-parse", "--show-toplevel"]) + if top.returncode == 128 and "not a git repository" in top.stderr: + return + if top.returncode != 0: + raise LoopError(f"git rev-parse --show-toplevel in {path} exited with code {top.returncode}: " + f"{(top.stderr or top.stdout).strip()}") + root = os.path.realpath(top.stdout.strip()) + if root != path: + rel = os.path.relpath(path, root).replace(os.sep, "/") + "/" + ignored = git_in(root, ["check-ignore", "-q", "--", rel]) + if ignored.returncode == 0: + return + if ignored.returncode != 1: + raise LoopError(f"git check-ignore {rel} in {root} exited with code {ignored.returncode}: " + f"{(ignored.stderr or ignored.stdout).strip()}") + raise LoopError(f"the loop folder {path} lies inside the git repository {root}, and git does not ignore it: " + "every round file would show as untracked, and round pages hold host paths. Name a loop folder " + f"outside the repository, or add a line for it to {root}/.gitignore.") + + def adapter_of(loop_dir: str) -> Adapter: return load_adapter(os.path.join(loop_dir, "adapter.json")) @@ -90,6 +122,7 @@ def gate(loop_dir: str, words: Optional[str]) -> Optional[dict]: def cmd_read(args: argparse.Namespace) -> None: + guard_loop_folder(args.loop) adapter = adapter_of(args.loop) runs = runs_py() specs = [("run", args.run)] + ([("baseline", args.baseline)] if args.baseline else []) From 3b8423911d1722521f469a87d1e030f1371342e1 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:17:45 +0100 Subject: [PATCH 30/92] docs(run-to-fix): say where a loop folder may lie and how read checks it --- .claude/skills/run-to-fix/SKILL.md | 3 ++- .claude/skills/run-to-fix/references/files.md | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/SKILL.md b/.claude/skills/run-to-fix/SKILL.md index 34914582..c5f7d45a 100644 --- a/.claude/skills/run-to-fix/SKILL.md +++ b/.claude/skills/run-to-fix/SKILL.md @@ -20,7 +20,8 @@ causes, writing tests and fixes in the agent's code. `render`, and stops with a message naming the missing file when it is absent. - Python 3 with the standard library, git, and bash on a POSIX system: Linux or WSL. The tests of this skill ran on Python 3.12. -- A loop folder the person names, outside the repository or in a folder git ignores. `loop.py` writes only there. +- A loop folder the person names, outside every git repository or in a folder its repository ignores. `loop.py` + writes only there, and `read` refuses a loop folder that git would list as untracked. - An adapter, `adapter.json` in the loop folder: one file per agent with its checkout, its commands and the limits of the run (`references/adapter.md`). For an agent of the Nullius catalogue, draft it from `catalogue/agents//agent.json` (`versions..upstream.repository` and `tag`) and the run's settings, and diff --git a/.claude/skills/run-to-fix/references/files.md b/.claude/skills/run-to-fix/references/files.md index 1082701c..8dc9f762 100644 --- a/.claude/skills/run-to-fix/references/files.md +++ b/.claude/skills/run-to-fix/references/files.md @@ -3,6 +3,12 @@ A person names the loop folder. `loop.py` writes only there, plus worktree records in the agent checkout's git folder while a check runs. Tests and fixes are commits that the assistant makes on `source.work_branch`. +Before it writes anything, `read` asks git where the loop folder lies (`git rev-parse --show-toplevel`, then +`git check-ignore`). A folder outside every repository, or one its repository ignores, is read. A folder inside a +work tree that git does not ignore, the repository root included, is refused, and the message names the repository +root and the two ways out: a folder outside the repository, or a line in its `.gitignore`. Any other answer of git +stops `read` with git's output. The rule holds for every repository, the agent's checkout included. + ## The loop folder | file | written by | holds | From b280125fb6a19d07cbeb1606c4f432e5b98565f2 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:26:50 +0100 Subject: [PATCH 31/92] test(run-to-fix): read a loop folder outside a repository under a translated git --- .claude/skills/run-to-fix/tests/test_read.py | 31 ++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_read.py b/.claude/skills/run-to-fix/tests/test_read.py index 72966c77..4fad58c4 100644 --- a/.claude/skills/run-to-fix/tests/test_read.py +++ b/.claude/skills/run-to-fix/tests/test_read.py @@ -221,3 +221,34 @@ def test_the_repository_root_is_refused(self) -> None: self.assertNotEqual(r.code, 0) self.assertIn(os.path.realpath(self.repo), r.err) self.assertFalse(os.path.exists(os.path.join(self.repo, "round-01"))) + + +STUB_GIT = """#!/bin/sh +if [ "$LC_ALL" = "C" ]; then + echo "fatal: not a git repository (or any of the parent directories): .git" >&2 +else + echo "fatal: ni ceci ni aucun de ses repertoires parents n'est un depot git : .git" >&2 +fi +exit 128 +""" + + +class LoopFolderWithATranslatedGit(LoopCase): + """The guard reads git's English text, so it runs git in the C locale.""" + + def test_a_folder_outside_a_repository_passes_under_a_french_locale(self) -> None: + bin_dir = os.path.join(self.tmp, "bin") + os.makedirs(bin_dir) + stub = os.path.join(bin_dir, "git") + with open(stub, "w", encoding="utf-8", newline="\n") as fh: + fh.write(STUB_GIT) + os.chmod(stub, 0o755) + run_dir = healthy(self.jobs, "run", "alpha") + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1", LC_ALL="fr_FR.UTF-8", LANG="fr_FR.UTF-8", + PATH=bin_dir + os.pathsep + os.environ.get("PATH", "")) + p = subprocess.run([sys.executable, LOOP, "read", "--loop", self.loop, "--run", run_dir, "--budget-usd", "1", + "--pass-threshold", "1"], capture_output=True, text=True, env=env, + stdin=subprocess.DEVNULL, timeout=120) + self.assertEqual(p.returncode, 0, p.stdout + p.stderr) + self.assertTrue(os.path.isfile(self.round_file("input.json"))) From 8499bf426fff41ec48797c397a804089e1928baf Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Wed, 18 Nov 2026 22:27:22 +0100 Subject: [PATCH 32/92] fix(run-to-fix): run git in the C locale when read checks the loop folder The guard reads the English text 'not a git repository'. A translated git under another locale prints it in that language, and read then refused every loop folder outside a repository. --- .claude/skills/run-to-fix/scripts/reading.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/scripts/reading.py b/.claude/skills/run-to-fix/scripts/reading.py index 6b410a25..abdcbb19 100644 --- a/.claude/skills/run-to-fix/scripts/reading.py +++ b/.claude/skills/run-to-fix/scripts/reading.py @@ -18,8 +18,10 @@ def git_in(cwd: str, args: List[str]) -> "subprocess.CompletedProcess[str]": + """git in the C locale: the guard reads git's English text, which a translated git prints in another language.""" try: - return subprocess.run(["git", "-C", cwd] + args, capture_output=True, text=True, stdin=subprocess.DEVNULL) + return subprocess.run(["git", "-C", cwd] + args, capture_output=True, text=True, stdin=subprocess.DEVNULL, + env=dict(os.environ, LC_ALL="C")) except FileNotFoundError as exc: raise LoopError("git is needed to check the loop folder, and it is not on PATH") from exc From 5aab4c54e507994eeb99d80a4210b67dd83ec438 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:05:35 +0100 Subject: [PATCH 33/92] test(run-to-fix): replay before the fix, and the effect a candidate predicts A candidate's table compares the replay at its test commit with the replay at its fix. candidate takes --effect outcome or report; a report candidate predicts no pass, and select guards the passing trials whose label it changes. The replay may take {base_ref} and {before_ref}. --- .claude/skills/run-to-fix/tests/agentrepo.py | 19 ++- .claude/skills/run-to-fix/tests/test_check.py | 147 +++++++++++++++--- .../skills/run-to-fix/tests/test_decide.py | 87 ++++++++++- .claude/skills/run-to-fix/tests/test_read.py | 16 ++ 4 files changed, 235 insertions(+), 34 deletions(-) diff --git a/.claude/skills/run-to-fix/tests/agentrepo.py b/.claude/skills/run-to-fix/tests/agentrepo.py index 7efcd6b2..a0da2afb 100644 --- a/.claude/skills/run-to-fix/tests/agentrepo.py +++ b/.claude/skills/run-to-fix/tests/agentrepo.py @@ -41,9 +41,11 @@ BASE_LABELS = {"p1__r": {"label": "done"}, "p2__r": {"label": "done"}, "p3__r": {"label": "not done"}, "f1__r": {"label": "done"}, "f2__r": {"label": "done"}, "f3__r": {"label": "not done"}} GOOD_LABELS = {"p1__r": {"label": "done", "touched": True}, "p2__r": {"label": "done", "touched": False}, - "p3__r": {"label": "done", "touched": False}, "f1__r": {"label": "not done", "touched": True}, + "p3__r": {"label": "not done", "touched": False}, "f1__r": {"label": "not done", "touched": True}, "f2__r": {"label": "not done", "touched": True}, "f3__r": {"label": "not done", "touched": False}} BAD_LABELS = dict(GOOD_LABELS, f3__r={"label": "done", "touched": True}) +#: a fix that changes only what the agent reports on the passing trial p3 +REPORT_LABELS = dict(BASE_LABELS, p3__r={"label": "done", "touched": True}) def git(repo: str, args: List[str]) -> str: @@ -71,6 +73,7 @@ class Repo: test_ref: str good: str bad: str + report: str def make_repo(root: str) -> Repo: @@ -83,8 +86,11 @@ def make_repo(root: str) -> Repo: good = commit(path, {"agent.py": agent_file(True, False), "labels.json": json.dumps(GOOD_LABELS)}, "fix: good") git(path, ["checkout", "-q", "--detach", test_ref]) bad = commit(path, {"agent.py": agent_file(True, True), "labels.json": json.dumps(BAD_LABELS)}, "fix: bad") + git(path, ["checkout", "-q", "--detach", test_ref]) + report = commit(path, {"agent.py": agent_file(True, False), "labels.json": json.dumps(REPORT_LABELS)}, + "fix: report") git(path, ["checkout", "-q", "main"]) - return Repo(path=path, base=base, test_ref=test_ref, good=good, bad=bad) + return Repo(path=path, base=base, test_ref=test_ref, good=good, bad=bad, report=report) #: what each recorded trial cost; the failed ones cost more, as in real runs @@ -138,9 +144,12 @@ def start(self, commands: Dict[str, Optional[str]], paid: List[str]) -> None: write(os.path.join(self.loop, "round-01", "causes", "state.md"), "# The agent reports done in the wrong state\n\nRule: g01\n\nTrials: 2 (f1__r, f2__r).\n") - def candidate(self, cid: str, ref: str, test_ref: str) -> None: - r = run(["candidate", "--loop", self.loop, "--id", cid, "--cause", "state", "--ref", ref, - "--test", "tests/test_fix.py", "--test-ref", test_ref]) + def candidate(self, cid: str, ref: str, test_ref: str, effect: str) -> None: + self.candidate_of("state", cid, ref, test_ref, effect) + + def candidate_of(self, cause: str, cid: str, ref: str, test_ref: str, effect: str) -> None: + r = run(["candidate", "--loop", self.loop, "--id", cid, "--cause", cause, "--ref", ref, + "--test", "tests/test_fix.py", "--test-ref", test_ref, "--effect", effect]) self.assertEqual(r.code, 0, r.out + r.err) def check(self, cid: str) -> dict: diff --git a/.claude/skills/run-to-fix/tests/test_check.py b/.claude/skills/run-to-fix/tests/test_check.py index f1fb52a2..6cfc1b3b 100644 --- a/.claude/skills/run-to-fix/tests/test_check.py +++ b/.claude/skills/run-to-fix/tests/test_check.py @@ -1,6 +1,7 @@ """`loop.py candidate` and `loop.py check`: the test, the suite and the replay of a candidate, in worktrees.""" from __future__ import annotations +import json import os from agentrepo import RepoCase, git @@ -10,18 +11,19 @@ class Candidate(RepoCase): def test_records_refs_and_the_prediction_of_its_cause(self) -> None: self.start(self.commands(), ["run_tasks"]) - self.candidate("good", self.repo.good[:8], self.repo.test_ref[:8]) + self.candidate("good", self.repo.good[:8], self.repo.test_ref[:8], "outcome") data = load(self.round_file("candidates/good/candidate.json")) self.assertEqual((data["ref"], data["test_ref"]), (self.repo.good, self.repo.test_ref)) self.assertEqual(data["cause"], "state") self.assertEqual(data["rule"], "g01") + self.assertEqual(data["effect"], "outcome") self.assertEqual(data["prediction"], {"trials": ["f1__r", "f2__r"], "tasks": ["bench/f1", "bench/f2"]}) def test_a_cause_without_a_rule_is_refused(self) -> None: self.start(self.commands(), ["run_tasks"]) write(self.round_file("causes/vague.md"), "# Something is wrong\n") r = run(["candidate", "--loop", self.loop, "--id", "c1", "--cause", "vague", "--ref", self.repo.good, - "--test", "tests/test_fix.py", "--test-ref", self.repo.test_ref]) + "--test", "tests/test_fix.py", "--test-ref", self.repo.test_ref, "--effect", "outcome"]) self.assertNotEqual(r.code, 0) self.assertIn("Rule:", r.err) @@ -29,14 +31,14 @@ def test_a_rule_that_counts_nothing_is_refused(self) -> None: self.start(self.commands(), ["run_tasks"]) write(self.round_file("causes/ghost.md"), "# A cause\n\nRule: g09\n") r = run(["candidate", "--loop", self.loop, "--id", "c1", "--cause", "ghost", "--ref", self.repo.good, - "--test", "tests/test_fix.py", "--test-ref", self.repo.test_ref]) + "--test", "tests/test_fix.py", "--test-ref", self.repo.test_ref, "--effect", "outcome"]) self.assertNotEqual(r.code, 0) self.assertIn("g09", r.err) def test_an_unknown_ref_is_refused(self) -> None: self.start(self.commands(), ["run_tasks"]) r = run(["candidate", "--loop", self.loop, "--id", "c1", "--cause", "state", "--ref", "no-such-ref", - "--test", "tests/test_fix.py", "--test-ref", self.repo.test_ref]) + "--test", "tests/test_fix.py", "--test-ref", self.repo.test_ref, "--effect", "outcome"]) self.assertNotEqual(r.code, 0) self.assertIn("no-such-ref", r.err) @@ -44,7 +46,7 @@ def test_an_unknown_ref_is_refused(self) -> None: class Check(RepoCase): def test_a_good_candidate(self) -> None: self.start(self.commands(), ["run_tasks"]) - self.candidate("good", self.repo.good, self.repo.test_ref) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") checks = self.check("good") self.assertEqual(checks["test"]["ran"], True) self.assertTrue(checks["test"]["fails_before"]) @@ -54,14 +56,16 @@ def test_a_good_candidate(self) -> None: replay = checks["replay"] self.assertTrue(replay["ran"]) self.assertEqual(replay["before"], {"done_pass": 2, "done_fail": 2, "not_done_pass": 1, "not_done_fail": 1}) - self.assertEqual(replay["after"], {"done_pass": 3, "done_fail": 0, "not_done_pass": 0, "not_done_fail": 3}) + self.assertEqual(replay["after"], {"done_pass": 2, "done_fail": 0, "not_done_pass": 1, "not_done_fail": 3}) self.assertEqual(replay["false_success_added"], -2) + self.assertEqual(replay["cause"], {"trials": 2, "changed": 2, "by_direction": {"done → not done": 2}}) + self.assertEqual(replay["changed_trials"], ["f1__r", "f2__r"]) self.assertEqual(replay["touched"], ["f1__r", "f2__r", "p1__r"]) self.assertEqual(checks["diff"]["files"], 2) def test_a_candidate_that_adds_failures_and_false_successes(self) -> None: self.start(self.commands(), ["run_tasks"]) - self.candidate("bad", self.repo.bad, self.repo.test_ref) + self.candidate("bad", self.repo.bad, self.repo.test_ref, "outcome") checks = self.check("bad") self.assertEqual(checks["suite"]["added"], ["test_other"]) self.assertEqual(checks["replay"]["after"]["done_fail"], 1) @@ -70,22 +74,22 @@ def test_a_candidate_that_adds_failures_and_false_successes(self) -> None: def test_a_test_that_does_not_fail_before_the_fix(self) -> None: self.start(self.commands(), ["run_tasks"]) - self.candidate("weak", self.repo.good, self.repo.good) + self.candidate("weak", self.repo.good, self.repo.good, "outcome") checks = self.check("weak") self.assertFalse(checks["test"]["fails_before"]) - def test_the_base_is_checked_once_per_round(self) -> None: + def test_the_ref_before_the_fix_is_replayed_once_per_round(self) -> None: self.start(self.commands(), ["run_tasks"]) - self.candidate("good", self.repo.good, self.repo.test_ref) - self.candidate("bad", self.repo.bad, self.repo.test_ref) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") + self.candidate("bad", self.repo.bad, self.repo.test_ref, "outcome") self.check("good") self.check("bad") - self.assertEqual(self.marker("replay.marker"), [self.repo.base, self.repo.good, self.repo.bad]) + self.assertEqual(self.marker("replay.marker"), [self.repo.test_ref, self.repo.good, self.repo.bad]) def test_worktrees_are_removed_and_the_checkout_is_untouched(self) -> None: self.start(self.commands(), ["run_tasks"]) head = git(self.repo.path, ["rev-parse", "HEAD"]) - self.candidate("good", self.repo.good, self.repo.test_ref) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") self.check("good") self.assertEqual(git(self.repo.path, ["worktree", "list", "--porcelain"]).count("worktree "), 1) self.assertEqual(os.listdir(os.path.join(self.loop, "worktrees")), []) @@ -96,7 +100,7 @@ def test_a_failing_replay_still_removes_its_worktree(self) -> None: commands = self.commands() commands["replay"] = "exit 4 # {worktree} {ref} {job_dir} {out_json}" self.start(commands, ["run_tasks"]) - self.candidate("good", self.repo.good, self.repo.test_ref) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") r = run(["check", "--loop", self.loop, "--candidate", "good"]) self.assertNotEqual(r.code, 0) self.assertIn("commands.replay", r.err) @@ -108,23 +112,23 @@ class CheckFive(RepoCase): def test_a_replay_of_two_refs_gives_the_table_and_the_false_results(self) -> None: self.start(self.commands(), ["run_tasks"]) - self.candidate("bad", self.repo.bad, self.repo.test_ref) + self.candidate("bad", self.repo.bad, self.repo.test_ref, "outcome") replay = self.check("bad")["replay"] - self.assertEqual(replay["base_ref"], self.repo.base) + self.assertEqual(replay["before_ref"], self.repo.test_ref) self.assertEqual(replay["ref"], self.repo.bad) self.assertEqual(replay["before"], {"done_pass": 2, "done_fail": 2, "not_done_pass": 1, "not_done_fail": 1}) - self.assertEqual(replay["after"], {"done_pass": 3, "done_fail": 1, "not_done_pass": 0, "not_done_fail": 2}) + self.assertEqual(replay["after"], {"done_pass": 2, "done_fail": 1, "not_done_pass": 1, "not_done_fail": 2}) self.assertEqual((replay["false_success_before"], replay["false_success_after"]), (2, 1)) def test_a_paid_command_is_written_and_not_started(self) -> None: self.start(self.commands(), ["run_tasks", "replay"]) - self.candidate("good", self.repo.good, self.repo.test_ref) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") replay = self.check("good")["replay"] self.assertFalse(replay["ran"]) self.assertIn("costs money", replay["reason"]) self.assertEqual(self.marker("replay.marker"), []) written = sorted(os.listdir(self.round_file("commands"))) - self.assertEqual(written, sorted([f"replay-{self.repo.base[:12]}.sh", f"replay-{self.repo.good[:12]}.sh"])) + self.assertEqual(written, sorted([f"replay-{self.repo.test_ref[:12]}.sh", f"replay-{self.repo.good[:12]}.sh"])) with open(self.round_file(os.path.join("commands", f"replay-{self.repo.good[:12]}.sh")), encoding="utf-8") as fh: script = fh.read() self.assertIn(self.repo.good, script) @@ -136,7 +140,7 @@ def test_a_null_command_is_a_check_that_did_not_run(self) -> None: commands["test"] = None commands["replay"] = None self.start(commands, ["run_tasks"]) - self.candidate("good", self.repo.good, self.repo.test_ref) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") checks = self.check("good") self.assertFalse(checks["suite"]["ran"]) self.assertIn("commands.test is null", checks["suite"]["reason"]) @@ -156,8 +160,109 @@ def test_a_replay_that_leaves_out_a_trial_stops(self) -> None: commands["replay"] = (commands["replay"] or "") + " && python3 -c \"import json,sys; p=sys.argv[1]; " \ "d=json.load(open(p)); d['trials'].pop('f3__r'); json.dump(d, open(p, 'w'))\" {out_json}" self.start(commands, ["run_tasks"]) - self.candidate("good", self.repo.good, self.repo.test_ref) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") r = run(["check", "--loop", self.loop, "--candidate", "good"]) self.assertNotEqual(r.code, 0) self.assertIn("f3__r", r.err) + + +REPORT_RULE = {"id": "p3_reports", "meaning": "The trial p3, whose report the report fix changes.", + "file": "result.json", "regex": "\"trial_name\": \"p3__r\""} + + +class Effects(RepoCase): + """A candidate says which effect its fix has: failing trials turn into passes, or the agent's report changes.""" + + def with_report_cause(self, rule: dict) -> None: + write(os.path.join(self.loop, "found-classes.json"), + json.dumps({"schema": "run-to-fix.found-classes/v1", "rules": [rule]})) + r = run(["classify", "--loop", self.loop]) + self.assertEqual(r.code, 0, r.out + r.err) + write(self.round_file("causes/report.md"), f"# The agent reports p3 wrongly\n\nRule: {rule['id']}\n") + + def test_a_report_candidate_predicts_no_pass(self) -> None: + self.start(self.commands(), ["run_tasks"]) + self.with_report_cause(REPORT_RULE) + self.candidate_of("report", "rep", self.repo.report, self.repo.test_ref, "report") + data = load(self.round_file("candidates/rep/candidate.json")) + self.assertEqual(data["effect"], "report") + self.assertEqual(data["prediction"], {"trials": ["p3__r"], "tasks": []}) + replay = self.check("rep")["replay"] + self.assertEqual(replay["cause"], {"trials": 1, "changed": 1, "by_direction": {"not done → done": 1}}) + self.assertEqual(replay["changed_trials"], ["p3__r"]) + + def test_an_outcome_candidate_on_passing_trials_is_refused(self) -> None: + self.start(self.commands(), ["run_tasks"]) + self.with_report_cause(REPORT_RULE) + r = run(["candidate", "--loop", self.loop, "--id", "rep", "--cause", "report", "--ref", self.repo.report, + "--test", "tests/test_fix.py", "--test-ref", self.repo.test_ref, "--effect", "outcome"]) + self.assertNotEqual(r.code, 0) + self.assertIn("counts no failing trial", r.err) + + def test_a_report_candidate_needs_the_replay(self) -> None: + commands = self.commands() + commands["replay"] = None + self.start(commands, ["run_tasks"]) + self.with_report_cause(REPORT_RULE) + r = run(["candidate", "--loop", self.loop, "--id", "rep", "--cause", "report", "--ref", self.repo.report, + "--test", "tests/test_fix.py", "--test-ref", self.repo.test_ref, "--effect", "report"]) + self.assertNotEqual(r.code, 0) + self.assertIn("commands.replay", r.err) + + def test_a_report_candidate_on_a_rule_that_matches_nothing(self) -> None: + self.start(self.commands(), ["run_tasks"]) + self.with_report_cause(dict(REPORT_RULE, regex="no-such-trial")) + r = run(["candidate", "--loop", self.loop, "--id", "rep", "--cause", "report", "--ref", self.repo.report, + "--test", "tests/test_fix.py", "--test-ref", self.repo.test_ref, "--effect", "report"]) + self.assertNotEqual(r.code, 0) + self.assertIn("matches no trial", r.err) + + def test_the_effect_has_no_default(self) -> None: + self.start(self.commands(), ["run_tasks"]) + r = run(["candidate", "--loop", self.loop, "--id", "c1", "--cause", "state", "--ref", self.repo.good, + "--test", "tests/test_fix.py", "--test-ref", self.repo.test_ref]) + self.assertNotEqual(r.code, 0) + self.assertIn("--effect", r.err) + + +class RefsOfTheTable(RepoCase): + def test_a_candidate_without_a_test_is_compared_with_the_base(self) -> None: + self.start(self.commands(), ["run_tasks"]) + r = run(["candidate", "--loop", self.loop, "--id", "notest", "--cause", "state", "--ref", self.repo.good, + "--effect", "outcome"]) + self.assertEqual(r.code, 0, r.out + r.err) + replay = self.check("notest")["replay"] + self.assertEqual((replay["before_ref"], replay["ref"]), (self.repo.base, self.repo.good)) + + def test_the_replay_gets_the_base_and_the_ref_before(self) -> None: + commands = self.commands() + commands["replay"] = (commands["replay"] or "") + f" && echo {{base_ref}} {{before_ref}} {{ref}} >> {self.tmp}/args.marker" + self.start(commands, ["run_tasks"]) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") + self.check("good") + with open(os.path.join(self.tmp, "args.marker"), encoding="utf-8") as fh: + lines = fh.read().splitlines() + self.assertEqual(lines, [f"{self.repo.base} {self.repo.test_ref} {self.repo.test_ref}", + f"{self.repo.base} {self.repo.test_ref} {self.repo.good}"]) + + def test_registering_again_drops_the_old_checks(self) -> None: + self.start(self.commands(), ["run_tasks"]) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") + self.check("good") + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") + self.assertFalse(os.path.exists(self.round_file("candidates/good/checks.json"))) + + +class OldCandidate(RepoCase): + def test_a_candidate_without_an_effect_is_refused(self) -> None: + self.start(self.commands(), ["run_tasks"]) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") + path = self.round_file("candidates/good/candidate.json") + data = load(path) + del data["effect"] + write(path, json.dumps(data)) + for args in (["check", "--loop", self.loop, "--candidate", "good"], ["decide", "--loop", self.loop, "--stop", "1"]): + r = run(args) + self.assertNotEqual(r.code, 0, args) + self.assertIn("register the candidate again", r.err) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index 910bbc13..90f60c11 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -5,7 +5,7 @@ from typing import List from agentrepo import RepoCase -from jobs import Result, load, run, trial, write_job +from jobs import Result, load, run, trial, write, write_job, write_json KEEP_GOOD = "Keep good. bad breaks test_other." @@ -15,8 +15,8 @@ class Flow(RepoCase): def checked(self) -> None: self.start(self.commands(), ["run_tasks"]) - self.candidate("good", self.repo.good, self.repo.test_ref) - self.candidate("bad", self.repo.bad, self.repo.test_ref) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") + self.candidate("bad", self.repo.bad, self.repo.test_ref, "outcome") self.check("good") self.check("bad") @@ -59,8 +59,11 @@ def test_one_row_per_candidate(self) -> None: self.assertEqual(len(rows), 2) good = next(ln for ln in rows if ln.startswith("| good |")) bad = next(ln for ln in rows if ln.startswith("| bad |")) - self.assertIn("2 / 2 / 1 / 1 → 3 / 0 / 0 / 3", good) - self.assertIn("2 / 2 / 1 / 1 → 3 / 1 / 0 / 2", bad) + self.assertIn("2 / 2 / 1 / 1 → 2 / 0 / 1 / 3", good) + self.assertIn("2 / 2 / 1 / 1 → 2 / 1 / 1 / 2", bad) + self.assertIn(f"{self.repo.test_ref[:12]} → {self.repo.good[:12]}", good) + self.assertIn("2 of 2: done → not done 2", good) + self.assertIn("| outcome |", good) self.assertIn("test_other", bad) self.assertIn("f3__r", bad) self.assertIn("fails, then passes", good) @@ -75,8 +78,9 @@ def test_the_answer_is_stored_in_the_person_s_words(self) -> None: self.assertEqual(entry["answers"][0]["text"], KEEP_GOOD) self.assertEqual(entry["answers"][0]["about"], "stop 1: the candidates to keep") self.assertEqual(entry["kept"], ["good"]) - self.assertEqual(entry["predictions"], [{"candidate": "good", "cause": "state", "rule": "g01", - "tasks": ["bench/f1", "bench/f2"], "expect": "pass"}]) + self.assertEqual(entry["predictions"], [{"candidate": "good", "cause": "state", "rule": "g01", "effect": "outcome", + "trials": ["f1__r", "f2__r"], "tasks": ["bench/f1", "bench/f2"], + "expect": "pass"}]) def test_an_answer_before_the_page_is_refused(self) -> None: self.checked() @@ -158,7 +162,7 @@ def test_without_touched_there_are_no_guards(self) -> None: commands = self.commands() commands["replay"] = None self.start(commands, ["run_tasks"]) - self.candidate("good", self.repo.good, self.repo.test_ref) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") self.check("good") run(["decide", "--loop", self.loop, "--stop", "1"]) run(["answer", "--loop", self.loop, "--stop", "1", "--keep", "good", "--text", "keep good"]) @@ -303,3 +307,70 @@ def test_a_trial_job_named_inside_a_folder_of_several_jobs(self) -> None: record = load(self.round_file("trial.json")) self.assertEqual(record["job_dir"], job) self.assertEqual(record["trials"], 9) + + +class NoLabelChange(Flow): + def test_a_fix_that_changes_no_label_of_its_cause_has_a_line_of_its_own(self) -> None: + self.start(self.commands(), ["run_tasks"]) + self.candidate("weak", self.repo.good, self.repo.good, "outcome") + self.check("weak") + r = run(["decide", "--loop", self.loop, "--stop", "1"]) + self.assertEqual(r.code, 0, r.out + r.err) + page = self.page("decision-1.md") + self.assertIn("## Candidates whose fix changes no label of their cause", page) + self.assertIn(f"- weak: 0 of 2 trials of its cause change their label between {self.repo.good[:12]} and " + f"{self.repo.good[:12]}.", page) + + +REPORT_RULE = {"id": "p3_reports", "meaning": "The trial p3, whose report the report fix changes.", + "file": "result.json", "regex": "\"trial_name\": \"p3__r\""} + + +class ReportCandidate(Flow): + """A kept candidate whose fix changes what the agent reports, and no pass or fail.""" + + def kept_report(self) -> None: + self.start(self.commands(), ["run_tasks"]) + write_json(os.path.join(self.loop, "found-classes.json"), + {"schema": "run-to-fix.found-classes/v1", "rules": [REPORT_RULE]}) + r = run(["classify", "--loop", self.loop]) + self.assertEqual(r.code, 0, r.out + r.err) + write(self.round_file("causes/report.md"), "# The agent reports p3 wrongly\n\nRule: p3_reports\n") + self.candidate_of("report", "rep", self.repo.report, self.repo.test_ref, "report") + self.check("rep") + run(["decide", "--loop", self.loop, "--stop", "1"]) + r = run(["answer", "--loop", self.loop, "--stop", "1", "--keep", "rep", "--text", "keep the report fix"]) + self.assertEqual(r.code, 0, r.out + r.err) + + def select_report(self, extra: List[str]) -> Result: + return run(["select", "--loop", self.loop, "--ref", self.repo.report, "--canaries", "0"] + extra) + + def test_the_page_of_stop_one_names_the_effect(self) -> None: + self.kept_report() + row = next(ln for ln in self.page("decision-1.md").splitlines() if ln.startswith("| rep |")) + self.assertIn("| report |", row) + self.assertIn("1 of 1: not done → done 1", row) + + def test_select_takes_no_targets_and_guards_the_changed_trials(self) -> None: + self.kept_report() + r = self.select_report([]) + self.assertEqual(r.code, 0, r.out + r.err) + self.assertIn("targets (0 tasks of the kept causes)", r.out) + self.assertIn("guards: bench/p3", r.out) + r = self.select_report(["--size", "guards", "--budget-usd", "0.5", "--text", "the guards"]) + self.assertEqual(r.code, 0, r.out + r.err) + sel = load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["selection"] + self.assertEqual((sel["targets"], sel["guards"]), ([], ["bench/p3"])) + + def test_the_pages_say_the_report_is_not_read(self) -> None: + self.kept_report() + self.select_report(["--size", "guards", "--budget-usd", "0.5", "--text", "the guards"]) + job = write_job(self.jobs, "trial", "alpha", "harbor", [trial(f"p3-{k}__t", "bench/p3", 1.0) for k in range(3)]) + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job]) + self.assertEqual(r.code, 0, r.out + r.err) + self.assertIn("rep: this version does not read the agent's report in the new run", self.page("decision-2.md")) + r = run(["compare", "--loop", self.loop, "--new", job, "--against", "run"]) + self.assertEqual(r.code, 0, r.out + r.err) + pred = load(self.round_file("result.json"))["predictions"][0] + self.assertEqual(pred["effect"], "report") + self.assertIn("does not read the agent's report", pred["result"]["note"]) diff --git a/.claude/skills/run-to-fix/tests/test_read.py b/.claude/skills/run-to-fix/tests/test_read.py index 4fad58c4..bf9d0ec5 100644 --- a/.claude/skills/run-to-fix/tests/test_read.py +++ b/.claude/skills/run-to-fix/tests/test_read.py @@ -252,3 +252,19 @@ def test_a_folder_outside_a_repository_passes_under_a_french_locale(self) -> Non stdin=subprocess.DEVNULL, timeout=120) self.assertEqual(p.returncode, 0, p.stdout + p.stderr) self.assertTrue(os.path.isfile(self.round_file("input.json"))) + + +class ReplayPlaceholders(LoopCase): + def test_the_replay_may_take_the_base_and_the_ref_before(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + data = adapter("alpha", NO_LIMITS, {"replay": "r {worktree} {ref} {job_dir} {out_json} {base_ref} {before_ref}"}) + data["replay_success_labels"] = ["done"] + self.set_adapter(data) + self.read_ok(run_dir, None) + + def test_another_command_may_not_take_them(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + self.set_adapter(adapter("alpha", NO_LIMITS, {"classes": "c {job_dir} {out_json} {base_ref}"})) + r = self.read(run_dir, None) + self.assertNotEqual(r.code, 0) + self.assertIn("{base_ref}", r.err) From f8fce3033905a20a6b6aa2983a9274367d02fd6e Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:15:07 +0100 Subject: [PATCH 34/92] feat(run-to-fix): replay before the fix, and the effect a candidate predicts A candidate's table compares the replay at its test commit, else at the base, with the replay at its fix; each pair of refs is replayed once per round, and the replay may take {base_ref} and {before_ref}. candidate takes --effect outcome or report. check counts the trials of the cause whose label the fix changes, the page of stop 1 shows the count and lists the fixes that change none, select takes targets from outcome candidates and guards the passing trials whose label a kept fix changes, and the pages say that a report is not read in the new run. A candidate.json without an effect is refused. --- .../skills/run-to-fix/assets/decision-1.md | 8 +- .../skills/run-to-fix/assets/decision-2.md | 1 + .claude/skills/run-to-fix/scripts/adapter.py | 2 +- .claude/skills/run-to-fix/scripts/checking.py | 102 ++++++++++++------ .../skills/run-to-fix/scripts/comparing.py | 11 +- .claude/skills/run-to-fix/scripts/deciding.py | 40 +++++-- .claude/skills/run-to-fix/scripts/loop.py | 2 + .../skills/run-to-fix/scripts/selecting.py | 21 ++-- 8 files changed, 134 insertions(+), 53 deletions(-) diff --git a/.claude/skills/run-to-fix/assets/decision-1.md b/.claude/skills/run-to-fix/assets/decision-1.md index daf0b433..e95288cd 100644 --- a/.claude/skills/run-to-fix/assets/decision-1.md +++ b/.claude/skills/run-to-fix/assets/decision-1.md @@ -2,13 +2,17 @@ {{SUMMARY}} -| candidate | cause | trials the cause explains | test before → after the fix | suite failures added | replay before → after | false successes added | diff | -|---|---|---|---|---|---|---|---| +| candidate | cause | effect | trials of the cause | test before → after the fix | suite failures added | replay before → after (refs) | trials of the cause whose label changes | false successes added | diff | +|---|---|---|---|---|---|---|---|---|---| {{ROWS}} Replay counts are of the {{TRIALS}} recorded trials: done and passed / done and failed / not done and passed / not done and failed. A false success is a failed trial that the agent reports as a success. +## Candidates whose fix changes no label of their cause + +{{UNCHANGED}} + ## Checks that did not run {{NOT_RUN}} diff --git a/.claude/skills/run-to-fix/assets/decision-2.md b/.claude/skills/run-to-fix/assets/decision-2.md index 74932c33..ac8295bc 100644 --- a/.claude/skills/run-to-fix/assets/decision-2.md +++ b/.claude/skills/run-to-fix/assets/decision-2.md @@ -10,6 +10,7 @@ Build: {{BUILD}}. Trial runs on it: {{TRIAL_JOB}}, {{TRIALS}} trials. Spent in t | option | what runs | price | command | |---|---|---|---| {{OPTIONS}} +{{REPORT_NOTE}} Each task is priced at the mean cost of its trials in this round's trial runs on this build, else in the run that was read, times the trials it gets. A task with no recorded cost takes the mean of the others. diff --git a/.claude/skills/run-to-fix/scripts/adapter.py b/.claude/skills/run-to-fix/scripts/adapter.py index b1c42afc..a4c47c53 100644 --- a/.claude/skills/run-to-fix/scripts/adapter.py +++ b/.claude/skills/run-to-fix/scripts/adapter.py @@ -13,7 +13,7 @@ "test": frozenset({"worktree"}), "test_one": frozenset({"worktree", "test"}), "build": frozenset({"worktree"}), - "replay": frozenset({"worktree", "ref", "job_dir", "out_json"}), + "replay": frozenset({"worktree", "ref", "job_dir", "out_json", "base_ref", "before_ref"}), "classes": frozenset({"job_dir", "out_json"}), "run_tasks": frozenset({"units_file", "repeat", "jobs_dir", "budget_usd", "build", "ref"}), } diff --git a/.claude/skills/run-to-fix/scripts/checking.py b/.claude/skills/run-to-fix/scripts/checking.py index 74612ed6..e8737472 100644 --- a/.claude/skills/run-to-fix/scripts/checking.py +++ b/.claude/skills/run-to-fix/scripts/checking.py @@ -2,6 +2,7 @@ from __future__ import annotations import argparse +import collections import csv import os import re @@ -18,10 +19,12 @@ CANDIDATE_ID = re.compile(r"^[a-z0-9][a-z0-9_-]{0,39}$") RULE_LINE = re.compile(r"(?m)^Rule:\s*(\S+)\s*$") TABLE = ("done_pass", "done_fail", "not_done_pass", "not_done_fail") +EFFECTS = ("outcome", "report") -def cause_trials(round_dir: str, cause: str) -> Tuple[str, List[str], List[str]]: - """The rule a cause names and the failing trials and tasks it counts.""" +def cause_trials(round_dir: str, cause: str, effect: str) -> Tuple[str, List[str], List[str]]: + """The rule a cause names, the trials it counts and their tasks: failing trials for an effect on the outcome, + every trial the rule matches for an effect on the report.""" path = os.path.join(round_dir, "causes", f"{cause}.md") if not os.path.isfile(path): raise LoopError(f"{path} is missing: write the cause before its candidates") @@ -37,16 +40,28 @@ def cause_trials(round_dir: str, cause: str) -> Tuple[str, List[str], List[str]] if rule in groups: names = set(groups[rule]["trial_names"]) elif table and f"found:{rule}" in table[0]: - names = {r["trial"] for r in table if r[f"found:{rule}"] == "1" and r["class"] != "pass"} + names = {r["trial"] for r in table if r[f"found:{rule}"] == "1" and (effect == "report" or r["class"] != "pass")} else: raise LoopError(f"{path} names the rule {rule}, which is neither a group of groups.json nor a found class " "of trials.csv; run loop.py classify after adding a found class") if not names: - raise LoopError(f"the rule {rule} of {path} counts no failing trial") + raise LoopError(f"the rule {rule} of {path} counts no failing trial" if effect == "outcome" + else f"the rule {rule} of {path} matches no trial") tasks = sorted({r["task"] for r in table if r["trial"] in names}) return rule, sorted(names), tasks +def load_candidate(path: str) -> dict: + """A candidate.json of this version: one that names the effect of its fix.""" + if not os.path.isfile(path): + raise LoopError(f"{path} is missing; register the candidate with loop.py candidate") + cand = read_json(path) + if not isinstance(cand, dict) or cand.get("effect") not in EFFECTS: + raise LoopError(f"{path} names no effect of its fix; register the candidate again with loop.py candidate " + "--effect outcome or --effect report") + return cand + + def cmd_candidate(args: argparse.Namespace) -> None: adapter = adapter_of(args.loop) round_dir = open_round(args.loop) @@ -57,14 +72,22 @@ def cmd_candidate(args: argparse.Namespace) -> None: raise LoopError("--test and --test-ref go together: the test file and the commit that adds it before the fix") if args.test is not None and (os.path.isabs(args.test) or ".." in re.split(r"[\\/]", args.test)): raise LoopError(f"--test is a path inside the checkout, got {args.test!r}") - rule, trials, tasks = cause_trials(round_dir, args.cause) + if args.effect == "report" and adapter.commands["replay"] is None: + raise LoopError("an effect on the report is seen only in the replay, and commands.replay is null in adapter.json") + rule, trials, tasks = cause_trials(round_dir, args.cause, args.effect) ref = resolve(adapter.checkout, args.ref) test_ref = resolve(adapter.checkout, args.test_ref) if args.test_ref else None record = {"schema": "run-to-fix.candidate/v1", "id": args.id, "created": now(), "cause": args.cause, "rule": rule, - "ref": ref, "test": args.test, "test_ref": test_ref, "prediction": {"trials": trials, "tasks": tasks}} - write_json(os.path.join(round_dir, "candidates", args.id, "candidate.json"), record) - print(f"candidate {args.id}: {ref[:12]} for the cause {args.cause}, which counts {len(trials)} trials " - f"on {len(tasks)} tasks ({rule})") + "effect": args.effect, "ref": ref, "test": args.test, "test_ref": test_ref, + "prediction": {"trials": trials, "tasks": tasks if args.effect == "outcome" else []}} + cdir = os.path.join(round_dir, "candidates", args.id) + stale = os.path.join(cdir, "checks.json") + if os.path.exists(stale): + os.remove(stale) + write_json(os.path.join(cdir, "candidate.json"), record) + expects = f"{len(tasks)} tasks expected to pass" if args.effect == "outcome" else "no task expected to pass" + print(f"candidate {args.id}: {ref[:12]} for the cause {args.cause}, effect {args.effect}: {len(trials)} trials " + f"of {rule}, {expects}") @dataclass(frozen=True) @@ -164,37 +187,57 @@ def false_successes(data: dict, rows: List[TrialRow], success: List[str]) -> Lis return sorted(r.trial for r in rows if not r.passed and data["trials"][r.trial]["label"] in success) -def check_replay(adapter: Adapter, args: argparse.Namespace, round_dir: str, cdir: str, base: str, ref: str, - job_dir: str, rows: List[TrialRow]) -> dict: +def replay_file(round_dir: str, sha: str, against: str) -> str: + return os.path.join(round_dir, "replays", f"{sha[:12]}-against-{against[:12]}.json") + + +def label_changes(before: dict, after: dict, names: List[str]) -> dict: + """How many of `names` change their label between the two replays, by direction.""" + by = collections.Counter(f"{before['trials'][n]['label']} → {after['trials'][n]['label']}" for n in names + if before["trials"][n]["label"] != after["trials"][n]["label"]) + return {"trials": len(names), "changed": sum(by.values()), "by_direction": dict(sorted(by.items()))} + + +def check_replay(adapter: Adapter, args: argparse.Namespace, round_dir: str, before_ref: str, ref: str, base: str, + job_dir: str, rows: List[TrialRow], cause: List[str]) -> dict: + """The replays at the ref before the fix and at the fix, each once per round; the second is given the first as + {before_ref}, so `touched` is the code that changed between the two.""" labels = {} - for when, sha, out in (("before", base, os.path.join(round_dir, "base", "replay.json")), - ("after", ref, os.path.join(cdir, "replay.json"))): - if when == "before" and os.path.isfile(out) and read_json(out).get("_loop_ref") == base: - labels[when] = replay_labels(out, rows) - continue - o = on_ref(adapter, args.loop, round_dir, "replay", sha, {"ref": sha, "job_dir": job_dir, "out_json": out}, - os.path.join(os.path.dirname(out), "replay.log"), args.command_timeout_s) + for when, sha, against in (("before", before_ref, before_ref), ("after", ref, before_ref)): + out = replay_file(round_dir, sha, against) + if os.path.isfile(out): + stored = read_json(out) + if stored.get("_loop_ref") == sha and stored.get("_loop_before_ref") == against: + labels[when] = replay_labels(out, rows) + continue + values = {"ref": sha, "before_ref": against, "base_ref": base, "job_dir": job_dir, "out_json": out} + o = on_ref(adapter, args.loop, round_dir, "replay", sha, values, out[:-len(".json")] + ".log", + args.command_timeout_s) if o.ran is None: if when == "before": - on_ref(adapter, args.loop, round_dir, "replay", ref, {"ref": ref, "job_dir": job_dir, "out_json": - os.path.join(cdir, "replay.json")}, os.path.join(cdir, "replay.log"), args.command_timeout_s) + on_ref(adapter, args.loop, round_dir, "replay", ref, + dict(values, ref=ref, out_json=replay_file(round_dir, ref, before_ref)), + replay_file(round_dir, ref, before_ref)[:-len(".json")] + ".log", args.command_timeout_s) return {"ran": False, "reason": o.reason} ensure_ok("replay", o) data = replay_labels(out, rows) - if when == "before": - write_json(out, dict(data, _loop_ref=base)) + write_json(out, dict(data, _loop_ref=sha, _loop_before_ref=against)) labels[when] = data success = adapter.replay_success_labels before, after = false_successes(labels["before"], rows, success), false_successes(labels["after"], rows, success) touched_given = any("touched" in v for v in labels["after"]["trials"].values()) - return {"ran": True, "reason": None, "base_ref": base, "ref": ref, + names = [r.trial for r in rows] + return {"ran": True, "reason": None, "before_ref": before_ref, "ref": ref, "counts": "recorded trials: done and passed / done and failed / not done and passed / not done and failed", "before": cross(labels["before"], rows, success), "after": cross(labels["after"], rows, success), "false_success_before": len(before), "false_success_after": len(after), "false_success_added": len(after) - len(before), "false_success_new": sorted(set(after) - set(before)), "kept_recorded": {w: len(labels[w].get("kept_recorded") or []) for w in ("before", "after")}, "touched": sorted(n for n, v in labels["after"]["trials"].items() if v.get("touched") is True) - if touched_given else None} + if touched_given else None, + "changed_trials": sorted(n for n in names + if labels["before"]["trials"][n]["label"] != labels["after"]["trials"][n]["label"]), + "cause": label_changes(labels["before"], labels["after"], cause)} def cmd_check(args: argparse.Namespace) -> None: @@ -202,20 +245,19 @@ def cmd_check(args: argparse.Namespace) -> None: round_dir = open_round(args.loop) record = read_input(round_dir) cdir = os.path.join(round_dir, "candidates", args.candidate) - path = os.path.join(cdir, "candidate.json") - if not os.path.isfile(path): - raise LoopError(f"{path} is missing; register the candidate with loop.py candidate") - cand = read_json(path) + cand = load_candidate(os.path.join(cdir, "candidate.json")) job_dir = record["run"]["job_dir"] rows = read_rows(job_dir, float(record["pass_threshold"])) base = resolve(adapter.checkout, adapter.base_ref) + before_ref = cand["test_ref"] or base checks = { "schema": "run-to-fix.checks/v1", "candidate": cand["id"], "created": now(), "ref": cand["ref"], "test_ref": cand["test_ref"], "base_ref": base, "test": check_test(adapter, args, round_dir, cdir, cand), "suite": check_suite(adapter, args, round_dir, cdir, base, cand["ref"]), - "replay": check_replay(adapter, args, round_dir, cdir, base, cand["ref"], job_dir, rows), - "diff": diff_size(adapter.checkout, cand["test_ref"] or base, cand["ref"]), + "replay": check_replay(adapter, args, round_dir, before_ref, cand["ref"], base, job_dir, rows, + cand["prediction"]["trials"]), + "diff": diff_size(adapter.checkout, before_ref, cand["ref"]), } write_json(os.path.join(cdir, "checks.json"), checks) print(f"wrote {cdir}/checks.json") diff --git a/.claude/skills/run-to-fix/scripts/comparing.py b/.claude/skills/run-to-fix/scripts/comparing.py index 11835d88..d279d53c 100644 --- a/.claude/skills/run-to-fix/scripts/comparing.py +++ b/.claude/skills/run-to-fix/scripts/comparing.py @@ -14,6 +14,7 @@ from trials import TrialRow TARGET_TRIALS = 3 +REPORT_NOT_READ = "this version does not read the agent's report in the new run" GROUP_MIN_TRIALS = 3 STOP_RULE = ("No new round when the verdict is noise and no group of failing trials of the new run has 3 trials " "or more, or when the budget left is under the price of the cheapest option: the lowest-priced trial run " @@ -53,10 +54,16 @@ def targets_of(selection: Optional[dict], before: List[TrialRow], new: List[Tria def predictions_of(predictions: List[dict], new: List[TrialRow]) -> List[dict]: - """Each prediction of stage 3 beside the passes of its tasks in the new run.""" + """Each prediction of stage 3 beside its result in the new run: the passes of its tasks for an effect on the + outcome, a note for an effect on the report.""" out = [] for pred in predictions: - tasks = set(pred.get("tasks") or []) + if pred.get("effect") == "report": + out.append(dict(pred, result={"note": REPORT_NOT_READ})) + continue + if pred.get("effect") != "outcome": + raise LoopError(f"a prediction of {pred.get('candidate')} names no effect; record the answer of stop 1 again") + tasks = set(pred["tasks"]) out.append(dict(pred, result=dict(passes([r for r in new if r.task in tasks]), tasks=len(tasks)))) return out diff --git a/.claude/skills/run-to-fix/scripts/deciding.py b/.claude/skills/run-to-fix/scripts/deciding.py index 6b6c95b7..c01ae078 100644 --- a/.claude/skills/run-to-fix/scripts/deciding.py +++ b/.claude/skills/run-to-fix/scripts/deciding.py @@ -6,8 +6,9 @@ import re from typing import Dict, List, Optional +from checking import load_candidate from classifying import read_rows -from comparing import TARGET_TRIALS, passes, targets_of, trial_spent +from comparing import REPORT_NOT_READ, TARGET_TRIALS, passes, targets_of, trial_spent from pricing import describe, price_of, task_costs, usd from reading import adapter_of, health from rounds import LoopError, ledger_round, load_ledger, now, open_round, read_input, read_json, save_ledger, write_json, write_paid @@ -37,11 +38,19 @@ def table_text(t: Dict[str, int]) -> str: return f"{t['done_pass']} / {t['done_fail']} / {t['not_done_pass']} / {t['not_done_fail']}" +def changes_text(replay: dict) -> str: + changes = replay["cause"] + text = f"{changes['changed']} of {changes['trials']}" + if changes["by_direction"]: + text += ": " + ", ".join(f"{d} {n}" for d, n in changes["by_direction"].items()) + return text + + def candidate_row(cand: dict, checks: Optional[dict]) -> List[str]: pred = cand["prediction"] - explains = f"{len(pred['trials'])} trials on {len(pred['tasks'])} tasks ({cand['rule']})" + explains = f"{len(pred['trials'])} trials, {len(pred['tasks'])} tasks expected to pass ({cand['rule']})" if checks is None: - return [cand["id"], cand["cause"], explains, "not checked", "not checked", "not checked", "not checked", ""] + return [cand["id"], cand["cause"], cand["effect"], explains] + ["not checked"] * 5 + [""] test, suite, replay, diff = checks["test"], checks["suite"], checks["replay"], checks["diff"] if not test["ran"]: test_text = "did not run" @@ -52,13 +61,16 @@ def candidate_row(cand: dict, checks: Optional[dict]) -> List[str]: suite_text = ("did not run" if not suite["ran"] else ", ".join(suite["added"]) if suite["added"] else "none") if replay["ran"]: - replay_text = f"{table_text(replay['before'])} → {table_text(replay['after'])}" + replay_text = (f"{table_text(replay['before'])} → {table_text(replay['after'])} " + f"({replay['before_ref'][:12]} → {replay['ref'][:12]})") + change_text = changes_text(replay) false_text = f"{replay['false_success_added']:+d}" + ( f" (new: {', '.join(replay['false_success_new'])})" if replay["false_success_new"] else "") else: - replay_text, false_text = "did not run", "did not run" + replay_text, change_text, false_text = "did not run", "did not run", "did not run" diff_text = f"{diff['files']} files, +{diff['added']} −{diff['removed']}" - return [cand["id"], cand["cause"], explains, test_text, suite_text, replay_text, false_text, diff_text] + return [cand["id"], cand["cause"], cand["effect"], explains, test_text, suite_text, replay_text, change_text, + false_text, diff_text] def decide_one(args: argparse.Namespace, round_dir: str, record: dict) -> None: @@ -67,9 +79,9 @@ def decide_one(args: argparse.Namespace, round_dir: str, record: dict) -> None: if os.path.isdir(base) else [] if not ids: raise LoopError("no candidate is registered: add one with loop.py candidate, then loop.py check") - rows, not_run = [], [] + rows, not_run, unchanged = [], [], [] for cid in ids: - cand = read_json(os.path.join(base, cid, "candidate.json")) + cand = load_candidate(os.path.join(base, cid, "candidate.json")) checks_path = os.path.join(base, cid, "checks.json") checks = read_json(checks_path) if os.path.isfile(checks_path) else None rows.append("| " + " | ".join(cell(c) for c in candidate_row(cand, checks)) + " |") @@ -79,11 +91,16 @@ def decide_one(args: argparse.Namespace, round_dir: str, record: dict) -> None: for name in ("test", "suite", "replay"): if not checks[name]["ran"]: not_run.append(f"- {cid}, {name}: {checks[name]['reason']}") + replay = checks["replay"] + if replay["ran"] and replay["cause"]["changed"] == 0: + unchanged.append(f"- {cid}: 0 of {replay['cause']['trials']} trials of its cause change their label " + f"between {replay['before_ref'][:12]} and {replay['ref'][:12]}.") trials = len(read_rows(record["run"]["job_dir"], float(record["pass_threshold"]))) fill_page("decision-1.md", { "ROUND": os.path.basename(round_dir), "LOOP": args.loop, "TRIALS": str(trials), "SUMMARY": f"{len(ids)} candidates, checked against the run {record['run']['job_dir']}.", "ROWS": "\n".join(rows), "NOT_RUN": "\n".join(not_run) or "Every check ran.", + "UNCHANGED": "\n".join(unchanged) or "None.", }, os.path.join(round_dir, "decision-1.md")) print(f"wrote {round_dir}/decision-1.md: {len(ids)} candidates, {len(not_run)} checks that did not run") @@ -166,6 +183,8 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: "SPENT": usd(spent) + ("" if trial_record["cost_complete"] else " at least: some trials report no cost"), "LEFT": usd(left), "BUDGET": usd(float(record["budget_usd"])), "RESULTS": "\n".join(results), "OPTIONS": "\n".join(option_rows), + "REPORT_NOTE": "".join(f"\n- {p['candidate']}: {REPORT_NOT_READ}." for p in entry.get("predictions") or [] + if p.get("effect") == "report"), }, os.path.join(round_dir, "decision-2.md")) print(f"wrote {round_dir}: trial.json, decision-2.md; budget left {usd(left)}") for o in options: @@ -196,6 +215,8 @@ def cmd_answer(args: argparse.Namespace) -> None: if args.keep is None: raise LoopError("answer --stop 1 needs --keep: candidate ids separated by commas, or none") keep = [] if args.keep == "none" else [k.strip() for k in args.keep.split(",") if k.strip()] + cands = {k: load_candidate(os.path.join(round_dir, "candidates", k, "candidate.json")) for k in keep + if os.path.isfile(os.path.join(round_dir, "candidates", k, "candidate.json"))} missing = [k for k in keep if not os.path.isfile(os.path.join(round_dir, "candidates", k, "checks.json"))] if missing: raise LoopError(f"these candidates are not checked: {', '.join(missing)}") @@ -204,8 +225,9 @@ def cmd_answer(args: argparse.Namespace) -> None: entry["kept"] = keep entry["predictions"] = [] for k in keep: - cand = read_json(os.path.join(round_dir, "candidates", k, "candidate.json")) + cand = cands[k] entry["predictions"].append({"candidate": k, "cause": cand["cause"], "rule": cand["rule"], + "effect": cand["effect"], "trials": cand["prediction"]["trials"], "tasks": cand["prediction"]["tasks"], "expect": "pass"}) else: if args.choice not in CHOICES: diff --git a/.claude/skills/run-to-fix/scripts/loop.py b/.claude/skills/run-to-fix/scripts/loop.py index efe7e5c5..bde9bdd6 100644 --- a/.claude/skills/run-to-fix/scripts/loop.py +++ b/.claude/skills/run-to-fix/scripts/loop.py @@ -54,6 +54,8 @@ def main(argv: List[str]) -> int: ca.add_argument("--ref", required=True, help="the commit of the fix in the agent's checkout") ca.add_argument("--test", help="the test file of the cause, relative to the checkout") ca.add_argument("--test-ref", help="the commit that adds the test, before the fix") + ca.add_argument("--effect", required=True, choices=("outcome", "report"), + help="outcome: failing trials of the rule turn into passes; report: the agent's report changes") ca.set_defaults(func=cmd_candidate) ch = sub.add_parser("check", parents=[timeout], help="stage 3: check a candidate") ch.add_argument("--loop", required=True) diff --git a/.claude/skills/run-to-fix/scripts/selecting.py b/.claude/skills/run-to-fix/scripts/selecting.py index e77565e5..2fda6f89 100644 --- a/.claude/skills/run-to-fix/scripts/selecting.py +++ b/.claude/skills/run-to-fix/scripts/selecting.py @@ -31,14 +31,15 @@ def kept_candidates(round_dir: str, entry: dict) -> List[dict]: def guards_of(kept: List[dict], rows_by_trial: Dict[str, Tuple[str, bool]], targets: List[str]) -> Tuple[List[str], Optional[str]]: - touched: Optional[set] = None - for checks in kept: - replay = checks["replay"] - if replay["ran"] and replay["touched"] is not None: - touched = (touched or set()) | set(replay["touched"]) - if touched is None: - return [], "no replay reports which trials ran the changed code" - tasks = {rows_by_trial[t][0] for t in touched if t in rows_by_trial and rows_by_trial[t][1]} + """Passing tasks whose recorded trial ran the changed code: trials the replay marks `touched`, and trials whose + label a kept fix changes.""" + ran = [checks["replay"] for checks in kept if checks["replay"]["ran"]] + if not ran: + return [], "no replay ran for the kept candidates" + names = set() + for replay in ran: + names |= set(replay["touched"] or []) | set(replay["changed_trials"]) + tasks = {rows_by_trial[t][0] for t in names if t in rows_by_trial and rows_by_trial[t][1]} return sorted(tasks - set(targets)), None @@ -85,7 +86,9 @@ def cmd_select(args: argparse.Namespace) -> None: rows = read_rows(record["run"]["job_dir"], float(record["pass_threshold"])) solved, _ = solved_tasks(rows) - targets = sorted({t for p in entry["predictions"] for t in p["tasks"]}) + if any(p.get("effect") not in ("outcome", "report") for p in entry["predictions"]): + raise LoopError("a prediction of stop 1 names no effect; record the answer of stop 1 again") + targets = sorted({t for p in entry["predictions"] if p["effect"] == "outcome" for t in p["tasks"]}) guards, no_guards = guards_of(kept, {r.trial: (r.task, r.passed) for r in rows}, targets) pool = sorted(t for t, ok in solved.items() if ok and t not in targets and t not in guards) canaries = sorted(random.Random(SEED).sample(pool, min(args.canaries, len(pool)))) From 9df77edc6f42195940a4984e80aba3fc70b4e1e4 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 02:15:07 +0100 Subject: [PATCH 35/92] docs(run-to-fix): describe the ref before a fix, the effect of a candidate and the replay's new placeholders --- .claude/skills/run-to-fix/SKILL.md | 31 ++++++++++++------- .../skills/run-to-fix/references/adapter.md | 15 +++++---- .claude/skills/run-to-fix/references/files.md | 12 ++++--- 3 files changed, 36 insertions(+), 22 deletions(-) diff --git a/.claude/skills/run-to-fix/SKILL.md b/.claude/skills/run-to-fix/SKILL.md index c5f7d45a..9a768938 100644 --- a/.claude/skills/run-to-fix/SKILL.md +++ b/.claude/skills/run-to-fix/SKILL.md @@ -70,19 +70,26 @@ For each cause in the agent layer: 2. Register and check each candidate: ```bash -python3 L candidate --loop DIR --id ID --cause CAUSE --ref FIX --test PATH --test-ref TEST_COMMIT +python3 L candidate --loop DIR --id ID --cause CAUSE --ref FIX --test PATH --test-ref TEST_COMMIT --effect E python3 L check --loop DIR --candidate ID python3 L decide --loop DIR --stop 1 ``` -`check` runs, each in a detached worktree of the commit that is removed afterwards: the test before and after the fix, -`commands.test` against the failures the suite has on `base_ref`, and `commands.replay` on every recorded trial at -`base_ref` and at the fix. The replay's labels crossed with pass and fail give the table done and passed / done and -failed / not done and passed / not done and failed, and the count of false successes, failed trials the agent reports -as a success. A command named in `paid` is written to `round-NN/commands/` and not started. +`--effect` says what the fix does, with no default. `outcome`: failing trials of the rule turn into passes; the rule +counts its failing trials, and their tasks are expected to pass. `report`: the agent's report changes on trials of +the rule and pass or fail stays; the rule counts every trial it matches, no task is expected to pass, and the +candidate needs `commands.replay`. -**Stop 1.** Show the person `decision-1.md`: one row per candidate with its cause, test, table before and after, false -successes added and diff size, and the checks that did not run. Ask which candidates to keep, and record the answer in +`check` runs, each in a detached worktree of the commit that is removed afterwards: the test before and after the fix, +`commands.test` against the failures the suite has on `base_ref`, and `commands.replay` on every recorded trial at the +ref before the fix (the candidate's `test_ref`, else `base_ref`) and at the fix. The replay's labels crossed with pass +and fail give the table done and passed / done and failed / not done and passed / not done and failed, and the count of +false successes, failed trials the agent reports as a success. `check` also counts the trials of the cause whose label +the fix changes, by direction. A command named in `paid` is written to `round-NN/commands/` and not started. + +**Stop 1.** Show the person `decision-1.md`: one row per candidate with its cause, effect, test, table before and after +with its two refs, the trials of the cause whose label changes, false successes added and diff size; the candidates +whose fix changes no label of their cause, under a heading of their own; and the checks that did not run. Ask which candidates to keep, and record the answer in the person's words: ```bash @@ -97,8 +104,9 @@ Merge the kept candidates into one commit of the work branch. Then: python3 L select --loop DIR --ref COMMIT --canaries N ``` -prints the trial run in three sizes with their price: the targets (the tasks the kept causes predict), plus the -guards (passing tasks whose recorded trial ran the changed code, when the replay reports `touched`), plus the canaries +prints the trial run in three sizes with their price: the targets (the tasks the kept `outcome` candidates predict), +plus the guards (passing tasks whose recorded trial ran the changed code: the replay marks it `touched`, or a kept fix +changes its label), plus the canaries (passing tasks drawn with a fixed seed). Every task gets 3 trials. A price is the sum over the tasks of the mean recorded cost of each task's trials, times its trials. The cost comes from this round's trial runs on the same commit when they hold the task, else from the run that was read. A task with no recorded cost takes the mean of the others, @@ -116,7 +124,8 @@ This writes `trial-tasks.txt`, runs `commands.build` and writes `build.json`, an python3 L decide --loop DIR --stop 2 --trial-job JOB ``` -**Stop 2.** Show the person `decision-2.md`: the build, the targets' trials before and in every trial run of this +**Stop 2.** Show the person `decision-2.md`. For a kept `report` candidate it says that this version does not read the +agent's report in the new run. The page shows the build, the targets' trials before and in every trial run of this round on that build with Fisher's exact test, the guards and canaries, the money spent and left, and three options with their price: stop, the targets and guards again, a full pass. The commands of the last two are written. Record the choice: diff --git a/.claude/skills/run-to-fix/references/adapter.md b/.claude/skills/run-to-fix/references/adapter.md index 67dc6387..b13adf95 100644 --- a/.claude/skills/run-to-fix/references/adapter.md +++ b/.claude/skills/run-to-fix/references/adapter.md @@ -42,6 +42,8 @@ Every key is present, and a value the agent does not have is null. `read` refuse | `{ref}` | `replay`, `run_tasks` | the full commit id under check | | `{job_dir}` | `replay`, `classes` | the job folder the person gave to `loop.py read` | | `{out_json}` | `replay`, `classes` | the file the command writes; the loop reads it after a zero exit | +| `{base_ref}` | `replay`, optional | the full commit id of `source.base_ref` | +| `{before_ref}` | `replay`, optional | the full commit id the replay is compared with: for the replay before a fix, its own `{ref}`; for the replay of the fix, the ref before it | | `{units_file}` | `run_tasks` | a text file with one task name per line | | `{repeat}` | `run_tasks` | trials per task | | `{jobs_dir}` | `run_tasks` | where the run writes its job folder | @@ -116,14 +118,15 @@ adds none. A failure on `base_ref` that a candidate removes is reported as remov 1. `trials` has one entry for every trial of `{job_dir}`. `label` is what the agent reports for that trial on the code of `ref`. -2. `touched` is optional: true when the code that changed between `base_ref` and `ref` ran in the trial. Guard tasks - of a trial run come from this field. Without it the list of guards is empty, and the decision page says so. -3. `kept_recorded` is optional: the trials the replay could not reproduce, which keep the label of the recorded run. - The decision page gives their count. +2. `touched` is optional: true when the code that changed between `{before_ref}` and `{ref}` ran in the trial. Guard + tasks of a trial run come from this field and from the passing trials whose label the fix changes. +3. `kept_recorded` is optional: the trials the replay could not reproduce. For those trials the command writes the + recorded label into `label`; the loop crosses `label` as given. The decision page gives their count. The loop crosses each label with the trial's recorded pass or fail. A label in `replay_success_labels` on a failed -trial is a false success. A candidate's table before and after is these four counts on `base_ref` and on the -candidate, and the false successes it adds are the difference. +trial is a false success. A candidate's table before and after is these four counts at the ref before its fix and at +its fix: the before ref is the candidate's `test_ref` when it has one, else `base_ref`. The false successes it adds +are the difference. The loop replays each pair of `{ref}` and `{before_ref}` once per round. ### The classes file diff --git a/.claude/skills/run-to-fix/references/files.md b/.claude/skills/run-to-fix/references/files.md index 8dc9f762..42cdbe1d 100644 --- a/.claude/skills/run-to-fix/references/files.md +++ b/.claude/skills/run-to-fix/references/files.md @@ -33,10 +33,11 @@ round that has no `result.json`. | `groups.json` | 2 | `classify` | class counts, the rules switched off, the groups with their warnings, the counts of found and agent classes | | `agent-classes.json` | 2 | `commands.classes` | the agent's own classes and flags per trial | | `causes/.md` | 2 | the assistant | one cause, below | -| `candidates//candidate.json` | 3 | `candidate` | the cause, the rule, the fix commit, the test and its commit, the trials and tasks the cause predicts | -| `candidates//checks.json` | 3 | `check` | the test before and after the fix, the suite against `base_ref`, the replay table before and after, the diff size | -| `candidates//*.log`, `replay.json` | 3 | `check` | the output of each command | -| `base/suite.json`, `base/replay.json` | 3 | `check` | the suite's failures and the replay on `base_ref`, made once per round | +| `candidates//candidate.json` | 3 | `candidate` | the cause, the rule, the effect (`outcome` or `report`), the fix commit, the test and its commit, the trials of the cause and the tasks expected to pass; registering a candidate again removes its `checks.json` | +| `candidates//checks.json` | 3 | `check` | the test before and after the fix, the suite against `base_ref`, the replay table at the ref before the fix and at the fix with both refs, the trials of the cause whose label changes, every trial whose label changes, the diff size | +| `candidates//*.log` | 3 | `check` | the output of each command | +| `base/suite.json` | 3 | `check` | the suite's failures on `base_ref`, made once per round | +| `replays/-against-.json`, `.log` | 3 | `check` | one replay per pair of refs of a round, made once | | `decision-1.md` | stop 1 | `decide --stop 1` | one row per candidate and the checks that did not run | | `commands/-.sh`, `commands/run_tasks*.sh` | 3, 4 | `check`, `select`, `decide --stop 2` | commands a person starts | | `trial-tasks.txt` | 4 | `select --size` | the tasks of the trial run, one per line, the `{units_file}` of `run_tasks` | @@ -74,7 +75,8 @@ Not checked: what the files cannot tell. "round": "round-01", "started": "", "answers": [{"about": "stop 1: the candidates to keep", "date": "", "text": ""}], "kept": [""], - "predictions": [{"candidate": "...", "cause": "...", "rule": "...", "tasks": ["..."], "expect": "pass"}], + "predictions": [{"candidate": "...", "cause": "...", "rule": "...", "effect": "outcome", "trials": ["..."], + "tasks": ["..."], "expect": "pass"}], "selection": {"ref": "...", "size": "guards", "targets": [], "guards": [], "canaries": [], "repeat": 3, "seed": 20261117, "price_usd": 0.0, "budget_usd": 0.0, "guards_reason": null}, "trial_runs": [{"job_dir": "...", "ref": "...", "trials": 0, "cost_usd": 0.0, "cost_complete": true}], From b3c2e80c01626e7340ee4eaa6a5b5ba996e9ba8f Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:27:16 +0100 Subject: [PATCH 36/92] test(run-to-fix): count the same trials as run-analysis when a standard job holds retried tries --- .../skills/run-to-fix/tests/test_classify.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/tests/test_classify.py b/.claude/skills/run-to-fix/tests/test_classify.py index 22c21afe..f1d5df21 100644 --- a/.claude/skills/run-to-fix/tests/test_classify.py +++ b/.claude/skills/run-to-fix/tests/test_classify.py @@ -5,7 +5,8 @@ import sys from typing import Dict, List -from jobs import (LIMITS, NO_LIMITS, LoopCase, Trial, adapter, load, run, trial, write, write_job, write_json) +from jobs import (LIMITS, NO_LIMITS, LoopCase, Trial, adapter, load, run, trial, write, write_job, write_json, + write_trial) FIVE = {"pass", "time_limit", "exception", "no_score", "verifier_failed"} SEVEN = FIVE | {"step_limit", "output_limit"} @@ -425,3 +426,22 @@ def test_a_small_group_only_says_the_task_fails_on_both_runs(self) -> None: self.assertEqual(warning["kind"], "fails_on_baseline") self.assertIn("fails on both runs", warning["text"]) self.assertNotIn("benchmark", warning["text"]) + + +class RetriedTries(LoopCase): + """A job of the standard keeps the earlier tries of an attempt in retried/.try-N; none is a trial.""" + + def test_the_skill_and_run_analysis_count_the_same_trials(self) -> None: + trials = [trial("u1__s", "made-up-bench/u1", 1.0), trial("u2__s", "made-up-bench/u2", 0.0), + trial("u3__s", "made-up-bench/u3", 1.0)] + run_dir = write_job(self.jobs, "standard-run", "alpha", "standard", trials) + earlier = trial("u2__s.try-1", "made-up-bench/u2", None) + earlier.exception = ("ConnectionError", "the relay closed") + write_trial(os.path.join(run_dir, "retried"), earlier, "alpha", "made-up-model", "standard", 9) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(run_dir, None) + metrics = load(self.round_file("metrics.json")) + self.assertEqual([(r["trials"], r["trials_passed"]) for r in metrics["runs"]], [(3, 2)]) + self.classify_ok() + self.assertEqual(sorted(self.by_trial()), ["u1__s", "u2__s", "u3__s"]) + self.assertEqual(load(self.round_file("groups.json"))["trials"], 3) From 48bb0c35df23cbb3a76adad6afc89d4ad8fbcdf9 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:34:37 +0100 Subject: [PATCH 37/92] test(run-to-fix): use a made-up agent's error and made-up counts --- .claude/skills/run-to-fix/tests/test_classify.py | 10 +++++----- .claude/skills/run-to-fix/tests/test_compare.py | 10 +++++----- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.claude/skills/run-to-fix/tests/test_classify.py b/.claude/skills/run-to-fix/tests/test_classify.py index f1d5df21..1abbbc44 100644 --- a/.claude/skills/run-to-fix/tests/test_classify.py +++ b/.claude/skills/run-to-fix/tests/test_classify.py @@ -350,12 +350,12 @@ def test_a_rule_with_two_patterns_is_refused(self) -> None: self.assertIn("both", r.err) -#: the last lines of two exception messages of one real run; they differ in the model text quoted after unit= +#: the last lines of two exception messages of a made-up agent; they differ in the model text quoted after unit= REPEATED_STREAM = [ - r"JiuwenSwarm failed: FrameworkError: [181001] model call failed, reason: LLM repeated stream output detected: " - r"field=reasoning_content, unit='OK.\n\nWriting.\n\nLet me go.\n\n', repeat_count=6", - r"JiuwenSwarm failed: FrameworkError: [181001] model call failed, reason: LLM repeated stream output detected: " - r"field=reasoning_content, unit='\n\nHmm.', repeat_count=27", + r"MadeUpAgent failed: LoopError: [4021] model call failed, reason: repeated stream output detected: " + r"field=reasoning, unit='Fine.\n\nNext step.\n\n', repeat_count=4", + r"MadeUpAgent failed: LoopError: [4021] model call failed, reason: repeated stream output detected: " + r"field=reasoning, unit='\n\nWait.', repeat_count=19", ] diff --git a/.claude/skills/run-to-fix/tests/test_compare.py b/.claude/skills/run-to-fix/tests/test_compare.py index 265d6b64..f07b3558 100644 --- a/.claude/skills/run-to-fix/tests/test_compare.py +++ b/.claude/skills/run-to-fix/tests/test_compare.py @@ -36,11 +36,11 @@ def compare(self, both: int, won: int, lost: int, neither: int) -> dict: self.last_out = r.out return load(self.round_file("result.json")) - def test_ten_won_and_thirteen_lost_is_noise(self) -> None: - result = self.compare(271 // 10, 10, 13, 6) - self.assertEqual(result["pairs"], {"common_tasks": 27 + 10 + 13 + 6, "both": 27, "won": 10, "lost": 13, - "neither": 6}) - self.assertAlmostEqual(result["test"]["p"], 0.6776394844055176, places=9) + def test_eight_won_and_twelve_lost_is_noise(self) -> None: + result = self.compare(30, 8, 12, 5) + self.assertEqual(result["pairs"], {"common_tasks": 30 + 8 + 12 + 5, "both": 30, "won": 8, "lost": 12, + "neither": 5}) + self.assertAlmostEqual(result["test"]["p"], 0.5034446716308594, places=9) self.assertEqual(result["verdict"], "noise") self.assertIn("noise", self.last_out) From 43d263f74cd1a2caef0c8e70b5cc1f33b5df37aa Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:34:37 +0100 Subject: [PATCH 38/92] test(run-to-fix): say why a trial run has no guards, and what a report candidate expects --- .../skills/run-to-fix/tests/test_decide.py | 28 +++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index 90f60c11..a1936944 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -342,6 +342,11 @@ def kept_report(self) -> None: r = run(["answer", "--loop", self.loop, "--stop", "1", "--keep", "rep", "--text", "keep the report fix"]) self.assertEqual(r.code, 0, r.out + r.err) + def test_a_report_candidate_expects_a_label_change(self) -> None: + self.kept_report() + pred = load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["predictions"][0] + self.assertEqual((pred["effect"], pred["tasks"], pred["expect"]), ("report", [], "label_change")) + def select_report(self, extra: List[str]) -> Result: return run(["select", "--loop", self.loop, "--ref", self.repo.report, "--canaries", "0"] + extra) @@ -374,3 +379,26 @@ def test_the_pages_say_the_report_is_not_read(self) -> None: pred = load(self.round_file("result.json"))["predictions"][0] self.assertEqual(pred["effect"], "report") self.assertIn("does not read the agent's report", pred["result"]["note"]) + + +STRIP_TOUCHED = (" && python3 -c \"import json,sys; p=sys.argv[1]; d=json.load(open(p)); " + "[v.pop('touched', None) for v in d['trials'].values()]; json.dump(d, open(p, 'w'))\" {out_json}") + + +class GuardReasons(Flow): + def test_no_touched_and_no_changed_passing_trial(self) -> None: + commands = self.commands() + commands["replay"] = (commands["replay"] or "") + STRIP_TOUCHED + self.start(commands, ["run_tasks"]) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") + self.check("good") + run(["decide", "--loop", self.loop, "--stop", "1"]) + run(["answer", "--loop", self.loop, "--stop", "1", "--keep", "good", "--text", "keep good"]) + reason = ("no replay of the kept candidates gives touched, and the kept fixes change the label of no passing " + "trial outside the targets") + r = self.select([]) + self.assertEqual(r.code, 0, r.out + r.err) + self.assertIn(f"guards: none ({reason})", r.out) + r = self.select(["--size", "guards", "--budget-usd", "0.5", "--text", "targets only, in effect"]) + self.assertEqual(r.code, 0, r.out + r.err) + self.assertEqual(load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["selection"]["guards_reason"], reason) From 647879e3d3a1a9d823707555a0cac5755e0ed512 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:36:00 +0100 Subject: [PATCH 39/92] fix(run-to-fix): say why a trial run has no guards, and what a report candidate expects select prints and records what the loop knows when it finds no guard: whether any replay gives touched, and that no kept fix changes the label of a passing trial. answer --stop 1 writes expect label_change for a report candidate. --- .claude/skills/run-to-fix/scripts/deciding.py | 3 ++- .../skills/run-to-fix/scripts/selecting.py | 19 +++++++++++++------ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/.claude/skills/run-to-fix/scripts/deciding.py b/.claude/skills/run-to-fix/scripts/deciding.py index c01ae078..821bfc35 100644 --- a/.claude/skills/run-to-fix/scripts/deciding.py +++ b/.claude/skills/run-to-fix/scripts/deciding.py @@ -228,7 +228,8 @@ def cmd_answer(args: argparse.Namespace) -> None: cand = cands[k] entry["predictions"].append({"candidate": k, "cause": cand["cause"], "rule": cand["rule"], "effect": cand["effect"], "trials": cand["prediction"]["trials"], - "tasks": cand["prediction"]["tasks"], "expect": "pass"}) + "tasks": cand["prediction"]["tasks"], + "expect": "pass" if cand["effect"] == "outcome" else "label_change"}) else: if args.choice not in CHOICES: raise LoopError(f"answer --stop 2 needs --choice, one of {', '.join(CHOICES)}") diff --git a/.claude/skills/run-to-fix/scripts/selecting.py b/.claude/skills/run-to-fix/scripts/selecting.py index 2fda6f89..cf9fe012 100644 --- a/.claude/skills/run-to-fix/scripts/selecting.py +++ b/.claude/skills/run-to-fix/scripts/selecting.py @@ -36,11 +36,18 @@ def guards_of(kept: List[dict], rows_by_trial: Dict[str, Tuple[str, bool]], targ ran = [checks["replay"] for checks in kept if checks["replay"]["ran"]] if not ran: return [], "no replay ran for the kept candidates" - names = set() - for replay in ran: - names |= set(replay["touched"] or []) | set(replay["changed_trials"]) - tasks = {rows_by_trial[t][0] for t in names if t in rows_by_trial and rows_by_trial[t][1]} - return sorted(tasks - set(targets)), None + + def passing(names: set) -> set: + return {rows_by_trial[t][0] for t in names if t in rows_by_trial and rows_by_trial[t][1]} - set(targets) + + touched = passing({t for replay in ran for t in replay["touched"] or []}) + changed = passing({t for replay in ran for t in replay["changed_trials"]}) + if touched or changed: + return sorted(touched | changed), None + given = any(replay["touched"] is not None for replay in ran) + return [], ("touched marks no passing trial outside the targets" if given + else "no replay of the kept candidates gives touched") + \ + ", and the kept fixes change the label of no passing trial outside the targets" def build_on(adapter: Adapter, loop_dir: str, round_dir: str, sha: str, timeout_s: float) -> dict: @@ -100,7 +107,7 @@ def cmd_select(args: argparse.Namespace) -> None: price = {k: price_of(v, TARGET_TRIALS, sources) for k, v in sizes.items()} print(f"targets ({len(targets)} tasks of the kept causes): {', '.join(targets)}") - print(f"guards: {', '.join(guards)}" if guards else f"guards: none ({no_guards or 'no passing task ran the changed code'})") + print(f"guards: {', '.join(guards)}" if guards else f"guards: none ({no_guards})") print(f"canaries ({len(canaries)} passing tasks, seed {SEED}): {', '.join(canaries) or 'none'}") for k in SIZES: print(f"{names[k]}: {price[k].tasks} tasks, {price[k].trials} trials, {describe(price[k])}") From c162efe1b7f919dba28b5b615f79f49b75d15380 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:36:00 +0100 Subject: [PATCH 40/92] docs(run-to-fix): name the skill in AGENTS.md and show a report prediction in files.md --- .claude/skills/run-to-fix/references/files.md | 4 +++- AGENTS.md | 13 +++++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/references/files.md b/.claude/skills/run-to-fix/references/files.md index 42cdbe1d..dbb2d6e2 100644 --- a/.claude/skills/run-to-fix/references/files.md +++ b/.claude/skills/run-to-fix/references/files.md @@ -76,7 +76,9 @@ Not checked: what the files cannot tell. "answers": [{"about": "stop 1: the candidates to keep", "date": "", "text": ""}], "kept": [""], "predictions": [{"candidate": "...", "cause": "...", "rule": "...", "effect": "outcome", "trials": ["..."], - "tasks": ["..."], "expect": "pass"}], + "tasks": ["..."], "expect": "pass"}, + {"candidate": "...", "cause": "...", "rule": "...", "effect": "report", "trials": ["..."], + "tasks": [], "expect": "label_change"}], "selection": {"ref": "...", "size": "guards", "targets": [], "guards": [], "canaries": [], "repeat": 3, "seed": 20261117, "price_usd": 0.0, "budget_usd": 0.0, "guards_reason": null}, "trial_runs": [{"job_dir": "...", "ref": "...", "trials": 0, "cost_usd": 0.0, "cost_complete": true}], diff --git a/AGENTS.md b/AGENTS.md index 6ba4ab10..08097cdb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -289,6 +289,19 @@ renders the page from the template beside it; the findings and the prose are the assistant's, by the procedure in the SKILL.md. Nothing in it is specific to one assistant or one benchmark. +**Run to fix** is the `/run-to-fix` skill (`.claude/skills/run-to-fix/SKILL.md`): +from a finished run to checked fix candidates and a compared rerun, in rounds of +five stages with two stops for a person. Its script, +`.claude/skills/run-to-fix/scripts/loop.py` (standard library only, no import +from `app/`, calls `runs.py` of `/run-analysis` for health, metrics and the +page, reads the job trees and the agent's adapter file, and writes only into a +loop folder that lies outside every repository or that git ignores, with +worktrees of the agent's checkout that it removes after each check), classifies +the trials, checks fix candidates, prices and records trial runs, and compares +runs with exact tests; the causes, tests and fixes are the assistant's, by the +procedure in the SKILL.md. Everything that belongs to one agent lives in its +adapter file, and nothing in the skill names an agent or a benchmark. + ## Conventions Use `nullius.thinkflowlab@proton.me` for Nullius-owned contact email addresses, From e9cd01afc14f25f8bfb14e801aff61373b7c42e8 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:41:57 +0100 Subject: [PATCH 41/92] test(run-to-fix): pin that a round and a trial run need a budget --- .claude/skills/run-to-fix/tests/test_decide.py | 9 +++++++++ .claude/skills/run-to-fix/tests/test_read.py | 10 ++++++++++ 2 files changed, 19 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index a1936944..5a8b79ec 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -402,3 +402,12 @@ def test_no_touched_and_no_changed_passing_trial(self) -> None: r = self.select(["--size", "guards", "--budget-usd", "0.5", "--text", "targets only, in effect"]) self.assertEqual(r.code, 0, r.out + r.err) self.assertEqual(load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["selection"]["guards_reason"], reason) + + +class TrialBudgetHasNoDefault(Flow): + def test_a_size_needs_the_budget(self) -> None: + self.kept() + r = self.select(["--size", "targets", "--text", "the targets"]) + self.assertNotEqual(r.code, 0) + self.assertIn("--budget-usd", r.err) + self.assertFalse(os.path.exists(self.round_file("trial-tasks.txt"))) diff --git a/.claude/skills/run-to-fix/tests/test_read.py b/.claude/skills/run-to-fix/tests/test_read.py index bf9d0ec5..5d900726 100644 --- a/.claude/skills/run-to-fix/tests/test_read.py +++ b/.claude/skills/run-to-fix/tests/test_read.py @@ -268,3 +268,13 @@ def test_another_command_may_not_take_them(self) -> None: r = self.read(run_dir, None) self.assertNotEqual(r.code, 0) self.assertIn("{base_ref}", r.err) + + +class BudgetHasNoDefault(LoopCase): + def test_read_needs_the_budget(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + r = run(["read", "--loop", self.loop, "--run", run_dir, "--pass-threshold", "1"]) + self.assertNotEqual(r.code, 0) + self.assertIn("--budget-usd", r.err) + self.assertFalse(os.path.exists(os.path.join(self.loop, "round-01"))) From da7edb1a48c927778847add65790bfa577dcffae Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 17:56:39 +0100 Subject: [PATCH 42/92] test(run-to-fix): keep the spending of a run unknown while a trial reports no cost --- .../skills/run-to-fix/tests/test_compare.py | 23 ++++++++ .../skills/run-to-fix/tests/test_decide.py | 54 +++++++++++++++++++ 2 files changed, 77 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_compare.py b/.claude/skills/run-to-fix/tests/test_compare.py index f07b3558..b91c29e2 100644 --- a/.claude/skills/run-to-fix/tests/test_compare.py +++ b/.claude/skills/run-to-fix/tests/test_compare.py @@ -311,3 +311,26 @@ def test_a_result_that_cannot_be_written_leaves_the_ledger_open(self) -> None: r = run(["compare", "--loop", self.loop, "--new", new_dir, "--against", "run"]) self.assertNotEqual(r.code, 0) self.assertIsNone(load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["result"]) + + +class IncompleteNewCost(LoopCase): + def compare_partial(self, extra: List[str]) -> dict: + before, new = paired(20, 1, 1, 1) + new[0].cost = None + before_dir = write_job(self.jobs, "before", "alpha", "harbor", before) + new_dir = write_job(self.jobs, "new", "alpha", "harbor", new) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(before_dir, None) + r = run(["compare", "--loop", self.loop, "--new", new_dir, "--against", "run"] + extra) + self.assertEqual(r.code, 0, r.out + r.err) + return load(self.round_file("result.json"))["stop_rule"] + + def test_a_new_run_of_unknown_cost_leaves_the_budget_unchecked(self) -> None: + rule = self.compare_partial([]) + self.assertIsNone(rule["spent_usd"]) + self.assertIsNone(rule["left_usd"]) + self.assertTrue(any("budget" in n for n in rule["not_checked"])) + + def test_the_person_s_figure_for_the_new_run(self) -> None: + rule = self.compare_partial(["--new-cost-usd", "0.5"]) + self.assertAlmostEqual(rule["spent_usd"], 0.5) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index 5a8b79ec..69c883d5 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -411,3 +411,57 @@ def test_a_size_needs_the_budget(self) -> None: self.assertNotEqual(r.code, 0) self.assertIn("--budget-usd", r.err) self.assertFalse(os.path.exists(self.round_file("trial-tasks.txt"))) + + +class IncompleteCosts(Flow): + """A trial run whose trials do not all report a cost has no known spending until the person gives it.""" + + def job_without_one_cost(self) -> str: + self.chosen() + trials = [] + for task, passes, cost in (("f1", 3, 0.04), ("f2", 2, 0.06), ("p1", 3, 0.01)): + for k in range(3): + t = trial(f"{task}-{k}__t", f"bench/{task}", 1.0 if k < passes else 0.0) + t.cost = None if (task, k) == ("f1", 0) else cost + trials.append(t) + return write_job(self.jobs, "trial", "alpha", "harbor", trials) + + def test_stop_two_refuses_an_unknown_spending(self) -> None: + job = self.job_without_one_cost() + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job]) + self.assertNotEqual(r.code, 0) + self.assertIn("--trial-cost-usd", r.err) + self.assertIn("$0.29", r.err) + self.assertEqual(load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["trial_runs"], []) + self.assertFalse(os.path.exists(self.round_file("commands/run_tasks-full.sh"))) + + def test_the_person_s_figure_is_the_spending(self) -> None: + job = self.job_without_one_cost() + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--trial-cost-usd", "0.40"]) + self.assertEqual(r.code, 0, r.out + r.err) + record = load(self.round_file("trial.json")) + self.assertAlmostEqual(record["cost_usd"], 0.40) + self.assertAlmostEqual(record["budget_left_usd"], 9.60) + run_entry = record["trial_runs"][0] + self.assertEqual(run_entry["cost_from"], "the person") + self.assertAlmostEqual(run_entry["cost_recorded_usd"], 0.29) + self.assertIn("the person's figure", self.page("decision-2.md")) + + def test_a_figure_under_the_recorded_cost_is_refused(self) -> None: + job = self.job_without_one_cost() + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--trial-cost-usd", "0.10"]) + self.assertNotEqual(r.code, 0) + self.assertIn("$0.29", r.err) + + def test_an_old_trial_run_of_unknown_cost_stops_select(self) -> None: + job = self.job_without_one_cost() + run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--trial-cost-usd", "0.40"]) + path = os.path.join(self.loop, "ledger.json") + ledger = load(path) + entry = ledger["rounds"][0]["trial_runs"][0] + entry.pop("cost_from") + entry["cost_complete"] = False + write_json(path, ledger) + r = self.select(["--size", "targets", "--budget-usd", "1", "--text", "again"]) + self.assertNotEqual(r.code, 0) + self.assertIn("unknown", r.err) From 8437b1562c8f2f0bb7562d924a1804807d58ac5a Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:03:34 +0100 Subject: [PATCH 43/92] fix(run-to-fix): keep the spending of a run unknown while a trial reports no cost A trial run whose trials do not all report a cost no longer counts its recorded part as the whole: decide --stop 2 refuses it until the person gives its cost with --trial-cost-usd, and the page says whose figure it is. compare leaves the budget unchecked for such a new run, or takes the person's figure from --new-cost-usd. A ledger entry of unknown cost stops select and compare. --- .../skills/run-to-fix/scripts/comparing.py | 28 +++++++++++++++---- .claude/skills/run-to-fix/scripts/deciding.py | 16 +++++++++-- .claude/skills/run-to-fix/scripts/loop.py | 4 +++ 3 files changed, 40 insertions(+), 8 deletions(-) diff --git a/.claude/skills/run-to-fix/scripts/comparing.py b/.claude/skills/run-to-fix/scripts/comparing.py index d279d53c..f68ea1f5 100644 --- a/.claude/skills/run-to-fix/scripts/comparing.py +++ b/.claude/skills/run-to-fix/scripts/comparing.py @@ -69,8 +69,15 @@ def predictions_of(predictions: List[dict], new: List[TrialRow]) -> List[dict]: def trial_spent(entry: dict) -> float: - """What the round's trial runs cost, from the list of them in the ledger.""" - return sum(t["cost_usd"] for t in entry.get("trial_runs") or []) + """What the round's trial runs cost, from the list of them in the ledger. A run whose trials do not all report + a cost counts only with the person's figure for it.""" + total = 0.0 + for t in entry.get("trial_runs") or []: + if not t.get("cost_complete", True) and t.get("cost_from") != "the person": + raise LoopError(f"the money the trial run {t['job_dir']} spent is unknown: some of its trials report no " + "cost. Record it again with loop.py decide --stop 2 and --trial-cost-usd.") + total += t["cost_usd"] + return total def spent_in_round(entry: dict, record: dict, new_dir: str, new_cost: Optional[float]) -> Optional[float]: @@ -100,9 +107,11 @@ def stop_rule(result_verdict: str, groups: List[dict], budget: float, spent: Opt f"{TARGET_TRIALS} trials each", "usd": price.usd, "trials": price.trials, "unpriced_tasks": price.unpriced_tasks, "text": describe(price)} left = budget - spent if spent is not None else None - if cheapest is not None: - if cheapest["usd"] is None or left is None: - not_checked.append("the budget: no trial of the groups' tasks, or of the new run, reports a cost") + if left is None: + not_checked.append("the budget: the money this round spent is unknown, because some trials report no cost") + elif cheapest is not None: + if cheapest["usd"] is None: + not_checked.append("the budget: no trial of the groups' tasks reports a cost") elif left < cheapest["usd"]: reasons.append(f"the budget left, {usd(left)}, is under the price of the cheapest option, " f"{usd(cheapest['usd'])}") @@ -144,7 +153,14 @@ def cmd_compare(args: argparse.Namespace) -> None: entry = ledger_round(ledger, os.path.basename(round_dir)) classes, signatures = classes_of(new_rows, adapter.limits) groups = failure_groups(new_rows, classes, signatures, None) - spent = spent_in_round(entry, record, new_dir, runs["new"]["cost_total"]) + missing = sum(r.cost_usd is None for r in new_rows) + recorded = sum(r.cost_usd for r in new_rows if r.cost_usd is not None) + if args.new_cost_usd is not None and not missing: + raise LoopError("every trial of the new run reports its cost; --new-cost-usd is for a run whose trials do not") + if args.new_cost_usd is not None and args.new_cost_usd < recorded: + raise LoopError(f"--new-cost-usd {usd(args.new_cost_usd)} is under the recorded {usd(recorded)}") + new_cost = (args.new_cost_usd if missing else recorded) if new_rows else None + spent = spent_in_round(entry, record, new_dir, new_cost) rule = stop_rule(result_verdict, groups, float(record["budget_usd"]), spent, task_costs(new_rows)) result = { "schema": "run-to-fix.result/v1", "created": now(), "round": os.path.basename(round_dir), diff --git a/.claude/skills/run-to-fix/scripts/deciding.py b/.claude/skills/run-to-fix/scripts/deciding.py index 821bfc35..599a8d12 100644 --- a/.claude/skills/run-to-fix/scripts/deciding.py +++ b/.claude/skills/run-to-fix/scripts/deciding.py @@ -122,9 +122,20 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: job_dir = checked.dir this_run: List[TrialRow] = read_rows(job_dir, threshold) costs = [r.cost_usd for r in this_run] + recorded = sum(c for c in costs if c is not None) + missing = sum(c is None for c in costs) + if missing and args.trial_cost_usd is None: + raise LoopError(f"{missing} of {len(costs)} trials of the trial run report no cost, so the money it spent is " + f"unknown. Give the cost of the whole run with --trial-cost-usd, at least the recorded " + f"{usd(recorded)}.") + if args.trial_cost_usd is not None and not missing: + raise LoopError("every trial of the trial run reports its cost; --trial-cost-usd is for a run whose trials do not") + if args.trial_cost_usd is not None and args.trial_cost_usd < recorded: + raise LoopError(f"--trial-cost-usd {usd(args.trial_cost_usd)} is under the recorded {usd(recorded)}") runs = [t for t in entry.setdefault("trial_runs", []) if t["job_dir"] != job_dir] runs.append({"job_dir": job_dir, "ref": selection["ref"], "trials": len(this_run), - "cost_usd": sum(c for c in costs if c is not None), "cost_complete": all(c is not None for c in costs), + "cost_usd": args.trial_cost_usd if missing else recorded, "cost_recorded_usd": recorded, + "cost_from": "the person" if missing else "the trials", "cost_complete": not missing, "added": now()}) entry["trial_runs"] = runs save_ledger(args.loop, ledger) @@ -180,7 +191,8 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: "ROUND": os.path.basename(round_dir), "LOOP": args.loop, "BUILD": f"{build['path']} from {build['ref'][:12]}" if build["path"] else f"{build['reason']} ({build['ref'][:12]})", "TRIAL_JOB": ", ".join(t["job_dir"] for t in same), "TRIALS": str(len(rows)), - "SPENT": usd(spent) + ("" if trial_record["cost_complete"] else " at least: some trials report no cost"), + "SPENT": usd(spent) + "".join(f" (the person's figure for {t['job_dir']}, whose trials record " + f"{usd(t['cost_recorded_usd'])})" for t in runs if t.get("cost_from") == "the person"), "LEFT": usd(left), "BUDGET": usd(float(record["budget_usd"])), "RESULTS": "\n".join(results), "OPTIONS": "\n".join(option_rows), "REPORT_NOTE": "".join(f"\n- {p['candidate']}: {REPORT_NOT_READ}." for p in entry.get("predictions") or [] diff --git a/.claude/skills/run-to-fix/scripts/loop.py b/.claude/skills/run-to-fix/scripts/loop.py index bde9bdd6..e505a007 100644 --- a/.claude/skills/run-to-fix/scripts/loop.py +++ b/.claude/skills/run-to-fix/scripts/loop.py @@ -65,6 +65,8 @@ def main(argv: List[str]) -> int: de.add_argument("--loop", required=True) de.add_argument("--stop", type=int, required=True, choices=(1, 2)) de.add_argument("--trial-job", help="stop 2: the job folder of the trial run") + de.add_argument("--trial-cost-usd", type=float, + help="stop 2: the person's figure for what the trial run cost, when its trials do not all report one") de.set_defaults(func=cmd_decide) an = sub.add_parser("answer", help="record the person's answer of a stop") an.add_argument("--loop", required=True) @@ -87,6 +89,8 @@ def main(argv: List[str]) -> int: cp.add_argument("--new", required=True, help="the job folder of the new run, as for read --run") cp.add_argument("--against", required=True, choices=("run", "baseline"), help="compare with the round's run (the code before the fix) or with its baseline") + cp.add_argument("--new-cost-usd", type=float, + help="the person's figure for what the new run cost, when its trials do not all report one") cp.set_defaults(func=cmd_compare) args = ap.parse_args(argv) try: From b41ddeb3c42b3c3f59bccf655f980c4c5d0f7e9b Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:04:44 +0100 Subject: [PATCH 44/92] test(run-to-fix): count a trial run only for the tasks and commit select chose --- .../skills/run-to-fix/tests/test_decide.py | 73 +++++++++++++++---- 1 file changed, 59 insertions(+), 14 deletions(-) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index 69c883d5..bf559ed7 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -5,7 +5,7 @@ from typing import List from agentrepo import RepoCase -from jobs import Result, load, run, trial, write, write_job, write_json +from jobs import Result, Trial, load, run, trial, write, write_job, write_json KEEP_GOOD = "Keep good. bad breaks test_other." @@ -42,6 +42,10 @@ def trial_job(self) -> str: trials.append(t) return write_job(self.jobs, "trial", "alpha", "harbor", trials) + def code_ref(self) -> str: + """The commit the person states the trial run ran: the one select chose.""" + return load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["selection"]["ref"] + def chosen(self) -> str: self.kept() r = self.select(["--size", "guards", "--budget-usd", "0.5", "--text", "targets and guards"]) @@ -175,7 +179,7 @@ def test_without_touched_there_are_no_guards(self) -> None: class StopTwo(Flow): def test_the_page_gives_three_options_with_their_cost(self) -> None: job = self.chosen() - r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job]) + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--code-ref", self.code_ref()]) self.assertEqual(r.code, 0, r.out + r.err) trial_record = load(self.round_file("trial.json")) self.assertAlmostEqual(trial_record["cost_usd"], 0.33) @@ -200,7 +204,7 @@ def test_the_page_gives_three_options_with_their_cost(self) -> None: def test_the_answer_of_stop_two(self) -> None: job = self.chosen() - run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job]) + run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--code-ref", self.code_ref()]) words = "A full pass, the targets look fixed." r = run(["answer", "--loop", self.loop, "--stop", "2", "--choice", "full", "--text", words]) self.assertEqual(r.code, 0, r.out + r.err) @@ -218,7 +222,7 @@ def test_compare_reads_the_selection_and_the_predictions(self) -> None: def test_the_stop_rule_counts_the_trial_run_and_the_new_run(self) -> None: job = self.chosen() - run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job]) + run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--code-ref", self.code_ref()]) run(["answer", "--loop", self.loop, "--stop", "2", "--choice", "full", "--text", "a full pass"]) full = write_job(self.jobs, "full", "alpha", "harbor", [trial(f"{t}__f", f"bench/{t}", 1.0) for t in ("p1", "p2", "p3", "f1", "f2", "f3")]) @@ -238,11 +242,11 @@ def rerun_job(self) -> str: def after_rerun(self) -> str: first = self.chosen() - r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", first]) + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", first, "--code-ref", self.code_ref()]) self.assertEqual(r.code, 0, r.out + r.err) run(["answer", "--loop", self.loop, "--stop", "2", "--choice", "rerun", "--text", "run them again"]) second = self.rerun_job() - r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", second]) + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", second, "--code-ref", self.code_ref()]) self.assertEqual(r.code, 0, r.out + r.err) return second @@ -266,13 +270,13 @@ def test_the_second_decision_counts_both_trial_runs(self) -> None: def test_deciding_twice_on_one_trial_run_counts_it_once(self) -> None: first = self.chosen() for _ in range(2): - run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", first]) + run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", first, "--code-ref", self.code_ref()]) self.assertEqual(len(load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["trial_runs"]), 1) self.assertAlmostEqual(load(self.round_file("trial.json"))["cost_usd"], 0.33) def test_select_after_a_trial_run_checks_the_budget_left(self) -> None: first = self.chosen() - run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", first]) + run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", first, "--code-ref", self.code_ref()]) r = self.select(["--size", "targets", "--budget-usd", "9.95", "--text", "again"]) self.assertNotEqual(r.code, 0) self.assertIn("$9.67", r.err) @@ -302,7 +306,7 @@ def test_a_trial_job_named_inside_a_folder_of_several_jobs(self) -> None: job = self.trial_job() write_job(holder, "other", "alpha", "harbor", [trial("x__o", "bench/x", 1.0)]) self.jobs = outer - r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", f"{holder}::trial"]) + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", f"{holder}::trial", "--code-ref", self.code_ref()]) self.assertEqual(r.code, 0, r.out + r.err) record = load(self.round_file("trial.json")) self.assertEqual(record["job_dir"], job) @@ -371,7 +375,7 @@ def test_the_pages_say_the_report_is_not_read(self) -> None: self.kept_report() self.select_report(["--size", "guards", "--budget-usd", "0.5", "--text", "the guards"]) job = write_job(self.jobs, "trial", "alpha", "harbor", [trial(f"p3-{k}__t", "bench/p3", 1.0) for k in range(3)]) - r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job]) + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--code-ref", self.code_ref()]) self.assertEqual(r.code, 0, r.out + r.err) self.assertIn("rep: this version does not read the agent's report in the new run", self.page("decision-2.md")) r = run(["compare", "--loop", self.loop, "--new", job, "--against", "run"]) @@ -428,7 +432,7 @@ def job_without_one_cost(self) -> str: def test_stop_two_refuses_an_unknown_spending(self) -> None: job = self.job_without_one_cost() - r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job]) + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--code-ref", self.code_ref()]) self.assertNotEqual(r.code, 0) self.assertIn("--trial-cost-usd", r.err) self.assertIn("$0.29", r.err) @@ -437,7 +441,7 @@ def test_stop_two_refuses_an_unknown_spending(self) -> None: def test_the_person_s_figure_is_the_spending(self) -> None: job = self.job_without_one_cost() - r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--trial-cost-usd", "0.40"]) + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--trial-cost-usd", "0.40", "--code-ref", self.code_ref()]) self.assertEqual(r.code, 0, r.out + r.err) record = load(self.round_file("trial.json")) self.assertAlmostEqual(record["cost_usd"], 0.40) @@ -449,13 +453,13 @@ def test_the_person_s_figure_is_the_spending(self) -> None: def test_a_figure_under_the_recorded_cost_is_refused(self) -> None: job = self.job_without_one_cost() - r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--trial-cost-usd", "0.10"]) + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--trial-cost-usd", "0.10", "--code-ref", self.code_ref()]) self.assertNotEqual(r.code, 0) self.assertIn("$0.29", r.err) def test_an_old_trial_run_of_unknown_cost_stops_select(self) -> None: job = self.job_without_one_cost() - run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--trial-cost-usd", "0.40"]) + run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--trial-cost-usd", "0.40", "--code-ref", self.code_ref()]) path = os.path.join(self.loop, "ledger.json") ledger = load(path) entry = ledger["rounds"][0]["trial_runs"][0] @@ -465,3 +469,44 @@ def test_an_old_trial_run_of_unknown_cost_stops_select(self) -> None: r = self.select(["--size", "targets", "--budget-usd", "1", "--text", "again"]) self.assertNotEqual(r.code, 0) self.assertIn("unknown", r.err) + + + +class TrialRunIdentity(Flow): + """A trial run counts only when it holds the tasks select chose, each with its trials, on the stated commit.""" + + def decide_on(self, trials: List[Trial], code_ref: str) -> Result: + self.chosen() + job = write_job(self.jobs, "other", "alpha", "harbor", trials) + return run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--code-ref", code_ref]) + + def three_each(self, tasks: List[str]) -> List[Trial]: + return [trial(f"{t}-{k}__o", f"bench/{t}", 1.0) for t in tasks for k in range(3)] + + def test_a_task_outside_the_selection_is_refused(self) -> None: + r = self.decide_on(self.three_each(["f1", "f2", "p1", "unrelated"]), self.repo.good) + self.assertNotEqual(r.code, 0) + self.assertIn("bench/unrelated", r.err) + self.assertEqual(load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["trial_runs"], []) + + def test_a_missing_target_is_refused(self) -> None: + r = self.decide_on(self.three_each(["f1", "p1"]), self.repo.good) + self.assertNotEqual(r.code, 0) + self.assertIn("bench/f2", r.err) + + def test_a_task_with_another_count_of_trials_is_refused(self) -> None: + r = self.decide_on([trial(f"{t}__o", f"bench/{t}", 1.0) for t in ("f1", "f2", "p1")], self.repo.good) + self.assertNotEqual(r.code, 0) + self.assertIn("3 trials", r.err) + + def test_another_commit_is_refused(self) -> None: + r = self.decide_on(self.three_each(["f1", "f2", "p1"]), self.repo.bad) + self.assertNotEqual(r.code, 0) + self.assertIn(self.repo.good[:12], r.err) + + def test_the_page_shows_the_commit_as_the_person_s_statement(self) -> None: + r = self.decide_on(self.three_each(["f1", "f2", "p1"]), self.repo.good[:9]) + self.assertEqual(r.code, 0, r.out + r.err) + self.assertIn(f"{self.repo.good[:12]}, as the person states", self.page("decision-2.md")) + self.assertEqual(load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["trial_runs"][0]["code_from"], + "the person") From 433f368b00417fe7bddd0ba4d5c6530c618fb6d1 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:09:29 +0100 Subject: [PATCH 45/92] fix(run-to-fix): count a trial run only for the tasks and commit select chose decide --stop 2 takes --code-ref, the commit the person states the trial run ran, and refuses it unless it is the commit select chose. The trial job must hold only selected tasks, every target, and the selection's count of trials for each task. The page shows the commit as the person's statement, since a job does not record the code it ran. --- .../skills/run-to-fix/assets/decision-2.md | 2 +- .claude/skills/run-to-fix/scripts/deciding.py | 28 +++++++++++++++++-- .claude/skills/run-to-fix/scripts/loop.py | 1 + 3 files changed, 28 insertions(+), 3 deletions(-) diff --git a/.claude/skills/run-to-fix/assets/decision-2.md b/.claude/skills/run-to-fix/assets/decision-2.md index ac8295bc..4dd6ff94 100644 --- a/.claude/skills/run-to-fix/assets/decision-2.md +++ b/.claude/skills/run-to-fix/assets/decision-2.md @@ -1,6 +1,6 @@ # Stop 2 of {{ROUND}}: what to run next -Build: {{BUILD}}. Trial runs on it: {{TRIAL_JOB}}, {{TRIALS}} trials. Spent in this round: {{SPENT}}. Budget left: +Build: {{BUILD}}. Code of the trial runs: {{CODE}}. Trial runs on it: {{TRIAL_JOB}}, {{TRIALS}} trials. Spent in this round: {{SPENT}}. Budget left: {{LEFT}} of {{BUDGET}}. | tasks | trials passed before | trials passed in the trial run | test | diff --git a/.claude/skills/run-to-fix/scripts/deciding.py b/.claude/skills/run-to-fix/scripts/deciding.py index 599a8d12..22da46d0 100644 --- a/.claude/skills/run-to-fix/scripts/deciding.py +++ b/.claude/skills/run-to-fix/scripts/deciding.py @@ -2,11 +2,13 @@ from __future__ import annotations import argparse +import collections import os import re from typing import Dict, List, Optional from checking import load_candidate +from checkout import resolve from classifying import read_rows from comparing import REPORT_NOT_READ, TARGET_TRIALS, passes, targets_of, trial_spent from pricing import describe, price_of, task_costs, usd @@ -121,6 +123,25 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: + ", ".join(repr(a) for a in checked.agents)) job_dir = checked.dir this_run: List[TrialRow] = read_rows(job_dir, threshold) + code_ref = resolve(adapter.checkout, args.code_ref) + if code_ref != selection["ref"]: + raise LoopError(f"a trial run of this round runs the commit select chose, {selection['ref'][:12]}; --code-ref " + f"names {code_ref[:12]}") + chosen = set(selection["targets"]) | set(selection["guards"]) | set(selection["canaries"]) + per_task = collections.Counter(r.task for r in this_run) + problems = [] + outside = sorted(set(per_task) - chosen) + if outside: + problems.append("tasks outside the selection: " + ", ".join(outside)) + absent = sorted(set(selection["targets"]) - set(per_task)) + if absent: + problems.append("targets it lacks: " + ", ".join(absent)) + uneven = sorted(t for t, n in per_task.items() if n != selection["repeat"]) + if uneven: + problems.append(f"tasks without {selection['repeat']} trials: " + + ", ".join(f"{t} ({per_task[t]})" for t in uneven)) + if problems: + raise LoopError("the trial job is not a trial run of the selection: " + "; ".join(problems)) costs = [r.cost_usd for r in this_run] recorded = sum(c for c in costs if c is not None) missing = sum(c is None for c in costs) @@ -136,6 +157,7 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: runs.append({"job_dir": job_dir, "ref": selection["ref"], "trials": len(this_run), "cost_usd": args.trial_cost_usd if missing else recorded, "cost_recorded_usd": recorded, "cost_from": "the person" if missing else "the trials", "cost_complete": not missing, + "code_ref": code_ref, "code_from": "the person", "added": now()}) entry["trial_runs"] = runs save_ledger(args.loop, ledger) @@ -191,6 +213,7 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: "ROUND": os.path.basename(round_dir), "LOOP": args.loop, "BUILD": f"{build['path']} from {build['ref'][:12]}" if build["path"] else f"{build['reason']} ({build['ref'][:12]})", "TRIAL_JOB": ", ".join(t["job_dir"] for t in same), "TRIALS": str(len(rows)), + "CODE": f"{selection['ref'][:12]}, as the person states; a trial job does not record the commit it ran", "SPENT": usd(spent) + "".join(f" (the person's figure for {t['job_dir']}, whose trials record " f"{usd(t['cost_recorded_usd'])})" for t in runs if t.get("cost_from") == "the person"), "LEFT": usd(left), "BUDGET": usd(float(record["budget_usd"])), "RESULTS": "\n".join(results), @@ -208,8 +231,9 @@ def cmd_decide(args: argparse.Namespace) -> None: record = read_input(round_dir) if args.stop == 1: decide_one(args, round_dir, record) - elif args.trial_job is None: - raise LoopError("decide --stop 2 needs --trial-job: the job folder of the trial run") + elif args.trial_job is None or args.code_ref is None: + raise LoopError("decide --stop 2 needs --trial-job, the job folder of the trial run, and --code-ref, the commit " + "the person states it ran") else: decide_two(args, round_dir, record) diff --git a/.claude/skills/run-to-fix/scripts/loop.py b/.claude/skills/run-to-fix/scripts/loop.py index e505a007..637dc122 100644 --- a/.claude/skills/run-to-fix/scripts/loop.py +++ b/.claude/skills/run-to-fix/scripts/loop.py @@ -65,6 +65,7 @@ def main(argv: List[str]) -> int: de.add_argument("--loop", required=True) de.add_argument("--stop", type=int, required=True, choices=(1, 2)) de.add_argument("--trial-job", help="stop 2: the job folder of the trial run") + de.add_argument("--code-ref", help="stop 2: the commit the person states the trial run ran") de.add_argument("--trial-cost-usd", type=float, help="stop 2: the person's figure for what the trial run cost, when its trials do not all report one") de.set_defaults(func=cmd_decide) From 73e0d35a6623124db59089cc6222b8c3ad89d377 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:10:18 +0100 Subject: [PATCH 46/92] test(run-to-fix): leave no earlier success behind a failing check, and run a changed adapter again --- .claude/skills/run-to-fix/tests/test_check.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_check.py b/.claude/skills/run-to-fix/tests/test_check.py index 6cfc1b3b..ce52dc98 100644 --- a/.claude/skills/run-to-fix/tests/test_check.py +++ b/.claude/skills/run-to-fix/tests/test_check.py @@ -266,3 +266,39 @@ def test_a_candidate_without_an_effect_is_refused(self) -> None: r = run(args) self.assertNotEqual(r.code, 0, args) self.assertIn("register the candidate again", r.err) + + +class StaleEvidence(RepoCase): + """A check that fails leaves no earlier success behind, and a changed adapter runs its commands again.""" + + def change_adapter(self, name: str, line: str) -> None: + path = os.path.join(self.loop, "adapter.json") + data = load(path) + data["commands"][name] = line + write(path, json.dumps(data)) + + def test_a_failing_suite_removes_the_old_checks(self) -> None: + self.start(self.commands(), ["run_tasks"]) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") + self.check("good") + self.change_adapter("test", "exit 23 # {worktree}") + r = run(["check", "--loop", self.loop, "--candidate", "good"]) + self.assertNotEqual(r.code, 0) + self.assertFalse(os.path.exists(self.round_file("candidates/good/checks.json"))) + + def test_a_changed_replay_is_not_served_from_the_cache(self) -> None: + self.start(self.commands(), ["run_tasks"]) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") + self.check("good") + self.change_adapter("replay", "exit 23 # {worktree} {ref} {job_dir} {out_json}") + r = run(["check", "--loop", self.loop, "--candidate", "good"]) + self.assertNotEqual(r.code, 0) + self.assertIn("commands.replay", r.err) + self.assertFalse(os.path.exists(self.round_file("candidates/good/checks.json"))) + + def test_an_unchanged_adapter_uses_the_cache(self) -> None: + self.start(self.commands(), ["run_tasks"]) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") + self.check("good") + self.check("good") + self.assertEqual(self.marker("replay.marker"), [self.repo.test_ref, self.repo.good]) From 664ad4212e7ab2e20bd2a2715b6288bd73ee2081 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:12:11 +0100 Subject: [PATCH 47/92] fix(run-to-fix): leave no earlier success behind a failing check, and run a changed adapter again check removes a candidate's checks.json before it runs, so a check that fails part way leaves nothing to read. The suite and replay caches of a round record the adapter's SHA-256 and are used only under the same adapter. --- .claude/skills/run-to-fix/scripts/checking.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/.claude/skills/run-to-fix/scripts/checking.py b/.claude/skills/run-to-fix/scripts/checking.py index e8737472..fc5c4285 100644 --- a/.claude/skills/run-to-fix/scripts/checking.py +++ b/.claude/skills/run-to-fix/scripts/checking.py @@ -146,15 +146,16 @@ def suite_failures(adapter: Adapter, name: str, outcome: Outcome) -> List[str]: def check_suite(adapter: Adapter, args: argparse.Namespace, round_dir: str, cdir: str, base: str, ref: str) -> dict: cache = os.path.join(round_dir, "base", "suite.json") - if os.path.isfile(cache) and read_json(cache)["ref"] == base: - base_failures = read_json(cache)["failures"] + stored = read_json(cache) if os.path.isfile(cache) else {} + if stored.get("ref") == base and stored.get("adapter_sha256") == adapter.sha256: + base_failures = stored["failures"] else: o = on_ref(adapter, args.loop, round_dir, "test", base, {}, os.path.join(round_dir, "base", "suite.log"), args.command_timeout_s) if o.ran is None: return {"ran": False, "reason": o.reason} base_failures = suite_failures(adapter, "base_ref", o) - write_json(cache, {"ref": base, "failures": base_failures}) + write_json(cache, {"ref": base, "adapter_sha256": adapter.sha256, "failures": base_failures}) o = on_ref(adapter, args.loop, round_dir, "test", ref, {}, os.path.join(cdir, "suite.log"), args.command_timeout_s) if o.ran is None: return {"ran": False, "reason": o.reason} @@ -207,7 +208,8 @@ def check_replay(adapter: Adapter, args: argparse.Namespace, round_dir: str, bef out = replay_file(round_dir, sha, against) if os.path.isfile(out): stored = read_json(out) - if stored.get("_loop_ref") == sha and stored.get("_loop_before_ref") == against: + if (stored.get("_loop_ref") == sha and stored.get("_loop_before_ref") == against + and stored.get("_loop_adapter_sha256") == adapter.sha256): labels[when] = replay_labels(out, rows) continue values = {"ref": sha, "before_ref": against, "base_ref": base, "job_dir": job_dir, "out_json": out} @@ -221,7 +223,7 @@ def check_replay(adapter: Adapter, args: argparse.Namespace, round_dir: str, bef return {"ran": False, "reason": o.reason} ensure_ok("replay", o) data = replay_labels(out, rows) - write_json(out, dict(data, _loop_ref=sha, _loop_before_ref=against)) + write_json(out, dict(data, _loop_ref=sha, _loop_before_ref=against, _loop_adapter_sha256=adapter.sha256)) labels[when] = data success = adapter.replay_success_labels before, after = false_successes(labels["before"], rows, success), false_successes(labels["after"], rows, success) @@ -246,6 +248,10 @@ def cmd_check(args: argparse.Namespace) -> None: record = read_input(round_dir) cdir = os.path.join(round_dir, "candidates", args.candidate) cand = load_candidate(os.path.join(cdir, "candidate.json")) + # a check that fails part way must not leave the success of an earlier check to be read + stale = os.path.join(cdir, "checks.json") + if os.path.exists(stale): + os.remove(stale) job_dir = record["run"]["job_dir"] rows = read_rows(job_dir, float(record["pass_threshold"])) base = resolve(adapter.checkout, adapter.base_ref) From 981df7d0ce3122981c8e4f8c8f504d036c7ad00b Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:12:42 +0100 Subject: [PATCH 48/92] test(run-to-fix): run a written paid command in paths that hold a space and a quote --- .claude/skills/run-to-fix/tests/test_check.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/tests/test_check.py b/.claude/skills/run-to-fix/tests/test_check.py index ce52dc98..99a5ad4f 100644 --- a/.claude/skills/run-to-fix/tests/test_check.py +++ b/.claude/skills/run-to-fix/tests/test_check.py @@ -3,8 +3,9 @@ import json import os +import subprocess -from agentrepo import RepoCase, git +from agentrepo import RepoCase, git, make_repo from jobs import load, run, write @@ -302,3 +303,23 @@ def test_an_unchanged_adapter_uses_the_cache(self) -> None: self.check("good") self.check("good") self.assertEqual(self.marker("replay.marker"), [self.repo.test_ref, self.repo.good]) + + +class PaidScriptsRun(RepoCase): + """A written paid command runs as written, in a checkout and a loop folder whose paths hold a space and a quote.""" + + def test_a_written_replay_runs_and_removes_its_worktree(self) -> None: + spaced = os.path.join(self.tmp, "it's a dir") + os.makedirs(spaced) + self.repo = make_repo(spaced) + self.loop = os.path.join(spaced, "loop space") + os.makedirs(self.loop) + self.start(self.commands(), ["run_tasks", "replay"]) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") + self.check("good") + script = self.round_file(os.path.join("commands", f"replay-{self.repo.good[:12]}.sh")) + p = subprocess.run(["bash", script], capture_output=True, text=True, stdin=subprocess.DEVNULL, timeout=120) + self.assertEqual(p.returncode, 0, p.stdout + p.stderr) + self.assertEqual(self.marker("replay.marker"), [self.repo.good]) + self.assertEqual(git(self.repo.path, ["worktree", "list", "--porcelain"]).count("worktree "), 1) + self.assertEqual(os.listdir(os.path.join(self.loop, "worktrees")), []) From 646a88613fa41d4e4e63877b17bf8c74f5db2238 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:17:29 +0100 Subject: [PATCH 49/92] fix(run-to-fix): run a written paid command in paths that hold a space and a quote The cleanup of a written paid command is quoted whole as the argument of trap; quoting its paths inside a single-quoted trap broke at a space. The folder of a replay's output exists before the command is written, so the written replay can write its file. --- .claude/skills/run-to-fix/scripts/checking.py | 1 + .claude/skills/run-to-fix/scripts/checkout.py | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/scripts/checking.py b/.claude/skills/run-to-fix/scripts/checking.py index fc5c4285..8f0a30f7 100644 --- a/.claude/skills/run-to-fix/scripts/checking.py +++ b/.claude/skills/run-to-fix/scripts/checking.py @@ -213,6 +213,7 @@ def check_replay(adapter: Adapter, args: argparse.Namespace, round_dir: str, bef labels[when] = replay_labels(out, rows) continue values = {"ref": sha, "before_ref": against, "base_ref": base, "job_dir": job_dir, "out_json": out} + os.makedirs(os.path.dirname(out), exist_ok=True) o = on_ref(adapter, args.loop, round_dir, "replay", sha, values, out[:-len(".json")] + ".log", args.command_timeout_s) if o.ran is None: diff --git a/.claude/skills/run-to-fix/scripts/checkout.py b/.claude/skills/run-to-fix/scripts/checkout.py index 23ba916f..d1322126 100644 --- a/.claude/skills/run-to-fix/scripts/checkout.py +++ b/.claude/skills/run-to-fix/scripts/checkout.py @@ -66,5 +66,6 @@ def write_paid_on_ref(path: str, checkout: str, loop_dir: str, sha: str, line: s with open(path, "w", encoding="utf-8", newline="\n") as fh: fh.write("#!/usr/bin/env bash\nset -euo pipefail\n") fh.write(f"git -C {q(checkout)} worktree add --detach {q(tree)} {q(sha)}\n") - fh.write(f"trap 'git -C {q(checkout)} worktree remove --force {q(tree)}' EXIT\n") + cleanup = f"git -C {q(checkout)} worktree remove --force {q(tree)}" + fh.write(f"trap {q(cleanup)} EXIT\n") fh.write(f"cd {q(tree)}\n{fill(line, dict(values, worktree=tree))}\n") From 6b8cf5c0388cae03d9395ac8289941df8d302302 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:17:58 +0100 Subject: [PATCH 50/92] test(run-to-fix): stop the adapter command and remove its worktree when the loop is stopped --- .claude/skills/run-to-fix/tests/test_check.py | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/tests/test_check.py b/.claude/skills/run-to-fix/tests/test_check.py index 99a5ad4f..5e10fcba 100644 --- a/.claude/skills/run-to-fix/tests/test_check.py +++ b/.claude/skills/run-to-fix/tests/test_check.py @@ -3,10 +3,13 @@ import json import os +import signal import subprocess +import sys +import time from agentrepo import RepoCase, git, make_repo -from jobs import load, run, write +from jobs import LOOP, load, run, write class Candidate(RepoCase): @@ -323,3 +326,45 @@ def test_a_written_replay_runs_and_removes_its_worktree(self) -> None: self.assertEqual(self.marker("replay.marker"), [self.repo.good]) self.assertEqual(git(self.repo.path, ["worktree", "list", "--porcelain"]).count("worktree "), 1) self.assertEqual(os.listdir(os.path.join(self.loop, "worktrees")), []) + + +class Cancellation(RepoCase): + """A loop stopped by SIGTERM or Ctrl+C stops the adapter command it runs and removes its worktree.""" + + def stop_during_a_command(self, signum: int) -> None: + pid_file = os.path.join(self.tmp, "command.pid") + commands = self.commands() + commands["test_one"] = f"cd {{worktree}} && echo $$ > {pid_file} && exec sleep 60 # {{test}}" + self.start(commands, ["run_tasks"]) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") + proc = subprocess.Popen([sys.executable, LOOP, "check", "--loop", self.loop, "--candidate", "good"], + stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1")) + deadline = time.monotonic() + 30 + while not (os.path.isfile(pid_file) and open(pid_file).read().strip()) and time.monotonic() < deadline: + time.sleep(0.1) + command_pid = int(open(pid_file).read()) + proc.send_signal(signum) + _, err = proc.communicate(timeout=60) + self.assertNotEqual(proc.returncode, 0) + self.assertIn("stopped", err) + deadline = time.monotonic() + 10 + while self.alive(command_pid) and time.monotonic() < deadline: + time.sleep(0.1) + self.assertFalse(self.alive(command_pid)) + self.assertEqual(git(self.repo.path, ["worktree", "list", "--porcelain"]).count("worktree "), 1) + self.assertEqual(os.listdir(os.path.join(self.loop, "worktrees")), []) + self.assertFalse(os.path.exists(self.round_file("candidates/good/checks.json"))) + + def alive(self, pid: int) -> bool: + try: + os.kill(pid, 0) + except ProcessLookupError: + return False + return True + + def test_sigterm(self) -> None: + self.stop_during_a_command(signal.SIGTERM) + + def test_ctrl_c(self) -> None: + self.stop_during_a_command(signal.SIGINT) From c82a7e47d98fd16a80c347f9fac1cca22ca48322 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:18:18 +0100 Subject: [PATCH 51/92] test(run-to-fix): close the file of the command's process id --- .claude/skills/run-to-fix/tests/test_check.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.claude/skills/run-to-fix/tests/test_check.py b/.claude/skills/run-to-fix/tests/test_check.py index 5e10fcba..622a11ca 100644 --- a/.claude/skills/run-to-fix/tests/test_check.py +++ b/.claude/skills/run-to-fix/tests/test_check.py @@ -341,9 +341,9 @@ def stop_during_a_command(self, signum: int) -> None: stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=dict(os.environ, PYTHONDONTWRITEBYTECODE="1")) deadline = time.monotonic() + 30 - while not (os.path.isfile(pid_file) and open(pid_file).read().strip()) and time.monotonic() < deadline: + while not self.read_pid(pid_file) and time.monotonic() < deadline: time.sleep(0.1) - command_pid = int(open(pid_file).read()) + command_pid = int(self.read_pid(pid_file)) proc.send_signal(signum) _, err = proc.communicate(timeout=60) self.assertNotEqual(proc.returncode, 0) @@ -356,6 +356,12 @@ def stop_during_a_command(self, signum: int) -> None: self.assertEqual(os.listdir(os.path.join(self.loop, "worktrees")), []) self.assertFalse(os.path.exists(self.round_file("candidates/good/checks.json"))) + def read_pid(self, path: str) -> str: + if not os.path.isfile(path): + return "" + with open(path, encoding="utf-8") as fh: + return fh.read().strip() + def alive(self, pid: int) -> bool: try: os.kill(pid, 0) From 6b4e22d6ff7519354f835fbc3683b5ac660dc534 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:18:55 +0100 Subject: [PATCH 52/92] fix(run-to-fix): stop the adapter command and remove its worktree when the loop is stopped SIGTERM raises as Ctrl+C does. run_command stops the command's whole process group on either, and the worktree is removed on the way out; the loop says it was stopped and exits with an error. --- .claude/skills/run-to-fix/scripts/loop.py | 8 +++++++- .claude/skills/run-to-fix/scripts/rounds.py | 12 ++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/scripts/loop.py b/.claude/skills/run-to-fix/scripts/loop.py index 637dc122..e56ff555 100644 --- a/.claude/skills/run-to-fix/scripts/loop.py +++ b/.claude/skills/run-to-fix/scripts/loop.py @@ -15,6 +15,7 @@ from __future__ import annotations import argparse +import signal import sys from typing import List @@ -25,7 +26,7 @@ from deciding import cmd_answer, cmd_decide from found import RuleError from reading import cmd_read -from rounds import LoopError +from rounds import LoopError, stop_on_signal from selecting import cmd_select from trials import TrialError @@ -94,11 +95,16 @@ def main(argv: List[str]) -> int: help="the person's figure for what the new run cost, when its trials do not all report one") cp.set_defaults(func=cmd_compare) args = ap.parse_args(argv) + # SIGTERM raises as Ctrl+C does, so a running adapter command is stopped and its worktree removed + signal.signal(signal.SIGTERM, stop_on_signal) try: args.func(args) except (LoopError, AdapterError, RuleError, TrialError) as exc: print(f"run-to-fix: {exc}", file=sys.stderr) return 1 + except KeyboardInterrupt: + print("run-to-fix: stopped by Ctrl+C; the command it ran is stopped", file=sys.stderr) + return 130 return 0 diff --git a/.claude/skills/run-to-fix/scripts/rounds.py b/.claude/skills/run-to-fix/scripts/rounds.py index bec8e6aa..afaf1f22 100644 --- a/.claude/skills/run-to-fix/scripts/rounds.py +++ b/.claude/skills/run-to-fix/scripts/rounds.py @@ -23,6 +23,14 @@ class LoopError(Exception): """The loop cannot go on; the message says why and what to do.""" +class Stopped(LoopError): + """A signal stopped the loop; the command it ran is stopped and its worktree removed on the way out.""" + + +def stop_on_signal(signum: int, frame: object) -> None: + raise Stopped(f"stopped by {signal.Signals(signum).name}; the command it ran is stopped") + + def now() -> str: return dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") @@ -139,6 +147,10 @@ def run_command(line: str, values: Dict[str, str], cwd: str, log_path: str, time os.killpg(proc.pid, signal.SIGKILL) proc.wait() code = None + except (Stopped, KeyboardInterrupt): + os.killpg(proc.pid, signal.SIGKILL) + proc.wait() + raise with open(log_path, encoding="utf-8", errors="replace") as fh: lines = fh.read().splitlines() return Ran(code=code, tail="\n".join(lines[-TAIL_LINES:]), log=log_path) From a80871bf796a9bc930ff9cf6532ec150d5dc9506 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:19:47 +0100 Subject: [PATCH 53/92] test(run-to-fix): refuse a found class that reads outside its trial through a link --- .../skills/run-to-fix/tests/test_classify.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_classify.py b/.claude/skills/run-to-fix/tests/test_classify.py index 1abbbc44..e694a9aa 100644 --- a/.claude/skills/run-to-fix/tests/test_classify.py +++ b/.claude/skills/run-to-fix/tests/test_classify.py @@ -445,3 +445,20 @@ def test_the_skill_and_run_analysis_count_the_same_trials(self) -> None: self.classify_ok() self.assertEqual(sorted(self.by_trial()), ["u1__s", "u2__s", "u3__s"]) self.assertEqual(load(self.round_file("groups.json"))["trials"], 3) + + +class FoundClassConfinement(LoopCase): + def test_a_rule_that_reaches_outside_the_trial_through_a_link_is_refused(self) -> None: + run_dir = write_job(self.jobs, "run", "alpha", "harbor", + [trial("p__r", "bench/p", 1.0), trial("f__r", "bench/f", 0.0)]) + outside = os.path.join(self.tmp, "outside.txt") + write(outside, "SYNTHETIC\n") + os.symlink(outside, os.path.join(run_dir, "f__r", "link")) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + write_json(os.path.join(self.loop, "found-classes.json"), {"schema": "run-to-fix.found-classes/v1", "rules": [ + {"id": "linked", "meaning": "A file the trial links to.", "file": "link", "regex": "SYNTHETIC"}]}) + self.read_ok(run_dir, None) + r = run(["classify", "--loop", self.loop]) + self.assertNotEqual(r.code, 0) + self.assertIn("outside the trial", r.err) + self.assertFalse(os.path.exists(self.round_file("trials.csv"))) From 495eb29c05055d7a8e9059783743154153cc48e8 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:20:47 +0100 Subject: [PATCH 54/92] fix(run-to-fix): refuse a found class that reads outside its trial through a link A rule's file is read only when its resolved path lies below the resolved trial folder; a link that leads elsewhere stops classify with the rule and the trial named. --- .claude/skills/run-to-fix/scripts/found.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.claude/skills/run-to-fix/scripts/found.py b/.claude/skills/run-to-fix/scripts/found.py index 3a22970c..922b800b 100644 --- a/.claude/skills/run-to-fix/scripts/found.py +++ b/.claude/skills/run-to-fix/scripts/found.py @@ -92,6 +92,9 @@ def apply(rule: Rule, trial_dir: str) -> str: path = os.path.join(trial_dir, rule.file) if not os.path.isfile(path): return "0" + root = os.path.realpath(trial_dir) + if not os.path.realpath(path).startswith(root + os.sep): + raise RuleError(f"the file {rule.file} of rule {rule.id} in {trial_dir} leads outside the trial folder") with open(path, encoding="utf-8", errors="replace") as fh: text = fh.read() if rule.regex is not None: From 8a27658212d77cef1d4bc80eebaf626f5d172ea4 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:21:16 +0100 Subject: [PATCH 55/92] test(run-to-fix): refuse a budget or a cost that is not a finite amount of 0 or more --- .../skills/run-to-fix/tests/test_decide.py | 23 +++++++++++++++++++ .claude/skills/run-to-fix/tests/test_read.py | 11 +++++++++ 2 files changed, 34 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index bf559ed7..818d13e6 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -510,3 +510,26 @@ def test_the_page_shows_the_commit_as_the_person_s_statement(self) -> None: self.assertIn(f"{self.repo.good[:12]}, as the person states", self.page("decision-2.md")) self.assertEqual(load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["trial_runs"][0]["code_from"], "the person") + + +class AmountsAreFinite(Flow): + def test_select_refuses_a_budget_that_is_not_an_amount(self) -> None: + self.kept() + r = self.select(["--size", "targets", "--budget-usd", "nan", "--text", "an invalid cap"]) + self.assertNotEqual(r.code, 0) + self.assertIn("--budget-usd", r.err) + self.assertFalse(os.path.exists(self.round_file("trial-tasks.txt"))) + + def test_stop_two_refuses_a_cost_that_is_not_an_amount(self) -> None: + job = self.chosen() + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--code-ref", self.code_ref(), + "--trial-cost-usd", "inf"]) + self.assertNotEqual(r.code, 0) + self.assertIn("--trial-cost-usd", r.err) + + def test_compare_refuses_a_cost_that_is_not_an_amount(self) -> None: + job = self.chosen() + r = run(["compare", "--loop", self.loop, "--new", job, "--against", "run", "--new-cost-usd", "nan"]) + self.assertNotEqual(r.code, 0) + self.assertIn("--new-cost-usd", r.err) + self.assertFalse(os.path.exists(self.round_file("result.json"))) diff --git a/.claude/skills/run-to-fix/tests/test_read.py b/.claude/skills/run-to-fix/tests/test_read.py index 5d900726..a67fbe8f 100644 --- a/.claude/skills/run-to-fix/tests/test_read.py +++ b/.claude/skills/run-to-fix/tests/test_read.py @@ -278,3 +278,14 @@ def test_read_needs_the_budget(self) -> None: self.assertNotEqual(r.code, 0) self.assertIn("--budget-usd", r.err) self.assertFalse(os.path.exists(os.path.join(self.loop, "round-01"))) + + +class BudgetIsAnAmount(LoopCase): + def test_read_refuses_a_budget_that_is_not_an_amount(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + for value in ("nan", "inf", "-1"): + r = run(["read", "--loop", self.loop, "--run", run_dir, "--budget-usd", value, "--pass-threshold", "1"]) + self.assertNotEqual(r.code, 0, value) + self.assertIn("--budget-usd", r.err) + self.assertFalse(os.path.exists(os.path.join(self.loop, "round-01"))) From 983a2826c8645dc58e474504f5dd2a88b76a9728 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:21:55 +0100 Subject: [PATCH 56/92] test(run-to-fix): give the amount tests runs of unknown cost, so the amount is what is refused --- .../skills/run-to-fix/tests/test_decide.py | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index 818d13e6..3cb0cc87 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -46,6 +46,16 @@ def code_ref(self) -> str: """The commit the person states the trial run ran: the one select chose.""" return load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["selection"]["ref"] + def job_without_one_cost(self) -> str: + self.chosen() + trials = [] + for task, passes, cost in (("f1", 3, 0.04), ("f2", 2, 0.06), ("p1", 3, 0.01)): + for k in range(3): + t = trial(f"{task}-{k}__t", f"bench/{task}", 1.0 if k < passes else 0.0) + t.cost = None if (task, k) == ("f1", 0) else cost + trials.append(t) + return write_job(self.jobs, "trial", "alpha", "harbor", trials) + def chosen(self) -> str: self.kept() r = self.select(["--size", "guards", "--budget-usd", "0.5", "--text", "targets and guards"]) @@ -420,16 +430,6 @@ def test_a_size_needs_the_budget(self) -> None: class IncompleteCosts(Flow): """A trial run whose trials do not all report a cost has no known spending until the person gives it.""" - def job_without_one_cost(self) -> str: - self.chosen() - trials = [] - for task, passes, cost in (("f1", 3, 0.04), ("f2", 2, 0.06), ("p1", 3, 0.01)): - for k in range(3): - t = trial(f"{task}-{k}__t", f"bench/{task}", 1.0 if k < passes else 0.0) - t.cost = None if (task, k) == ("f1", 0) else cost - trials.append(t) - return write_job(self.jobs, "trial", "alpha", "harbor", trials) - def test_stop_two_refuses_an_unknown_spending(self) -> None: job = self.job_without_one_cost() r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--code-ref", self.code_ref()]) @@ -521,14 +521,14 @@ def test_select_refuses_a_budget_that_is_not_an_amount(self) -> None: self.assertFalse(os.path.exists(self.round_file("trial-tasks.txt"))) def test_stop_two_refuses_a_cost_that_is_not_an_amount(self) -> None: - job = self.chosen() + job = self.job_without_one_cost() r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--code-ref", self.code_ref(), "--trial-cost-usd", "inf"]) self.assertNotEqual(r.code, 0) self.assertIn("--trial-cost-usd", r.err) def test_compare_refuses_a_cost_that_is_not_an_amount(self) -> None: - job = self.chosen() + job = self.job_without_one_cost() r = run(["compare", "--loop", self.loop, "--new", job, "--against", "run", "--new-cost-usd", "nan"]) self.assertNotEqual(r.code, 0) self.assertIn("--new-cost-usd", r.err) From bb12b04eba89923980ea7387f122c6eb8d1123d1 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:23:27 +0100 Subject: [PATCH 57/92] fix(run-to-fix): refuse a budget or a cost that is not a finite amount of 0 or more Every amount in US dollars on the command line, the budgets of read and select and the person's costs of decide and compare, is a finite number of 0 or more; nan, inf and negative values are refused before any file is written. --- .claude/skills/run-to-fix/scripts/loop.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/.claude/skills/run-to-fix/scripts/loop.py b/.claude/skills/run-to-fix/scripts/loop.py index e56ff555..75e490ea 100644 --- a/.claude/skills/run-to-fix/scripts/loop.py +++ b/.claude/skills/run-to-fix/scripts/loop.py @@ -15,6 +15,7 @@ from __future__ import annotations import argparse +import math import signal import sys from typing import List @@ -31,6 +32,17 @@ from trials import TrialError +def usd_amount(text: str) -> float: + """An amount in US dollars: a finite number, 0 or more.""" + try: + value = float(text) + except ValueError as exc: + raise argparse.ArgumentTypeError(f"{text!r} is not a number") from exc + if not math.isfinite(value) or value < 0: + raise argparse.ArgumentTypeError(f"{text!r} is not a finite amount of 0 or more") + return value + + def main(argv: List[str]) -> int: ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = ap.add_subparsers(dest="cmd", required=True) @@ -38,7 +50,7 @@ def main(argv: List[str]) -> int: rd.add_argument("--loop", required=True, help="the loop folder a person named; it holds adapter.json") rd.add_argument("--run", required=True, help="the job folder of the run, or a folder holding it as PATH::JOB") rd.add_argument("--baseline", help="the job folder the run is compared with, as for --run") - rd.add_argument("--budget-usd", type=float, required=True, help="the budget the person named for this round") + rd.add_argument("--budget-usd", type=usd_amount, required=True, help="the budget the person named for this round") rd.add_argument("--pass-threshold", type=float, default=1.0, help="reward at or above this is a pass (default 1)") rd.add_argument("--continue-after-stop", help="the person's words to start a round after a stop rule said stop") rd.set_defaults(func=cmd_read) @@ -67,7 +79,7 @@ def main(argv: List[str]) -> int: de.add_argument("--stop", type=int, required=True, choices=(1, 2)) de.add_argument("--trial-job", help="stop 2: the job folder of the trial run") de.add_argument("--code-ref", help="stop 2: the commit the person states the trial run ran") - de.add_argument("--trial-cost-usd", type=float, + de.add_argument("--trial-cost-usd", type=usd_amount, help="stop 2: the person's figure for what the trial run cost, when its trials do not all report one") de.set_defaults(func=cmd_decide) an = sub.add_parser("answer", help="record the person's answer of a stop") @@ -83,7 +95,7 @@ def main(argv: List[str]) -> int: se.add_argument("--canaries", type=int, required=True, help="how many passing tasks to draw as canaries") se.add_argument("--size", choices=("targets", "guards", "canaries"), help="the person's choice: targets, targets and guards, or all three") - se.add_argument("--budget-usd", type=float, help="the budget the person named for the trial run") + se.add_argument("--budget-usd", type=usd_amount, help="the budget the person named for the trial run") se.add_argument("--text", help="the person's words") se.set_defaults(func=cmd_select) cp = sub.add_parser("compare", help="stage 5: compare a new run and close the round") @@ -91,7 +103,7 @@ def main(argv: List[str]) -> int: cp.add_argument("--new", required=True, help="the job folder of the new run, as for read --run") cp.add_argument("--against", required=True, choices=("run", "baseline"), help="compare with the round's run (the code before the fix) or with its baseline") - cp.add_argument("--new-cost-usd", type=float, + cp.add_argument("--new-cost-usd", type=usd_amount, help="the person's figure for what the new run cost, when its trials do not all report one") cp.set_defaults(func=cmd_compare) args = ap.parse_args(argv) From 4729695a76854540e0406a6306dbf4708f372471 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:24:00 +0100 Subject: [PATCH 58/92] test(run-to-fix): show on stop 1 the trials whose replay keeps their recorded label --- .claude/skills/run-to-fix/tests/test_decide.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index 3cb0cc87..e51a3c10 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -533,3 +533,20 @@ def test_compare_refuses_a_cost_that_is_not_an_amount(self) -> None: self.assertNotEqual(r.code, 0) self.assertIn("--new-cost-usd", r.err) self.assertFalse(os.path.exists(self.round_file("result.json"))) + + +ADD_KEPT = (" && python3 -c \\"import json,sys; p=sys.argv[1]; d=json.load(open(p)); " + "d['kept_recorded'] = ['f3__r', 'p2__r']; json.dump(d, open(p, 'w'))\\" {out_json}") + + +class KeptRecorded(Flow): + def test_the_page_shows_the_trials_that_keep_their_recorded_label(self) -> None: + commands = self.commands() + commands["replay"] = (commands["replay"] or "") + ADD_KEPT + self.start(commands, ["run_tasks"]) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") + self.check("good") + r = run(["decide", "--loop", self.loop, "--stop", "1"]) + self.assertEqual(r.code, 0, r.out + r.err) + row = next(ln for ln in self.page("decision-1.md").splitlines() if ln.startswith("| good |")) + self.assertIn("2 and 2 trials keep their recorded label", row) From 919f499c9ad04e88855d18dfa12a28cd2ecc978a Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:24:16 +0100 Subject: [PATCH 59/92] test(run-to-fix): quote the replay's post-processing of kept trials once --- .claude/skills/run-to-fix/tests/test_decide.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index e51a3c10..07923d92 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -535,8 +535,8 @@ def test_compare_refuses_a_cost_that_is_not_an_amount(self) -> None: self.assertFalse(os.path.exists(self.round_file("result.json"))) -ADD_KEPT = (" && python3 -c \\"import json,sys; p=sys.argv[1]; d=json.load(open(p)); " - "d['kept_recorded'] = ['f3__r', 'p2__r']; json.dump(d, open(p, 'w'))\\" {out_json}") +ADD_KEPT = (" && python3 -c \"import json,sys; p=sys.argv[1]; d=json.load(open(p)); " + "d['kept_recorded'] = ['f3__r', 'p2__r']; json.dump(d, open(p, 'w'))\" {out_json}") class KeptRecorded(Flow): From 0035b450d192a30c3a5cce9eb5ec34cedf5ac2f1 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:28:43 +0100 Subject: [PATCH 60/92] fix(run-to-fix): show on stop 1 the trials whose replay keeps their recorded label The replay cell of each candidate names how many trials keep their recorded label before and after the fix, and the page says that their labels come from the recorded run. --- .claude/skills/run-to-fix/assets/decision-1.md | 3 ++- .claude/skills/run-to-fix/scripts/deciding.py | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.claude/skills/run-to-fix/assets/decision-1.md b/.claude/skills/run-to-fix/assets/decision-1.md index e95288cd..27a6d89e 100644 --- a/.claude/skills/run-to-fix/assets/decision-1.md +++ b/.claude/skills/run-to-fix/assets/decision-1.md @@ -7,7 +7,8 @@ {{ROWS}} Replay counts are of the {{TRIALS}} recorded trials: done and passed / done and failed / not done and passed / not -done and failed. A false success is a failed trial that the agent reports as a success. +done and failed. A false success is a failed trial that the agent reports as a success. A trial that keeps its +recorded label is one the replay could not reproduce; its label comes from the recorded run. ## Candidates whose fix changes no label of their cause diff --git a/.claude/skills/run-to-fix/scripts/deciding.py b/.claude/skills/run-to-fix/scripts/deciding.py index 22da46d0..971cbb1c 100644 --- a/.claude/skills/run-to-fix/scripts/deciding.py +++ b/.claude/skills/run-to-fix/scripts/deciding.py @@ -63,8 +63,10 @@ def candidate_row(cand: dict, checks: Optional[dict]) -> List[str]: suite_text = ("did not run" if not suite["ran"] else ", ".join(suite["added"]) if suite["added"] else "none") if replay["ran"]: + kept = replay["kept_recorded"] replay_text = (f"{table_text(replay['before'])} → {table_text(replay['after'])} " - f"({replay['before_ref'][:12]} → {replay['ref'][:12]})") + f"({replay['before_ref'][:12]} → {replay['ref'][:12]}); {kept['before']} and {kept['after']} " + "trials keep their recorded label") change_text = changes_text(replay) false_text = f"{replay['false_success_added']:+d}" + ( f" (new: {', '.join(replay['false_success_new'])})" if replay["false_success_new"] else "") From 2a0f0ccde229f25d617246730472892f901c2988 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:29:20 +0100 Subject: [PATCH 61/92] test(run-to-fix): record a timed-out check, fix the stop rule at read, check ids, and name a report candidate's trials --- .claude/skills/run-to-fix/tests/test_check.py | 24 +++++++++++++++++++ .../skills/run-to-fix/tests/test_compare.py | 18 ++++++++++++++ .../skills/run-to-fix/tests/test_decide.py | 8 +++++++ 3 files changed, 50 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_check.py b/.claude/skills/run-to-fix/tests/test_check.py index 622a11ca..ffbdea2d 100644 --- a/.claude/skills/run-to-fix/tests/test_check.py +++ b/.claude/skills/run-to-fix/tests/test_check.py @@ -374,3 +374,27 @@ def test_sigterm(self) -> None: def test_ctrl_c(self) -> None: self.stop_during_a_command(signal.SIGINT) + + +class SmallItems(RepoCase): + def test_a_command_past_its_time_is_a_check_that_did_not_finish(self) -> None: + commands = self.commands() + commands["replay"] = "sleep 5 # {worktree} {ref} {job_dir} {out_json}" + self.start(commands, ["run_tasks"]) + self.candidate("good", self.repo.good, self.repo.test_ref, "outcome") + r = run(["check", "--loop", self.loop, "--candidate", "good", "--command-timeout-s", "1"]) + self.assertEqual(r.code, 0, r.out + r.err) + replay = load(self.round_file("candidates/good/checks.json"))["replay"] + self.assertFalse(replay["ran"]) + self.assertIn("did not finish in 1 s", replay["reason"]) + self.assertEqual(os.listdir(os.path.join(self.loop, "worktrees")), []) + + def test_a_candidate_id_or_a_cause_that_leaves_the_round_is_refused(self) -> None: + self.start(self.commands(), ["run_tasks"]) + r = run(["check", "--loop", self.loop, "--candidate", "../escape"]) + self.assertNotEqual(r.code, 0) + self.assertIn("candidate id", r.err) + r = run(["candidate", "--loop", self.loop, "--id", "c1", "--cause", "../escape", "--ref", self.repo.good, + "--effect", "outcome"]) + self.assertNotEqual(r.code, 0) + self.assertIn("cause", r.err) diff --git a/.claude/skills/run-to-fix/tests/test_compare.py b/.claude/skills/run-to-fix/tests/test_compare.py index b91c29e2..df91bceb 100644 --- a/.claude/skills/run-to-fix/tests/test_compare.py +++ b/.claude/skills/run-to-fix/tests/test_compare.py @@ -334,3 +334,21 @@ def test_a_new_run_of_unknown_cost_leaves_the_budget_unchecked(self) -> None: def test_the_person_s_figure_for_the_new_run(self) -> None: rule = self.compare_partial(["--new-cost-usd", "0.5"]) self.assertAlmostEqual(rule["spent_usd"], 0.5) + + +class StopRuleOfTheRound(LoopCase): + def test_compare_refuses_a_stop_rule_other_than_the_one_read_recorded(self) -> None: + before, new = paired(5, 1, 1, 1) + before_dir = write_job(self.jobs, "before", "alpha", "harbor", before) + new_dir = write_job(self.jobs, "new", "alpha", "harbor", new) + self.set_adapter(adapter("alpha", NO_LIMITS, {})) + self.read_ok(before_dir, None) + path = self.round_file("input.json") + data = load(path) + self.assertIn("No new round when", data["stop_rule"]) + data["stop_rule"] = "Another rule." + write_json(path, data) + r = run(["compare", "--loop", self.loop, "--new", new_dir, "--against", "run"]) + self.assertNotEqual(r.code, 0) + self.assertIn("stop rule", r.err) + self.assertFalse(os.path.exists(self.round_file("result.json"))) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index 07923d92..539940fc 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -102,6 +102,13 @@ def test_an_answer_before_the_page_is_refused(self) -> None: self.assertNotEqual(r.code, 0) self.assertIn("decision-1.md", r.err) + def test_keeping_a_candidate_id_that_leaves_the_round_is_refused(self) -> None: + self.checked() + run(["decide", "--loop", self.loop, "--stop", "1"]) + r = run(["answer", "--loop", self.loop, "--stop", "1", "--keep", "good,../escape", "--text", "keep"]) + self.assertNotEqual(r.code, 0) + self.assertIn("candidate id", r.err) + def test_keeping_an_unchecked_candidate_is_refused(self) -> None: self.checked() run(["decide", "--loop", self.loop, "--stop", "1"]) @@ -369,6 +376,7 @@ def test_the_page_of_stop_one_names_the_effect(self) -> None: row = next(ln for ln in self.page("decision-1.md").splitlines() if ln.startswith("| rep |")) self.assertIn("| report |", row) self.assertIn("1 of 1: not done → done 1", row) + self.assertIn("1 trials of the rule, no task expected to pass (p3_reports)", row) def test_select_takes_no_targets_and_guards_the_changed_trials(self) -> None: self.kept_report() From 9a4da9bfd88663dc6d1d603cc9964dd3d8af58e0 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:31:23 +0100 Subject: [PATCH 62/92] fix(run-to-fix): record a check whose command ran past its time as one that did not finish A command stopped after --command-timeout-s no longer stops check: the part it belongs to is recorded as not run, with the time and the log, as adapter.md says. --- .claude/skills/run-to-fix/scripts/checking.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.claude/skills/run-to-fix/scripts/checking.py b/.claude/skills/run-to-fix/scripts/checking.py index 8f0a30f7..895e99e0 100644 --- a/.claude/skills/run-to-fix/scripts/checking.py +++ b/.claude/skills/run-to-fix/scripts/checking.py @@ -109,7 +109,8 @@ def on_ref(adapter: Adapter, loop_dir: str, round_dir: str, name: str, sha: str, with worktree(adapter.checkout, loop_dir, sha) as tree: ran = run_command(line, dict(values, worktree=tree), tree, log, timeout_s) if ran.code is None: - raise LoopError(f"commands.{name} on {sha[:12]} did not finish in time; the log is {ran.log}") + return Outcome(None, f"commands.{name} on {sha[:12]} did not finish in {timeout_s:g} s and was stopped; " + f"the log is {ran.log}") return Outcome(ran, None) @@ -217,7 +218,7 @@ def check_replay(adapter: Adapter, args: argparse.Namespace, round_dir: str, bef o = on_ref(adapter, args.loop, round_dir, "replay", sha, values, out[:-len(".json")] + ".log", args.command_timeout_s) if o.ran is None: - if when == "before": + if when == "before" and "replay" in adapter.paid: on_ref(adapter, args.loop, round_dir, "replay", ref, dict(values, ref=ref, out_json=replay_file(round_dir, ref, before_ref)), replay_file(round_dir, ref, before_ref)[:-len(".json")] + ".log", args.command_timeout_s) From 9bf9d24e84b88fd984d0dbe735cd9259509c7841 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:32:23 +0100 Subject: [PATCH 63/92] fix(run-to-fix): check a candidate id and a cause name wherever they name a folder or a file check and answer take a candidate id only in the form candidate gives it, and candidate takes a cause only as the name of a file in causes/, so neither can lead outside the round. --- .claude/skills/run-to-fix/scripts/checking.py | 14 +++++++++++--- .claude/skills/run-to-fix/scripts/deciding.py | 4 ++-- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.claude/skills/run-to-fix/scripts/checking.py b/.claude/skills/run-to-fix/scripts/checking.py index 895e99e0..f6cc6d34 100644 --- a/.claude/skills/run-to-fix/scripts/checking.py +++ b/.claude/skills/run-to-fix/scripts/checking.py @@ -22,9 +22,18 @@ EFFECTS = ("outcome", "report") +def candidate_id(cid: str) -> str: + """A candidate id names a folder of the round: lower-case letters, digits, - and _.""" + if not CANDIDATE_ID.match(cid): + raise LoopError(f"a candidate id is lower-case letters, digits, - and _, got {cid!r}") + return cid + + def cause_trials(round_dir: str, cause: str, effect: str) -> Tuple[str, List[str], List[str]]: """The rule a cause names, the trials it counts and their tasks: failing trials for an effect on the outcome, every trial the rule matches for an effect on the report.""" + if not CANDIDATE_ID.match(cause): + raise LoopError(f"a cause is the name of a file in causes/: lower-case letters, digits, - and _, got {cause!r}") path = os.path.join(round_dir, "causes", f"{cause}.md") if not os.path.isfile(path): raise LoopError(f"{path} is missing: write the cause before its candidates") @@ -66,8 +75,7 @@ def cmd_candidate(args: argparse.Namespace) -> None: adapter = adapter_of(args.loop) round_dir = open_round(args.loop) read_input(round_dir) - if not CANDIDATE_ID.match(args.id): - raise LoopError(f"a candidate id is lower-case letters, digits, - and _, got {args.id!r}") + candidate_id(args.id) if (args.test is None) != (args.test_ref is None): raise LoopError("--test and --test-ref go together: the test file and the commit that adds it before the fix") if args.test is not None and (os.path.isabs(args.test) or ".." in re.split(r"[\\/]", args.test)): @@ -248,7 +256,7 @@ def cmd_check(args: argparse.Namespace) -> None: adapter = adapter_of(args.loop) round_dir = open_round(args.loop) record = read_input(round_dir) - cdir = os.path.join(round_dir, "candidates", args.candidate) + cdir = os.path.join(round_dir, "candidates", candidate_id(args.candidate)) cand = load_candidate(os.path.join(cdir, "candidate.json")) # a check that fails part way must not leave the success of an earlier check to be read stale = os.path.join(cdir, "checks.json") diff --git a/.claude/skills/run-to-fix/scripts/deciding.py b/.claude/skills/run-to-fix/scripts/deciding.py index 971cbb1c..eafec0b1 100644 --- a/.claude/skills/run-to-fix/scripts/deciding.py +++ b/.claude/skills/run-to-fix/scripts/deciding.py @@ -7,7 +7,7 @@ import re from typing import Dict, List, Optional -from checking import load_candidate +from checking import candidate_id, load_candidate from checkout import resolve from classifying import read_rows from comparing import REPORT_NOT_READ, TARGET_TRIALS, passes, targets_of, trial_spent @@ -252,7 +252,7 @@ def cmd_answer(args: argparse.Namespace) -> None: if args.stop == 1: if args.keep is None: raise LoopError("answer --stop 1 needs --keep: candidate ids separated by commas, or none") - keep = [] if args.keep == "none" else [k.strip() for k in args.keep.split(",") if k.strip()] + keep = [] if args.keep == "none" else [candidate_id(k.strip()) for k in args.keep.split(",") if k.strip()] cands = {k: load_candidate(os.path.join(round_dir, "candidates", k, "candidate.json")) for k in keep if os.path.isfile(os.path.join(round_dir, "candidates", k, "candidate.json"))} missing = [k for k in keep if not os.path.isfile(os.path.join(round_dir, "candidates", k, "checks.json"))] From e7c0512092e05eaaa3b9bc0792f6329618edc920 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:32:54 +0100 Subject: [PATCH 64/92] fix(run-to-fix): name a report candidate's trials on stop 1 without a count of tasks to pass --- .claude/skills/run-to-fix/scripts/deciding.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/scripts/deciding.py b/.claude/skills/run-to-fix/scripts/deciding.py index eafec0b1..e7a6b31a 100644 --- a/.claude/skills/run-to-fix/scripts/deciding.py +++ b/.claude/skills/run-to-fix/scripts/deciding.py @@ -50,7 +50,9 @@ def changes_text(replay: dict) -> str: def candidate_row(cand: dict, checks: Optional[dict]) -> List[str]: pred = cand["prediction"] - explains = f"{len(pred['trials'])} trials, {len(pred['tasks'])} tasks expected to pass ({cand['rule']})" + explains = (f"{len(pred['trials'])} trials, {len(pred['tasks'])} tasks expected to pass ({cand['rule']})" + if cand["effect"] == "outcome" else + f"{len(pred['trials'])} trials of the rule, no task expected to pass ({cand['rule']})") if checks is None: return [cand["id"], cand["cause"], cand["effect"], explains] + ["not checked"] * 5 + [""] test, suite, replay, diff = checks["test"], checks["suite"], checks["replay"], checks["diff"] From 22eb83c16e63787d534bf71aff2b252322c2b4ac Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:34:51 +0100 Subject: [PATCH 65/92] fix(run-to-fix): record the stop rule when a round starts, and apply only that rule read writes the text of the stop rule into input.json; compare refuses a round whose recorded rule is not the rule of the skill it runs, so a rule changed by an update of the skill never decides a round that began under another. --- .claude/skills/run-to-fix/scripts/comparing.py | 10 +++++----- .claude/skills/run-to-fix/scripts/reading.py | 7 ++++--- .claude/skills/run-to-fix/scripts/rounds.py | 4 ++++ 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/.claude/skills/run-to-fix/scripts/comparing.py b/.claude/skills/run-to-fix/scripts/comparing.py index f68ea1f5..69079477 100644 --- a/.claude/skills/run-to-fix/scripts/comparing.py +++ b/.claude/skills/run-to-fix/scripts/comparing.py @@ -7,8 +7,8 @@ from classifying import classes_of, cross_check, failure_groups, read_rows from reading import adapter_of, analyze, health -from rounds import (LoopError, call_runs, ledger_round, load_ledger, now, open_round, read_input, read_json, - save_ledger, write_json) +from rounds import (STOP_RULE, LoopError, call_runs, ledger_round, load_ledger, now, open_round, read_input, + read_json, save_ledger, write_json) from pricing import describe, price_of, task_costs, usd from stats import binomial_two_sided, fisher_two_sided, verdict from trials import TrialRow @@ -16,9 +16,6 @@ TARGET_TRIALS = 3 REPORT_NOT_READ = "this version does not read the agent's report in the new run" GROUP_MIN_TRIALS = 3 -STOP_RULE = ("No new round when the verdict is noise and no group of failing trials of the new run has 3 trials " - "or more, or when the budget left is under the price of the cheapest option: the lowest-priced trial run " - "of the tasks of one such group, 3 trials each, each task at the mean cost of its trials in the new run.") def page_labels(before_job: str, new_job: str) -> Tuple[str, str]: @@ -124,6 +121,9 @@ def cmd_compare(args: argparse.Namespace) -> None: adapter = adapter_of(args.loop) round_dir = open_round(args.loop) record = read_input(round_dir) + if record.get("stop_rule") != STOP_RULE: + raise LoopError("the stop rule of this round, recorded in input.json when read started it, is not the stop " + "rule of this version of the skill; start a new round with loop.py read") threshold = float(record["pass_threshold"]) against = record["run"] if args.against == "run" else record["baseline"] if against is None: diff --git a/.claude/skills/run-to-fix/scripts/reading.py b/.claude/skills/run-to-fix/scripts/reading.py index abdcbb19..cb73c7bb 100644 --- a/.claude/skills/run-to-fix/scripts/reading.py +++ b/.claude/skills/run-to-fix/scripts/reading.py @@ -9,8 +9,8 @@ from typing import List, Optional, Tuple from adapter import Adapter, load_adapter -from rounds import (LoopError, call_runs, last_result, ledger_round, load_ledger, now, round_for_read, runs_py, - save_ledger, sha256_of, write_json) +from rounds import (STOP_RULE, LoopError, call_runs, last_result, ledger_round, load_ledger, now, round_for_read, + runs_py, save_ledger, sha256_of, write_json) #: verdicts of `runs.py health` that make a run not analysable, matched by their opening words REFUSING = ("no trials ran", "every trial raised an exception", "systemic failure", "incomplete:", @@ -151,7 +151,8 @@ def cmd_read(args: argparse.Namespace) -> None: "run": {"spec": args.run, "job_dir": None}, "baseline": {"spec": args.baseline, "job_dir": None} if args.baseline else None, "adapter_sha256": adapter.sha256, "budget_usd": args.budget_usd, "pass_threshold": args.pass_threshold, - "run_analysis": {"runs_py": runs, "sha256": sha256_of(runs)}, "refused": bool(refused), + "run_analysis": {"runs_py": runs, "sha256": sha256_of(runs)}, "stop_rule": STOP_RULE, + "refused": bool(refused), "reason": "; ".join(refused) or None, } if refused: diff --git a/.claude/skills/run-to-fix/scripts/rounds.py b/.claude/skills/run-to-fix/scripts/rounds.py index afaf1f22..9e61b6f2 100644 --- a/.claude/skills/run-to-fix/scripts/rounds.py +++ b/.claude/skills/run-to-fix/scripts/rounds.py @@ -16,6 +16,10 @@ SCRIPTS = os.path.dirname(os.path.abspath(__file__)) RUNS_PY = os.path.join(os.path.dirname(os.path.dirname(SCRIPTS)), "run-analysis", "scripts", "runs.py") ROUND = re.compile(r"^round-(\d{2,})$") +#: the stop rule; read records it in input.json, and compare applies it only when it is still the same text +STOP_RULE = ("No new round when the verdict is noise and no group of failing trials of the new run has 3 trials " + "or more, or when the budget left is under the price of the cheapest option: the lowest-priced trial run " + "of the tasks of one such group, 3 trials each, each task at the mean cost of its trials in the new run.") TAIL_LINES = 12 From 820ab69faa7838503520789b69868e5634b6b23d Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:35:29 +0100 Subject: [PATCH 66/92] test(run-to-fix): name the expected canary and the expected classes of both run paths --- .claude/skills/run-to-fix/tests/test_classify.py | 1 + .claude/skills/run-to-fix/tests/test_decide.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_classify.py b/.claude/skills/run-to-fix/tests/test_classify.py index e694a9aa..9ee373c0 100644 --- a/.claude/skills/run-to-fix/tests/test_classify.py +++ b/.claude/skills/run-to-fix/tests/test_classify.py @@ -211,6 +211,7 @@ def test_the_same_columns_for_harbor_and_standard(self) -> None: classes.append({r["trial"]: r["class"] for r in rows}) self.assertEqual(headers[0], headers[1]) self.assertEqual(classes[0], classes[1]) + self.assertEqual(classes[0], {"p__x": "pass", "f__x": "verifier_failed", "t__x": "time_limit"}) for column in ("trial", "task", "class", "signature", "reward", "pair", "cost_usd", "model_calls", "agent_s", "max_completion_tokens", "exception_type"): self.assertIn(column, headers[0]) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index 539940fc..9562e888 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -178,6 +178,8 @@ def test_canaries_come_from_a_fixed_seed(self) -> None: self.kept() first = self.select([]).out self.assertEqual(self.select([]).out, first) + self.assertIn("canaries (1 passing tasks, seed 20261117): bench/p2 +", first) def test_without_touched_there_are_no_guards(self) -> None: commands = self.commands() From 76e5bbfb3dd5820bd5a9254c520625752a13e79c Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:36:24 +0100 Subject: [PATCH 67/92] test(run-to-fix): end the expected canary line with an escaped newline --- .claude/skills/run-to-fix/tests/test_decide.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index 9562e888..e39f5888 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -178,8 +178,7 @@ def test_canaries_come_from_a_fixed_seed(self) -> None: self.kept() first = self.select([]).out self.assertEqual(self.select([]).out, first) - self.assertIn("canaries (1 passing tasks, seed 20261117): bench/p2 -", first) + self.assertIn("canaries (1 passing tasks, seed 20261117): bench/p2\n", first) def test_without_touched_there_are_no_guards(self) -> None: commands = self.commands() From ba6131879cea77db30370e1672f2fec4b2905e07 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:37:10 +0100 Subject: [PATCH 68/92] test(run-to-fix): use a made-up verifier line with no personal name --- .claude/skills/run-to-fix/tests/test_classify.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.claude/skills/run-to-fix/tests/test_classify.py b/.claude/skills/run-to-fix/tests/test_classify.py index 9ee373c0..9f8b1673 100644 --- a/.claude/skills/run-to-fix/tests/test_classify.py +++ b/.claude/skills/run-to-fix/tests/test_classify.py @@ -401,7 +401,7 @@ def test_double_quotes_are_masked(self) -> None: def test_verifier_and_timeout_details_keep_their_quotes(self) -> None: failed = trial("f__a", "bench/f", 0.0) - failed.verifier_text = "FAIL: Kevin Zhao's email does not have the 'Clients' label" + failed.verifier_text = "FAIL: the sample's invoice does not have the 'Done' label" late = trial("t__a", "bench/t", None) late.exception = ("AgentTimeoutError", "the agent did not answer 'ping' in time") run_dir = write_job(self.jobs, "run", "alpha", "harbor", [failed, late, trial("p__a", "bench/p", 1.0)]) @@ -410,7 +410,7 @@ def test_verifier_and_timeout_details_keep_their_quotes(self) -> None: self.classify_ok() rows = self.by_trial() self.assertEqual(rows["f__a"]["signature"], - "verifier_failed: FAIL: Kevin Zhao's email does not have the 'Clients' label") + "verifier_failed: FAIL: the sample's invoice does not have the 'Done' label") self.assertEqual(rows["t__a"]["signature"], "time_limit: AgentTimeoutError: the agent did not answer 'ping' in time") From 8d05cbb4fdb502f8b0a4d86299b66497d3fe5683 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:37:48 +0100 Subject: [PATCH 69/92] docs(run-to-fix): describe the stated commit and costs of a trial run, a stopped loop and the recorded stop rule --- .claude/skills/run-to-fix/SKILL.md | 19 ++++++++++++++++--- .../skills/run-to-fix/references/adapter.md | 5 +++++ .../skills/run-to-fix/references/classes.md | 3 ++- .claude/skills/run-to-fix/references/files.md | 9 ++++++--- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/.claude/skills/run-to-fix/SKILL.md b/.claude/skills/run-to-fix/SKILL.md index 9a768938..e55072ec 100644 --- a/.claude/skills/run-to-fix/SKILL.md +++ b/.claude/skills/run-to-fix/SKILL.md @@ -121,9 +121,15 @@ This writes `trial-tasks.txt`, runs `commands.build` and writes `build.json`, an `round-NN/commands/run_tasks.sh`. The person starts it. When its job folder exists: ```bash -python3 L decide --loop DIR --stop 2 --trial-job JOB +python3 L decide --loop DIR --stop 2 --trial-job JOB --code-ref COMMIT [--trial-cost-usd X] ``` +`--code-ref` is the commit the person states the trial run ran; a job does not record its code, so the page shows it +as the person's statement, and `decide` refuses any commit other than the one `select` chose. The job must hold only +the selected tasks, every target, and the selection's count of trials for each task. When some of its trials report no +cost, the money it spent is unknown: `decide` refuses it until the person gives the run's whole cost with +`--trial-cost-usd`, at least what its trials record. + **Stop 2.** Show the person `decision-2.md`. For a kept `report` candidate it says that this version does not read the agent's report in the new run. The page shows the build, the targets' trials before and in every trial run of this round on that build with Fisher's exact test, the guards and canaries, the money spent and left, and three options @@ -140,13 +146,16 @@ the earlier ones. ## Stage 5. Compare and record ```bash -python3 L compare --loop DIR --new JOB --against run +python3 L compare --loop DIR --new JOB --against run [--new-cost-usd X] ``` `compare` reads the new run against the round's run (`--against baseline` compares with the round's baseline) and writes `result.json`: the pass rates with their standard error; the tasks both runs share, won and lost; the verdict "improved" or "regressed" when an exact two-sided binomial test on won and lost gives p under 0.05, "noise" otherwise; -the targets' trials with Fisher's exact test; each prediction of stage 3 beside its result; and the stop rule. It +the targets' trials with Fisher's exact test; each prediction of stage 3 beside its result; and the stop rule. When +some trials of the new run report no cost, the budget part of the stop rule is not checked, unless the person gives the +run's cost with `--new-cost-usd`. `read` records the text of the stop rule, and `compare` refuses a round whose recorded +rule is not the rule of the skill it runs. It closes the round and writes `report.html` with `render` of `run-analysis`; fill its prose slots as that skill describes (`runs.py render --slots`, then `runs.py check`). @@ -191,3 +200,7 @@ These follow `run-analysis`, with what the loop adds. - The classes read only shared files. What one agent writes reaches the loop through a found class or `commands.classes`. - A trial run gives every task the same number of trials, because `run_tasks` takes one `{repeat}`. +- A loop stopped with SIGTERM or Ctrl+C stops the adapter command it runs and removes its worktree. A loop killed with + SIGKILL cannot: its worktree stays under `/worktrees/`, and `git worktree remove --force ` in the + agent's checkout removes it. +- Every amount in US dollars on the command line is a finite number of 0 or more. diff --git a/.claude/skills/run-to-fix/references/adapter.md b/.claude/skills/run-to-fix/references/adapter.md index b13adf95..4f20e3e3 100644 --- a/.claude/skills/run-to-fix/references/adapter.md +++ b/.claude/skills/run-to-fix/references/adapter.md @@ -86,6 +86,11 @@ means. evidence is missing because of it. 5. The loop writes the output of every command to a file under `round-NN/`. A command that has not exited within the time `loop.py` was given is stopped, and the check is recorded as not finished. +6. A loop stopped with SIGTERM or Ctrl+C stops the whole process group of the command it runs, then removes its + worktree. A loop killed with SIGKILL leaves the worktree; `git worktree remove --force ` in the checkout + removes it. +7. The caches of a round, the suite's failures on `base_ref` and each replay, are used only under the adapter that made + them; a changed `adapter.json` runs its commands again. ## Exit codes and output diff --git a/.claude/skills/run-to-fix/references/classes.md b/.claude/skills/run-to-fix/references/classes.md index a6b271dd..21857c9a 100644 --- a/.claude/skills/run-to-fix/references/classes.md +++ b/.claude/skills/run-to-fix/references/classes.md @@ -77,7 +77,8 @@ included. The rules live in `/found-classes.json`; the skill ships } ``` -1. `id` is lower-case letters, digits and `_`. `meaning` is one sentence. `file` is a path inside the trial folder. +1. `id` is lower-case letters, digits and `_`. `meaning` is one sentence. `file` is a path inside the trial folder; + a file whose resolved path, through any link, lies outside the trial folder stops `classify`. 2. A rule has either `regex` (searched in the file's text) or `json_path` with `value`. A `json_path` segment is a key, a list index, or `*` for every element; the rule matches when any value reached equals `value`. 3. A found class may read a file that only one agent writes. diff --git a/.claude/skills/run-to-fix/references/files.md b/.claude/skills/run-to-fix/references/files.md index dbb2d6e2..28c42641 100644 --- a/.claude/skills/run-to-fix/references/files.md +++ b/.claude/skills/run-to-fix/references/files.md @@ -26,7 +26,7 @@ round that has no `result.json`. | file | stage | written by | holds | |---|---|---|---| -| `input.json` | 1 | `read` | the run and baseline job folders, the adapter's SHA-256, the budget, the pass threshold, the `runs.py` used; `refused` with its reason when the run is not analysable | +| `input.json` | 1 | `read` | the run and baseline job folders, the adapter's SHA-256, the budget, the pass threshold, the `runs.py` used, the text of the stop rule; `refused` with its reason when the run is not analysable | | `health.txt` | 1 | `read` | `runs.py health` of the run and the baseline | | `metrics.json` | 1 | `read` | `runs.py analyze` of the baseline and the run, labels `baseline` and `run` | | `trials.csv` | 2 | `classify` | one row per trial: `trial`, `task`, `class`, `signature`, `reward`, `pair`, `cost_usd`, `model_calls`, `agent_s`, `max_completion_tokens`, `exception_type`, then `found:`, `agent_class:`, `agent_flag:` | @@ -81,7 +81,8 @@ Not checked: what the files cannot tell. "tasks": [], "expect": "label_change"}], "selection": {"ref": "...", "size": "guards", "targets": [], "guards": [], "canaries": [], "repeat": 3, "seed": 20261117, "price_usd": 0.0, "budget_usd": 0.0, "guards_reason": null}, - "trial_runs": [{"job_dir": "...", "ref": "...", "trials": 0, "cost_usd": 0.0, "cost_complete": true}], + "trial_runs": [{"job_dir": "...", "ref": "...", "trials": 0, "cost_usd": 0.0, "cost_recorded_usd": 0.0, + "cost_from": "the trials", "cost_complete": true, "code_ref": "...", "code_from": "the person"}], "next": "rerun", "result": {"verdict": "noise", "p": 0.68, "won": 10, "lost": 13, "stop": true, "reasons": ["..."]} }] @@ -90,7 +91,9 @@ Not checked: what the files cannot tell. Every answer of the person is stored with its date and in the person's words: the candidates kept at stop 1, the size and budget of the trial run, the choice at stop 2, and the words that start a round after a stop rule said -stop. The money a round has spent is the sum of its `trial_runs` plus the new run that `compare` reads. +stop. The money a round has spent is the sum of its `trial_runs` plus the new run that `compare` reads. A trial run +whose trials do not all report a cost counts with the person's figure (`cost_from`: `the person`) and keeps what its +trials record in `cost_recorded_usd`; `code_ref` is the commit the person states it ran. ## Prices From a59f8f5a7c8ac175e4b908a36c60a5f8c86c9e83 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 18:38:06 +0100 Subject: [PATCH 70/92] docs(run-to-fix): state the stated commit and the unknown cost of a trial run in plain sentences --- .claude/skills/run-to-fix/SKILL.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.claude/skills/run-to-fix/SKILL.md b/.claude/skills/run-to-fix/SKILL.md index e55072ec..043855d6 100644 --- a/.claude/skills/run-to-fix/SKILL.md +++ b/.claude/skills/run-to-fix/SKILL.md @@ -124,11 +124,11 @@ This writes `trial-tasks.txt`, runs `commands.build` and writes `build.json`, an python3 L decide --loop DIR --stop 2 --trial-job JOB --code-ref COMMIT [--trial-cost-usd X] ``` -`--code-ref` is the commit the person states the trial run ran; a job does not record its code, so the page shows it -as the person's statement, and `decide` refuses any commit other than the one `select` chose. The job must hold only -the selected tasks, every target, and the selection's count of trials for each task. When some of its trials report no -cost, the money it spent is unknown: `decide` refuses it until the person gives the run's whole cost with -`--trial-cost-usd`, at least what its trials record. +`--code-ref` is the commit the person states the trial run ran. A job does not record its code; the page shows the +commit as the person's statement, and `decide` refuses any commit other than the one `select` chose. The job must hold +only the selected tasks, every target, and the selection's count of trials for each task. When some of its trials +report no cost, `decide` refuses the run until the person gives its whole cost with `--trial-cost-usd`, at least what +its trials record. **Stop 2.** Show the person `decision-2.md`. For a kept `report` candidate it says that this version does not read the agent's report in the new run. The page shows the build, the targets' trials before and in every trial run of this From 8ed6bf4fd59ccaaeee7f67a9e87abbbb64964fa6 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:00:33 +0100 Subject: [PATCH 71/92] test(run-to-fix): count a trial's model calls by the standard's own count when its result holds one --- .../skills/run-to-fix/tests/test_classify.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_classify.py b/.claude/skills/run-to-fix/tests/test_classify.py index 9f8b1673..792f5905 100644 --- a/.claude/skills/run-to-fix/tests/test_classify.py +++ b/.claude/skills/run-to-fix/tests/test_classify.py @@ -463,3 +463,21 @@ def test_a_rule_that_reaches_outside_the_trial_through_a_link_is_refused(self) - self.assertNotEqual(r.code, 0) self.assertIn("outside the trial", r.err) self.assertFalse(os.path.exists(self.round_file("trials.csv"))) + + +#: the trajectory of the standard path: one step per agent message, with no usage per step +STEPS_WITHOUT_USAGE = json.dumps({"schema_version": "ATIF-v1.6", "steps": [ + {"step_id": i, "source": "agent", "message": "step"} for i in range(31)]}) + + +class StandardModelCalls(LoopCase): + def test_the_count_of_the_standard_wins_over_the_steps_of_the_trajectory(self) -> None: + t = trial("s1__s", "made-up-bench/s1", 0.0) + t.calls = 20 + t.files = {"agent/trajectory.json": STEPS_WITHOUT_USAGE} + run_dir = write_job(self.jobs, "run", "alpha", "standard", [t, trial("p__s", "made-up-bench/p", 1.0)]) + self.set_adapter(adapter("alpha", {"step_limit": 30}, {})) + self.read_ok(run_dir, None) + self.classify_ok() + row = self.by_trial()["s1__s"] + self.assertEqual((row["model_calls"], row["class"]), ("20", "verifier_failed")) From 1cd0551ed08f56d0f6493d860d754041a08dcb5d Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:00:45 +0100 Subject: [PATCH 72/92] test(run-to-fix): import json at the top of the classify tests --- .claude/skills/run-to-fix/tests/test_classify.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.claude/skills/run-to-fix/tests/test_classify.py b/.claude/skills/run-to-fix/tests/test_classify.py index 792f5905..8dd06abd 100644 --- a/.claude/skills/run-to-fix/tests/test_classify.py +++ b/.claude/skills/run-to-fix/tests/test_classify.py @@ -1,6 +1,7 @@ """`loop.py classify`: the seven classes, signatures, groups, warnings and the agent's own classes.""" from __future__ import annotations +import json import os import sys from typing import Dict, List From 20c7bfab9ca2e2caca774b462cf00845e4b2ac36 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:01:44 +0100 Subject: [PATCH 73/92] fix(run-to-fix): count a trial's model calls by the standard's own count when its result holds one A job of the standard writes agent_result.n_model_calls, and its ATIF trajectory has one step per agent message with no usage per step, so the steps overcount or undercount the calls. step_limit now reads that count when result.json holds it, and the rule of run-analysis otherwise. --- .claude/skills/run-to-fix/scripts/trials.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/scripts/trials.py b/.claude/skills/run-to-fix/scripts/trials.py index 4049b02d..b259779f 100644 --- a/.claude/skills/run-to-fix/scripts/trials.py +++ b/.claude/skills/run-to-fix/scripts/trials.py @@ -115,7 +115,12 @@ def read_trial(trial_dir: str, pass_threshold: float) -> TrialRow: exc = result.get("exception_info") or {} calls, max_out = _usage(trial_dir) info = result.get("agent_info") or {} - cost = (result.get("agent_result") or {}).get("cost_usd") + agent_result = result.get("agent_result") or {} + cost = agent_result.get("cost_usd") + counted = agent_result.get("n_model_calls") + # a job of the standard counts its model calls; its trajectory has one step per message, not per call + if isinstance(counted, int) and not isinstance(counted, bool): + calls = counted return TrialRow( trial=str(result.get("trial_name") or os.path.basename(trial_dir)), task=str(result["task_name"]), dir=trial_dir, reward=reward, passed=reward is not None and reward >= pass_threshold, From 9aa619505bb7ac9fd64262dea1aca77f8197c50d Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:01:58 +0100 Subject: [PATCH 74/92] test(run-to-fix): name output_limit as switched off when no trial records the tokens of a reply --- .claude/skills/run-to-fix/tests/test_classify.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_classify.py b/.claude/skills/run-to-fix/tests/test_classify.py index 8dd06abd..cf2aff6c 100644 --- a/.claude/skills/run-to-fix/tests/test_classify.py +++ b/.claude/skills/run-to-fix/tests/test_classify.py @@ -482,3 +482,17 @@ def test_the_count_of_the_standard_wins_over_the_steps_of_the_trajectory(self) - self.classify_ok() row = self.by_trial()["s1__s"] self.assertEqual((row["model_calls"], row["class"]), ("20", "verifier_failed")) + + +class OutputLimitUnreadable(LoopCase): + def test_a_set_limit_that_no_trial_records_is_named_as_switched_off(self) -> None: + t = trial("s1__s", "made-up-bench/s1", 0.0) + t.files = {"agent/trajectory.json": STEPS_WITHOUT_USAGE} + p = trial("p__s", "made-up-bench/p", 1.0) + p.files = {"agent/trajectory.json": STEPS_WITHOUT_USAGE} + run_dir = write_job(self.jobs, "run", "alpha", "standard", [t, p]) + self.set_adapter(adapter("alpha", {"output_token_limit": 4096}, {})) + self.read_ok(run_dir, None) + self.classify_ok() + self.assertIn("output_limit: no trial records the tokens of one reply", + load(self.round_file("groups.json"))["rules_off"]) From 0a3ab375571196a866b52aed61a1beba88401d95 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:02:57 +0100 Subject: [PATCH 75/92] fix(run-to-fix): name output_limit as switched off when no trial records the tokens of a reply A job of the standard records no tokens per reply in the files every path writes. groups.json then lists output_limit under rules_off with that reason, instead of keeping a rule that can never hold. --- .claude/skills/run-to-fix/scripts/classifying.py | 2 +- .claude/skills/run-to-fix/scripts/trials.py | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.claude/skills/run-to-fix/scripts/classifying.py b/.claude/skills/run-to-fix/scripts/classifying.py index d4d5eff0..8d40a37f 100644 --- a/.claude/skills/run-to-fix/scripts/classifying.py +++ b/.claude/skills/run-to-fix/scripts/classifying.py @@ -166,7 +166,7 @@ def cmd_classify(args: argparse.Namespace) -> None: "schema": "run-to-fix.groups/v1", "counts": "trials of the run; each trial has one class, and a group is the failing trials of one signature", "trials": len(rows), "failures": len(failing), - "classes": {c: count[c] for c in CLASSES}, "rules_off": rules_off(adapter.limits), + "classes": {c: count[c] for c in CLASSES}, "rules_off": rules_off(adapter.limits, rows), "groups": groups, "found_classes": {r.id: {"meaning": r.meaning, "file": r.file, "trials": found_counts[r.id]["1"], "failing_trials": found_counts[r.id]["failing"], diff --git a/.claude/skills/run-to-fix/scripts/trials.py b/.claude/skills/run-to-fix/scripts/trials.py index b259779f..6f0c583c 100644 --- a/.claude/skills/run-to-fix/scripts/trials.py +++ b/.claude/skills/run-to-fix/scripts/trials.py @@ -150,7 +150,8 @@ def classify(row: TrialRow, limits: Limits) -> str: return "verifier_failed" -def rules_off(limits: Limits) -> List[str]: +def rules_off(limits: Limits, rows: List[TrialRow]) -> List[str]: + """The rules a null limit switches off, and output_limit when no trial records the tokens of one reply.""" off = [] if limits.time_limit_s is None: off.append("time_limit by the agent phase") @@ -158,6 +159,8 @@ def rules_off(limits: Limits) -> List[str]: off.append("step_limit") if limits.output_token_limit is None: off.append("output_limit") + elif all(r.max_completion_tokens is None for r in rows): + off.append("output_limit: no trial records the tokens of one reply") return off From 1ed8e08caa165a537d1ed46a6fb613de2a6fa190 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:03:16 +0100 Subject: [PATCH 76/92] test(run-to-fix): write unit ids into the task lists of a trial run on a job of the standard --- .../skills/run-to-fix/tests/test_decide.py | 22 ++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index e39f5888..21c7435c 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -4,7 +4,7 @@ import os from typing import List -from agentrepo import RepoCase +from agentrepo import RepoCase, recorded_trials from jobs import Result, Trial, load, run, trial, write, write_job, write_json KEEP_GOOD = "Keep good. bad breaks test_other." @@ -559,3 +559,23 @@ def test_the_page_shows_the_trials_that_keep_their_recorded_label(self) -> None: self.assertEqual(r.code, 0, r.out + r.err) row = next(ln for ln in self.page("decision-1.md").splitlines() if ln.startswith("| good |")) self.assertIn("2 and 2 trials keep their recorded label", row) + + +class StandardUnits(Flow): + """On a job of the standard, the task lists of a trial run hold the unit ids that --units of the standard takes.""" + + def setUp(self) -> None: + super().setUp() + self.run_dir = write_job(self.jobs, "recorded-standard", "alpha", "standard", recorded_trials()) + + def lines(self, name: str) -> List[str]: + with open(self.round_file(name), encoding="utf-8") as fh: + return fh.read().split() + + def test_the_task_lists_hold_unit_ids(self) -> None: + job = self.chosen() + self.assertEqual(self.lines("trial-tasks.txt"), ["f1", "f2", "p1"]) + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--code-ref", self.code_ref()]) + self.assertEqual(r.code, 0, r.out + r.err) + self.assertEqual(self.lines("rerun-tasks.txt"), ["f1", "f2", "p1"]) + self.assertEqual(self.lines("full-tasks.txt"), ["f1", "f2", "f3", "p1", "p2", "p3"]) From 5afc2be7ff2be5a9fd5ba9c4e876bb12b535b512 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:08:06 +0100 Subject: [PATCH 77/92] fix(run-to-fix): write unit ids into the task lists of a trial run on a job of the standard A trial of the standard names its task as / and its unit in standard.unit, and --units of the standard takes the unit. When every trial of the run has a unit, trial-tasks.txt, rerun-tasks.txt and full-tasks.txt hold the units; a Harbor run keeps its task names. --- .claude/skills/run-to-fix/scripts/deciding.py | 4 ++-- .claude/skills/run-to-fix/scripts/selecting.py | 4 ++-- .claude/skills/run-to-fix/scripts/trials.py | 17 ++++++++++++++++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.claude/skills/run-to-fix/scripts/deciding.py b/.claude/skills/run-to-fix/scripts/deciding.py index e7a6b31a..f74805ad 100644 --- a/.claude/skills/run-to-fix/scripts/deciding.py +++ b/.claude/skills/run-to-fix/scripts/deciding.py @@ -14,7 +14,7 @@ from pricing import describe, price_of, task_costs, usd from reading import adapter_of, health from rounds import LoopError, ledger_round, load_ledger, now, open_round, read_input, read_json, save_ledger, write_json, write_paid -from trials import TrialRow +from trials import TrialRow, run_names ASSETS = os.path.join(os.path.dirname(os.path.dirname(os.path.abspath(__file__))), "assets") CHOICES = ("stop", "rerun", "full") @@ -180,7 +180,7 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: for name, tasks, repeat in (("rerun", rerun_tasks, TARGET_TRIALS), ("full", full_tasks, 1)): units = os.path.join(round_dir, f"{name}-tasks.txt") with open(units, "w", encoding="utf-8", newline="\n") as fh: - fh.write("".join(t + "\n" for t in tasks)) + fh.write("".join(t + "\n" for t in run_names(before, tasks))) command = None line = adapter.commands["run_tasks"] if line is not None: diff --git a/.claude/skills/run-to-fix/scripts/selecting.py b/.claude/skills/run-to-fix/scripts/selecting.py index cf9fe012..81e7feb8 100644 --- a/.claude/skills/run-to-fix/scripts/selecting.py +++ b/.claude/skills/run-to-fix/scripts/selecting.py @@ -15,7 +15,7 @@ from reading import adapter_of from rounds import (LoopError, ledger_round, load_ledger, now, open_round, read_input, read_json, run_command, save_ledger, write_json, write_paid) -from trials import solved_tasks +from trials import run_names, solved_tasks #: canaries are drawn with this seed, so the same round proposes the same canaries SEED = 20261117 @@ -127,7 +127,7 @@ def cmd_select(args: argparse.Namespace) -> None: tasks = sizes[args.size] units = os.path.join(round_dir, "trial-tasks.txt") with open(units, "w", encoding="utf-8", newline="\n") as fh: - fh.write("".join(t + "\n" for t in tasks)) + fh.write("".join(t + "\n" for t in run_names(rows, tasks))) build = build_on(adapter, args.loop, round_dir, ref, args.command_timeout_s) write_json(os.path.join(round_dir, "build.json"), build) entry["selection"] = {"ref": ref, "size": args.size, "targets": targets, diff --git a/.claude/skills/run-to-fix/scripts/trials.py b/.claude/skills/run-to-fix/scripts/trials.py index 6f0c583c..f58b25bf 100644 --- a/.claude/skills/run-to-fix/scripts/trials.py +++ b/.claude/skills/run-to-fix/scripts/trials.py @@ -41,6 +41,7 @@ class TrialRow: cost_usd: Optional[float] agent: Optional[str] verifier_ran: bool + unit: Optional[str] def read_json(path: str) -> object: @@ -107,6 +108,20 @@ def _usage(trial_dir: str) -> Tuple[Optional[int], Optional[int]]: return calls, (max(outputs) if outputs else None) +def unit_of(result: dict) -> Optional[str]: + """The unit id a job of the standard gives a trial; a Harbor trial has none.""" + unit = (result.get("standard") or {}).get("unit") if isinstance(result.get("standard"), dict) else None + return unit if isinstance(unit, str) and unit else None + + +def run_names(rows: List[TrialRow], tasks: List[str]) -> List[str]: + """The names a runner takes for `tasks`: the unit ids of a job of the standard, else the task names.""" + units = {r.task: r.unit for r in rows} + if rows and all(r.unit is not None for r in rows): + return [units[t] for t in tasks] + return list(tasks) + + def read_trial(trial_dir: str, pass_threshold: float) -> TrialRow: result = read_json(os.path.join(trial_dir, "result.json")) if not isinstance(result, dict) or "task_name" not in result: @@ -128,7 +143,7 @@ def read_trial(trial_dir: str, pass_threshold: float) -> TrialRow: exception_message=str(exc.get("exception_message") or "") if exc else None, agent_s=_span(result.get("agent_execution")), model_calls=calls, max_completion_tokens=max_out, cost_usd=float(cost) if isinstance(cost, (int, float)) else None, agent=info.get("name"), - verifier_ran=reward is not None) + verifier_ran=reward is not None, unit=unit_of(result)) def classify(row: TrialRow, limits: Limits) -> str: From 1548aaa0747d5269321f3b46ed074a83be964dbf Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:08:35 +0100 Subject: [PATCH 78/92] test(run-to-fix): run stages 1, 2 and 5 for an agent with no checkout of its own --- .claude/skills/run-to-fix/tests/test_read.py | 40 +++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/tests/test_read.py b/.claude/skills/run-to-fix/tests/test_read.py index a67fbe8f..4e435d5a 100644 --- a/.claude/skills/run-to-fix/tests/test_read.py +++ b/.claude/skills/run-to-fix/tests/test_read.py @@ -7,7 +7,7 @@ import subprocess import sys -from jobs import (LIMITS, LOOP, NO_LIMITS, LoopCase, adapter, load, run, trial, write_job, write_json) +from jobs import (LIMITS, LOOP, NO_LIMITS, LoopCase, adapter, load, run, trial, write, write_job, write_json) def healthy(root: str, name: str, agent: str) -> str: @@ -289,3 +289,41 @@ def test_read_refuses_a_budget_that_is_not_an_amount(self) -> None: self.assertNotEqual(r.code, 0, value) self.assertIn("--budget-usd", r.err) self.assertFalse(os.path.exists(os.path.join(self.loop, "round-01"))) + + +class NoSource(LoopCase): + """An agent with no checkout of its own: stages 1, 2 and 5 run, the stages that touch its code refuse.""" + + def adapter_without_source(self, commands: dict) -> dict: + data = adapter("alpha", NO_LIMITS, commands) + data["source"] = None + return data + + def test_read_classify_and_compare_run_without_a_checkout(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + base_dir = healthy(self.jobs, "base", "alpha") + self.set_adapter(self.adapter_without_source({})) + self.read_ok(run_dir, base_dir) + self.classify_ok() + r = run(["compare", "--loop", self.loop, "--new", run_dir, "--against", "baseline"]) + self.assertEqual(r.code, 0, r.out + r.err) + + def test_a_candidate_needs_the_checkout(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + self.set_adapter(self.adapter_without_source({})) + self.read_ok(run_dir, None) + self.classify_ok() + write(self.round_file("causes/c1.md"), "# A cause\n\nRule: g01\n") + r = run(["candidate", "--loop", self.loop, "--id", "c1", "--cause", "c1", "--ref", "HEAD", "--effect", "outcome"]) + self.assertNotEqual(r.code, 0) + self.assertIn("source", r.err) + + def test_a_command_on_the_code_needs_the_checkout(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + data = self.adapter_without_source({"test": "pytest {worktree}"}) + data["test_failed_pattern"] = r"^FAILED (\S+)" + self.set_adapter(data) + r = self.read(run_dir, None) + self.assertNotEqual(r.code, 0) + self.assertIn("commands.test", r.err) + self.assertIn("source", r.err) From 8af19eac116cb092826f24f2bb72a710c4822517 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:11:16 +0100 Subject: [PATCH 79/92] fix(run-to-fix): run stages 1, 2 and 5 for an agent with no checkout of its own source in adapter.json may be null. read, classify and compare never open the agent's code and run without it; candidate, check, select and decide --stop 2 refuse with a message that names source. An adapter with no source refuses every command that runs on the code; commands.classes then runs in the loop folder. --- .claude/skills/run-to-fix/scripts/adapter.py | 30 +++++++++++++++---- .claude/skills/run-to-fix/scripts/checking.py | 8 +++-- .../skills/run-to-fix/scripts/classifying.py | 6 ++-- .claude/skills/run-to-fix/scripts/deciding.py | 3 +- .../skills/run-to-fix/scripts/selecting.py | 4 +-- 5 files changed, 37 insertions(+), 14 deletions(-) diff --git a/.claude/skills/run-to-fix/scripts/adapter.py b/.claude/skills/run-to-fix/scripts/adapter.py index a4c47c53..95df568d 100644 --- a/.claude/skills/run-to-fix/scripts/adapter.py +++ b/.claude/skills/run-to-fix/scripts/adapter.py @@ -9,6 +9,8 @@ SCHEMA = "run-to-fix.adapter/v1" COMMANDS = ("test", "test_one", "build", "run_tasks", "replay", "classes") +#: the commands that run on the agent's code, so need source.checkout +CODE_COMMANDS = ("test", "test_one", "build", "run_tasks", "replay") TAKES: Dict[str, FrozenSet[str]] = { "test": frozenset({"worktree"}), "test_one": frozenset({"worktree", "test"}), @@ -42,9 +44,9 @@ class Limits: @dataclass(frozen=True) class Adapter: agent: str - checkout: str - base_ref: str - work_branch: str + checkout: Optional[str] + base_ref: Optional[str] + work_branch: Optional[str] commands: Dict[str, Optional[str]] test_failed_pattern: Optional["re.Pattern[str]"] replay_success_labels: List[str] @@ -91,6 +93,14 @@ def _command(name: str, value: object) -> Optional[str]: return line +def source_of(adapter: Adapter, step: str) -> str: + """The agent's checkout, for a step that works on its code.""" + if adapter.checkout is None or adapter.base_ref is None: + raise AdapterError(f"loop.py {step} works on the agent's code, and adapter.json has no source; read, " + "classify and compare run without it") + return adapter.checkout + + def load_adapter(path: str) -> Adapter: with open(path, "rb") as fh: raw = fh.read() @@ -102,11 +112,19 @@ def load_adapter(path: str) -> Adapter: raise AdapterError(f"adapter.json: schema must be {SCHEMA}, got {data.get('schema')!r}") agent = _text(_key(data, "agent", ""), "agent") source = _key(data, "source", "") - checkout = _text(_key(source, "checkout", "source."), "source.checkout") - base_ref = _text(_key(source, "base_ref", "source."), "source.base_ref") - work_branch = _text(_key(source, "work_branch", "source."), "source.work_branch") + if source is None: + checkout = base_ref = work_branch = None + else: + checkout = _text(_key(source, "checkout", "source."), "source.checkout") + base_ref = _text(_key(source, "base_ref", "source."), "source.base_ref") + work_branch = _text(_key(source, "work_branch", "source."), "source.work_branch") raw_commands = _key(data, "commands", "") commands = {name: _command(name, _key(raw_commands, name, "commands.")) for name in COMMANDS} + if source is None: + on_code = [name for name in CODE_COMMANDS if commands[name] is not None] + if on_code: + raise AdapterError("adapter.json: " + ", ".join(f"commands.{n}" for n in on_code) + " works on the agent's " + "code, and source is null; give the checkout in source or make the command null") unknown = sorted(set(raw_commands) - set(COMMANDS)) if unknown: raise AdapterError(f"adapter.json: unknown commands {unknown}; the commands are {list(COMMANDS)}") diff --git a/.claude/skills/run-to-fix/scripts/checking.py b/.claude/skills/run-to-fix/scripts/checking.py index f6cc6d34..22e2588a 100644 --- a/.claude/skills/run-to-fix/scripts/checking.py +++ b/.claude/skills/run-to-fix/scripts/checking.py @@ -9,7 +9,7 @@ from dataclasses import dataclass from typing import Dict, List, Optional, Tuple -from adapter import Adapter +from adapter import Adapter, source_of from checkout import diff_size, resolve, worktree, write_paid_on_ref from classifying import read_rows from reading import adapter_of @@ -82,9 +82,10 @@ def cmd_candidate(args: argparse.Namespace) -> None: raise LoopError(f"--test is a path inside the checkout, got {args.test!r}") if args.effect == "report" and adapter.commands["replay"] is None: raise LoopError("an effect on the report is seen only in the replay, and commands.replay is null in adapter.json") + checkout = source_of(adapter, "candidate") rule, trials, tasks = cause_trials(round_dir, args.cause, args.effect) - ref = resolve(adapter.checkout, args.ref) - test_ref = resolve(adapter.checkout, args.test_ref) if args.test_ref else None + ref = resolve(checkout, args.ref) + test_ref = resolve(checkout, args.test_ref) if args.test_ref else None record = {"schema": "run-to-fix.candidate/v1", "id": args.id, "created": now(), "cause": args.cause, "rule": rule, "effect": args.effect, "ref": ref, "test": args.test, "test_ref": test_ref, "prediction": {"trials": trials, "tasks": tasks if args.effect == "outcome" else []}} @@ -256,6 +257,7 @@ def cmd_check(args: argparse.Namespace) -> None: adapter = adapter_of(args.loop) round_dir = open_round(args.loop) record = read_input(round_dir) + source_of(adapter, "check") cdir = os.path.join(round_dir, "candidates", candidate_id(args.candidate)) cand = load_candidate(os.path.join(cdir, "candidate.json")) # a check that fails part way must not leave the success of an earlier check to be read diff --git a/.claude/skills/run-to-fix/scripts/classifying.py b/.claude/skills/run-to-fix/scripts/classifying.py index 8d40a37f..29ac3a3f 100644 --- a/.claude/skills/run-to-fix/scripts/classifying.py +++ b/.claude/skills/run-to-fix/scripts/classifying.py @@ -72,10 +72,12 @@ def agent_classes(adapter: Adapter, round_dir: str, job_dir: str, timeout_s: flo return {"ran": False, "reason": "commands.classes is null: the agent's own classes did not run", "trials": {}} out_json = os.path.join(round_dir, "agent-classes.json") values = {"job_dir": job_dir, "out_json": out_json} + # without a checkout, the agent's classes run in the loop folder; they read only the job + cwd = adapter.checkout or os.path.dirname(round_dir) if "classes" in adapter.paid: - path = write_paid(round_dir, "classes", line, values, adapter.checkout) + path = write_paid(round_dir, "classes", line, values, cwd) return {"ran": False, "reason": f"commands.classes costs money: written to {path}, not started", "trials": {}} - ran = run_command(line, values, adapter.checkout, os.path.join(round_dir, "logs", "classes.log"), timeout_s) + ran = run_command(line, values, cwd, os.path.join(round_dir, "logs", "classes.log"), timeout_s) if ran.code != 0: status = "did not finish in time" if ran.code is None else f"exited with code {ran.code}" raise LoopError(f"commands.classes {status}; the log is {ran.log}:\n{ran.tail}") diff --git a/.claude/skills/run-to-fix/scripts/deciding.py b/.claude/skills/run-to-fix/scripts/deciding.py index f74805ad..6c7f29a7 100644 --- a/.claude/skills/run-to-fix/scripts/deciding.py +++ b/.claude/skills/run-to-fix/scripts/deciding.py @@ -7,6 +7,7 @@ import re from typing import Dict, List, Optional +from adapter import source_of from checking import candidate_id, load_candidate from checkout import resolve from classifying import read_rows @@ -127,7 +128,7 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: + ", ".join(repr(a) for a in checked.agents)) job_dir = checked.dir this_run: List[TrialRow] = read_rows(job_dir, threshold) - code_ref = resolve(adapter.checkout, args.code_ref) + code_ref = resolve(source_of(adapter, "decide --stop 2"), args.code_ref) if code_ref != selection["ref"]: raise LoopError(f"a trial run of this round runs the commit select chose, {selection['ref'][:12]}; --code-ref " f"names {code_ref[:12]}") diff --git a/.claude/skills/run-to-fix/scripts/selecting.py b/.claude/skills/run-to-fix/scripts/selecting.py index 81e7feb8..b6a8a9e7 100644 --- a/.claude/skills/run-to-fix/scripts/selecting.py +++ b/.claude/skills/run-to-fix/scripts/selecting.py @@ -7,7 +7,7 @@ import random from typing import Dict, List, Optional, Tuple -from adapter import Adapter +from adapter import Adapter, source_of from checkout import is_ancestor, resolve, worktree, write_paid_on_ref from classifying import read_rows from comparing import TARGET_TRIALS, trial_spent @@ -85,7 +85,7 @@ def cmd_select(args: argparse.Namespace) -> None: ledger = load_ledger(args.loop) entry = ledger_round(ledger, os.path.basename(round_dir)) kept = kept_candidates(round_dir, entry) - ref = resolve(adapter.checkout, args.ref) + ref = resolve(source_of(adapter, "select"), args.ref) for checks in kept: if not is_ancestor(adapter.checkout, checks["ref"], ref): raise LoopError(f"{args.ref} does not contain the kept candidate {checks['id']} ({checks['ref'][:12]}): " From f4291c6d80281420df2d44d55cf8c9f7b90064d0 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 20:12:01 +0100 Subject: [PATCH 80/92] docs(run-to-fix): describe the standard's model calls and units, an agent with no checkout, and what a round counts classes.md: model calls from n_model_calls when result.json holds it, output_limit switched off when no trial records a reply's tokens, and found classes counted on the round's run. adapter.md: a null source and unit ids in the units file. files.md: a round's own run is not money it spent. --- .claude/skills/run-to-fix/SKILL.md | 3 ++- .claude/skills/run-to-fix/references/adapter.md | 3 ++- .claude/skills/run-to-fix/references/classes.md | 14 ++++++++------ .claude/skills/run-to-fix/references/files.md | 6 ++++-- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/.claude/skills/run-to-fix/SKILL.md b/.claude/skills/run-to-fix/SKILL.md index 043855d6..adbc1d7f 100644 --- a/.claude/skills/run-to-fix/SKILL.md +++ b/.claude/skills/run-to-fix/SKILL.md @@ -25,7 +25,8 @@ causes, writing tests and fixes in the agent's code. - An adapter, `adapter.json` in the loop folder: one file per agent with its checkout, its commands and the limits of the run (`references/adapter.md`). For an agent of the Nullius catalogue, draft it from `catalogue/agents//agent.json` (`versions..upstream.repository` and `tag`) and the run's settings, and - ask the person to confirm it. Ask the person for every value you cannot read from a file. + ask the person to confirm it. Ask the person for every value you cannot read from a file. An agent with no checkout + of its own takes `"source": null`: stages 1, 2 and 5 run, and the stages that change its code refuse. - The run: one job folder with one folder per trial. A baseline run to compare it with is optional. `L` below is `scripts/loop.py` beside this skill. Every command takes `--loop `. `references/files.md` diff --git a/.claude/skills/run-to-fix/references/adapter.md b/.claude/skills/run-to-fix/references/adapter.md index 4f20e3e3..ce848480 100644 --- a/.claude/skills/run-to-fix/references/adapter.md +++ b/.claude/skills/run-to-fix/references/adapter.md @@ -15,6 +15,7 @@ in `round-NN/input.json`. |---|---|---| | `schema` | string | `run-to-fix.adapter/v1` | | `agent` | string | the value of `agent_info.name` in the trials of the run; `read` refuses a run whose trials name another agent | +| `source` | object or null | null for an agent with no checkout of its own: `read`, `classify` and `compare` run, and `candidate`, `check`, `select` and `decide --stop 2` refuse; every command but `classes` is then null, and `classes` runs in the loop folder | | `source.checkout` | path | the agent's git checkout on this machine | | `source.base_ref` | git ref | the commit the run was built from | | `source.work_branch` | branch | the branch that receives test and fix commits | @@ -44,7 +45,7 @@ Every key is present, and a value the agent does not have is null. `read` refuse | `{out_json}` | `replay`, `classes` | the file the command writes; the loop reads it after a zero exit | | `{base_ref}` | `replay`, optional | the full commit id of `source.base_ref` | | `{before_ref}` | `replay`, optional | the full commit id the replay is compared with: for the replay before a fix, its own `{ref}`; for the replay of the fix, the ref before it | -| `{units_file}` | `run_tasks` | a text file with one task name per line | +| `{units_file}` | `run_tasks` | a text file with one task per line: the unit id (`standard.unit`) when every trial of the run is a trial of the standard, else the task name | | `{repeat}` | `run_tasks` | trials per task | | `{jobs_dir}` | `run_tasks` | where the run writes its job folder | | `{budget_usd}` | `run_tasks` | the budget the person named for this run, in US dollars | diff --git a/.claude/skills/run-to-fix/references/classes.md b/.claude/skills/run-to-fix/references/classes.md index 21857c9a..4b79c4fb 100644 --- a/.claude/skills/run-to-fix/references/classes.md +++ b/.claude/skills/run-to-fix/references/classes.md @@ -20,11 +20,13 @@ The first rule that holds gives the class. | `verifier_failed` | the verifier ran and the reward is under the threshold | never | With every limit null, each trial is `pass`, `time_limit`, `exception`, `no_score` or `verifier_failed`. -`groups.json` lists the rules a null limit switched off under `rules_off`. +`groups.json` lists the rules a null limit switched off under `rules_off`, and `output_limit` with its reason when the +adapter sets the limit and no trial records the tokens of one reply, as on a job of the standard. -The primary reward and the count of model calls follow the rules of `run-analysis`. The reward is -`verifier_result.rewards.reward`, else the first numeric reward; model calls are the per-call usage records -(`extra.raw_usage` or steps with `metrics`, whichever is more), else the agent's steps. The largest reply is the +The primary reward follows the rule of `run-analysis`: `verifier_result.rewards.reward`, else the first numeric +reward. Model calls are `agent_result.n_model_calls` when `result.json` holds it, as a job of the standard does; else +they follow `run-analysis`: the per-call usage records (`extra.raw_usage` or steps with `metrics`, whichever is more), +else the agent's steps. The largest reply is the largest `metrics.completion_tokens` of a step or `output_tokens` of a usage record. `classify` stops when its count of trials or passes, or of the pairs with the baseline, differs from the count in `metrics.json`. @@ -62,8 +64,8 @@ of one task. ## Found classes -While reading trials, the assistant may add a rule that the script then counts on every trial, the passing ones -included. The rules live in `/found-classes.json`; the skill ships with none. +While reading trials, the assistant may add a rule that the script then counts on every trial of the round's run, +the passing ones included; the baseline is not counted. The rules live in `/found-classes.json`; the skill ships with none. ```json { diff --git a/.claude/skills/run-to-fix/references/files.md b/.claude/skills/run-to-fix/references/files.md index 28c42641..d354407d 100644 --- a/.claude/skills/run-to-fix/references/files.md +++ b/.claude/skills/run-to-fix/references/files.md @@ -40,7 +40,7 @@ round that has no `result.json`. | `replays/-against-.json`, `.log` | 3 | `check` | one replay per pair of refs of a round, made once | | `decision-1.md` | stop 1 | `decide --stop 1` | one row per candidate and the checks that did not run | | `commands/-.sh`, `commands/run_tasks*.sh` | 3, 4 | `check`, `select`, `decide --stop 2` | commands a person starts | -| `trial-tasks.txt` | 4 | `select --size` | the tasks of the trial run, one per line, the `{units_file}` of `run_tasks` | +| `trial-tasks.txt` | 4 | `select --size` | the tasks of the trial run, one per line (unit ids on a job of the standard), the `{units_file}` of `run_tasks` | | `build.json` | 4 | `select --size` | the commit tried, the path `commands.build` printed and its SHA-256 | | `jobs/` | 4 | the person's run | the job folders that `run_tasks` writes | | `trial.json` | stop 2 | `decide --stop 2` | the round's trial runs, the results of targets, guards and canaries, the money spent and left, the options | @@ -91,7 +91,9 @@ Not checked: what the files cannot tell. Every answer of the person is stored with its date and in the person's words: the candidates kept at stop 1, the size and budget of the trial run, the choice at stop 2, and the words that start a round after a stop rule said -stop. The money a round has spent is the sum of its `trial_runs` plus the new run that `compare` reads. A trial run +stop. The money a round has spent is the sum of its `trial_runs` plus the new run that `compare` reads. A new run +that is the round's own run adds nothing, because the round did not pay for it; one that is a trial run of the round +is in the sum already. A trial run whose trials do not all report a cost counts with the person's figure (`cost_from`: `the person`) and keeps what its trials record in `cost_recorded_usd`; `code_ref` is the commit the person states it ran. From 4afbc086673632389a9b3f6330aee795ff238624 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 21:35:42 +0100 Subject: [PATCH 81/92] test(run-to-fix): refuse a recorded cost or budget that is not a finite amount of 0 or more --- .../skills/run-to-fix/tests/test_decide.py | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index 21c7435c..3c4f80e5 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -579,3 +579,53 @@ def test_the_task_lists_hold_unit_ids(self) -> None: self.assertEqual(r.code, 0, r.out + r.err) self.assertEqual(self.lines("rerun-tasks.txt"), ["f1", "f2", "p1"]) self.assertEqual(self.lines("full-tasks.txt"), ["f1", "f2", "f3", "p1", "p2", "p3"]) + + +class RecordedAmounts(Flow): + """A recorded cost or budget that is not a finite amount of 0 or more stops the loop before it is used.""" + + def job_with_cost(self, value: float) -> str: + self.chosen() + trials = [] + for task in ("f1", "f2", "p1"): + for k in range(3): + t = trial(f"{task}-{k}__t", f"bench/{task}", 1.0) + t.cost = value if (task, k) == ("f1", 0) else 0.01 + trials.append(t) + return write_job(self.jobs, "trial", "alpha", "harbor", trials) + + def assert_refused_before_any_write(self, job: str, value: str) -> None: + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--code-ref", self.code_ref()]) + self.assertNotEqual(r.code, 0) + self.assertIn("f1-0__t", r.err) + self.assertIn(value, r.err) + self.assertEqual(load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["trial_runs"], []) + self.assertFalse(os.path.exists(self.round_file("commands/run_tasks-full.sh"))) + + def test_a_nan_cost_of_a_trial_is_refused(self) -> None: + self.assert_refused_before_any_write(self.job_with_cost(float("nan")), "nan") + + def test_a_negative_cost_of_a_trial_is_refused(self) -> None: + self.assert_refused_before_any_write(self.job_with_cost(-0.01), "-0.01") + + def test_a_nan_cost_in_the_ledger_is_refused(self) -> None: + job = self.chosen() + run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--code-ref", self.code_ref()]) + path = os.path.join(self.loop, "ledger.json") + ledger = load(path) + ledger["rounds"][0]["trial_runs"][0]["cost_usd"] = float("nan") + write_json(path, ledger) + r = self.select(["--size", "targets", "--budget-usd", "1", "--text", "again"]) + self.assertNotEqual(r.code, 0) + self.assertIn("ledger.json", r.err) + + def test_a_nan_budget_in_the_input_is_refused(self) -> None: + self.kept() + path = self.round_file("input.json") + record = load(path) + record["budget_usd"] = float("nan") + write_json(path, record) + r = self.select(["--size", "targets", "--budget-usd", "1", "--text", "a budget"]) + self.assertNotEqual(r.code, 0) + self.assertIn("input.json", r.err) + self.assertFalse(os.path.exists(self.round_file("trial-tasks.txt"))) From 0d8e4db08fb709eb8572ba59aa514ac35c8df1bb Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 21:38:56 +0100 Subject: [PATCH 82/92] fix(run-to-fix): refuse a recorded cost or budget that is not a finite amount of 0 or more A trial's cost_usd, the budget in input.json and the cost of each trial run in ledger.json are read only as finite amounts of 0 or more; any other value stops the loop with the file, the trial and the value named, before a ledger entry or a command is written. --- .claude/skills/run-to-fix/scripts/comparing.py | 6 +++--- .claude/skills/run-to-fix/scripts/rounds.py | 9 +++++++++ .claude/skills/run-to-fix/scripts/trials.py | 7 ++++++- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/.claude/skills/run-to-fix/scripts/comparing.py b/.claude/skills/run-to-fix/scripts/comparing.py index 69079477..e352dc43 100644 --- a/.claude/skills/run-to-fix/scripts/comparing.py +++ b/.claude/skills/run-to-fix/scripts/comparing.py @@ -7,8 +7,8 @@ from classifying import classes_of, cross_check, failure_groups, read_rows from reading import adapter_of, analyze, health -from rounds import (STOP_RULE, LoopError, call_runs, ledger_round, load_ledger, now, open_round, read_input, - read_json, save_ledger, write_json) +from rounds import (STOP_RULE, LoopError, call_runs, ledger_round, load_ledger, now, open_round, read_input, read_json, + recorded_amount, save_ledger, write_json) from pricing import describe, price_of, task_costs, usd from stats import binomial_two_sided, fisher_two_sided, verdict from trials import TrialRow @@ -73,7 +73,7 @@ def trial_spent(entry: dict) -> float: if not t.get("cost_complete", True) and t.get("cost_from") != "the person": raise LoopError(f"the money the trial run {t['job_dir']} spent is unknown: some of its trials report no " "cost. Record it again with loop.py decide --stop 2 and --trial-cost-usd.") - total += t["cost_usd"] + total += recorded_amount(t.get("cost_usd"), f"ledger.json, the cost of the trial run {t.get('job_dir')},") return total diff --git a/.claude/skills/run-to-fix/scripts/rounds.py b/.claude/skills/run-to-fix/scripts/rounds.py index 9e61b6f2..36749f7a 100644 --- a/.claude/skills/run-to-fix/scripts/rounds.py +++ b/.claude/skills/run-to-fix/scripts/rounds.py @@ -4,6 +4,7 @@ import datetime as dt import hashlib import json +import math import os import re import shlex @@ -105,6 +106,13 @@ def open_round(loop_dir: str) -> str: return existing[-1] +def recorded_amount(value: object, where: str) -> float: + """An amount in US dollars read from a record: a finite number of 0 or more.""" + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0: + raise LoopError(f"{where} holds {value!r}, which is not a finite amount of 0 or more") + return float(value) + + def read_input(round_dir: str) -> dict: path = os.path.join(round_dir, "input.json") if not os.path.isfile(path): @@ -115,6 +123,7 @@ def read_input(round_dir: str) -> dict: if data.get("refused"): raise LoopError(f"{os.path.basename(round_dir)}: read refused this run ({data.get('reason')}); " "start a new loop folder for another run") + data["budget_usd"] = recorded_amount(data.get("budget_usd"), f"{path}, budget_usd,") return data diff --git a/.claude/skills/run-to-fix/scripts/trials.py b/.claude/skills/run-to-fix/scripts/trials.py index f58b25bf..c0684fca 100644 --- a/.claude/skills/run-to-fix/scripts/trials.py +++ b/.claude/skills/run-to-fix/scripts/trials.py @@ -3,6 +3,7 @@ import datetime as dt import json +import math import os import re from dataclasses import dataclass @@ -132,6 +133,10 @@ def read_trial(trial_dir: str, pass_threshold: float) -> TrialRow: info = result.get("agent_info") or {} agent_result = result.get("agent_result") or {} cost = agent_result.get("cost_usd") + if cost is not None and (isinstance(cost, bool) or not isinstance(cost, (int, float)) or not math.isfinite(cost) + or cost < 0): + raise TrialError(f"{trial_dir}/result.json: the trial {result.get('trial_name') or os.path.basename(trial_dir)} " + f"records a cost of {cost!r}, which is not a finite amount of 0 or more") counted = agent_result.get("n_model_calls") # a job of the standard counts its model calls; its trajectory has one step per message, not per call if isinstance(counted, int) and not isinstance(counted, bool): @@ -142,7 +147,7 @@ def read_trial(trial_dir: str, pass_threshold: float) -> TrialRow: exception_type=str(exc.get("exception_type") or "Exception") if exc else None, exception_message=str(exc.get("exception_message") or "") if exc else None, agent_s=_span(result.get("agent_execution")), model_calls=calls, max_completion_tokens=max_out, - cost_usd=float(cost) if isinstance(cost, (int, float)) else None, agent=info.get("name"), + cost_usd=float(cost) if cost is not None else None, agent=info.get("name"), verifier_ran=reward is not None, unit=unit_of(result)) From 5a087ac714bd0d7cae60b5ecc3831d877744f15f Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 21:39:38 +0100 Subject: [PATCH 83/92] test(run-to-fix): refuse a check made with another adapter at stop 1, in the answer and in select --- .../skills/run-to-fix/tests/test_decide.py | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index 3c4f80e5..7f82c1d4 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -629,3 +629,36 @@ def test_a_nan_budget_in_the_input_is_refused(self) -> None: self.assertNotEqual(r.code, 0) self.assertIn("input.json", r.err) self.assertFalse(os.path.exists(self.round_file("trial-tasks.txt"))) + + +class ChecksOfAnotherAdapter(Flow): + """A check is evidence only for the adapter it ran with; a changed adapter.json asks for the check again.""" + + def change_replay(self) -> None: + path = os.path.join(self.loop, "adapter.json") + data = load(path) + data["commands"]["replay"] = "exit 23 # {worktree} {ref} {job_dir} {out_json}" + write_json(path, data) + + def test_stop_one_refuses_a_check_of_another_adapter(self) -> None: + self.checked() + self.change_replay() + r = run(["decide", "--loop", self.loop, "--stop", "1"]) + self.assertNotEqual(r.code, 0) + self.assertIn("loop.py check --candidate", r.err) + + def test_the_answer_refuses_a_check_of_another_adapter(self) -> None: + self.checked() + run(["decide", "--loop", self.loop, "--stop", "1"]) + self.change_replay() + r = run(["answer", "--loop", self.loop, "--stop", "1", "--keep", "good", "--text", "keep good"]) + self.assertNotEqual(r.code, 0) + self.assertIn("loop.py check --candidate good", r.err) + + def test_select_refuses_a_check_of_another_adapter(self) -> None: + self.kept() + self.change_replay() + r = self.select(["--size", "guards", "--budget-usd", "0.5", "--text", "the guards"]) + self.assertNotEqual(r.code, 0) + self.assertIn("loop.py check --candidate good", r.err) + self.assertFalse(os.path.exists(self.round_file("trial-tasks.txt"))) From 676cc5d12b7fac8a0defaeb498529702e5ae0a6f Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 21:47:04 +0100 Subject: [PATCH 84/92] fix(run-to-fix): refuse a check made with another adapter at stop 1, in the answer and in select checks.json records the SHA-256 of the adapter it was made with. decide --stop 1, answer --stop 1 and select read a check only under that same adapter and otherwise ask for loop.py check again. --- .claude/skills/run-to-fix/scripts/checking.py | 15 ++++++++++++++- .claude/skills/run-to-fix/scripts/deciding.py | 9 +++++++-- .claude/skills/run-to-fix/scripts/selecting.py | 7 ++++--- 3 files changed, 25 insertions(+), 6 deletions(-) diff --git a/.claude/skills/run-to-fix/scripts/checking.py b/.claude/skills/run-to-fix/scripts/checking.py index 22e2588a..d9987fa5 100644 --- a/.claude/skills/run-to-fix/scripts/checking.py +++ b/.claude/skills/run-to-fix/scripts/checking.py @@ -60,6 +60,18 @@ def cause_trials(round_dir: str, cause: str, effect: str) -> Tuple[str, List[str return rule, sorted(names), tasks +def load_checks(round_dir: str, cid: str, adapter: Adapter) -> dict: + """The checks of a candidate, made with the adapter the loop has now.""" + path = os.path.join(round_dir, "candidates", cid, "checks.json") + if not os.path.isfile(path): + raise LoopError(f"the candidate {cid} is not checked: run loop.py check --candidate {cid}") + checks = read_json(path) + if not isinstance(checks, dict) or checks.get("adapter_sha256") != adapter.sha256: + raise LoopError(f"the checks of {cid} were made with another adapter.json than the loop has now; run " + f"loop.py check --candidate {cid} again") + return checks + + def load_candidate(path: str) -> dict: """A candidate.json of this version: one that names the effect of its fix.""" if not os.path.isfile(path): @@ -269,7 +281,8 @@ def cmd_check(args: argparse.Namespace) -> None: base = resolve(adapter.checkout, adapter.base_ref) before_ref = cand["test_ref"] or base checks = { - "schema": "run-to-fix.checks/v1", "candidate": cand["id"], "created": now(), "ref": cand["ref"], + "schema": "run-to-fix.checks/v1", "candidate": cand["id"], "created": now(), "adapter_sha256": adapter.sha256, + "ref": cand["ref"], "test_ref": cand["test_ref"], "base_ref": base, "test": check_test(adapter, args, round_dir, cdir, cand), "suite": check_suite(adapter, args, round_dir, cdir, base, cand["ref"]), diff --git a/.claude/skills/run-to-fix/scripts/deciding.py b/.claude/skills/run-to-fix/scripts/deciding.py index 6c7f29a7..5d11beb3 100644 --- a/.claude/skills/run-to-fix/scripts/deciding.py +++ b/.claude/skills/run-to-fix/scripts/deciding.py @@ -8,7 +8,7 @@ from typing import Dict, List, Optional from adapter import source_of -from checking import candidate_id, load_candidate +from checking import candidate_id, load_candidate, load_checks from checkout import resolve from classifying import read_rows from comparing import REPORT_NOT_READ, TARGET_TRIALS, passes, targets_of, trial_spent @@ -86,11 +86,12 @@ def decide_one(args: argparse.Namespace, round_dir: str, record: dict) -> None: if os.path.isdir(base) else [] if not ids: raise LoopError("no candidate is registered: add one with loop.py candidate, then loop.py check") + adapter = adapter_of(args.loop) rows, not_run, unchanged = [], [], [] for cid in ids: cand = load_candidate(os.path.join(base, cid, "candidate.json")) checks_path = os.path.join(base, cid, "checks.json") - checks = read_json(checks_path) if os.path.isfile(checks_path) else None + checks = load_checks(round_dir, cid, adapter) if os.path.isfile(checks_path) else None rows.append("| " + " | ".join(cell(c) for c in candidate_row(cand, checks)) + " |") if checks is None: not_run.append(f"- {cid}: not checked; run loop.py check --candidate {cid}") @@ -259,6 +260,10 @@ def cmd_answer(args: argparse.Namespace) -> None: cands = {k: load_candidate(os.path.join(round_dir, "candidates", k, "candidate.json")) for k in keep if os.path.isfile(os.path.join(round_dir, "candidates", k, "candidate.json"))} missing = [k for k in keep if not os.path.isfile(os.path.join(round_dir, "candidates", k, "checks.json"))] + if not missing: + adapter = adapter_of(args.loop) + for k in keep: + load_checks(round_dir, k, adapter) if missing: raise LoopError(f"these candidates are not checked: {', '.join(missing)}") entry["answers"].append({"about": "stop 1: the candidates to keep", "date": now(), "text": args.text, diff --git a/.claude/skills/run-to-fix/scripts/selecting.py b/.claude/skills/run-to-fix/scripts/selecting.py index b6a8a9e7..770464fc 100644 --- a/.claude/skills/run-to-fix/scripts/selecting.py +++ b/.claude/skills/run-to-fix/scripts/selecting.py @@ -8,6 +8,7 @@ from typing import Dict, List, Optional, Tuple from adapter import Adapter, source_of +from checking import load_checks from checkout import is_ancestor, resolve, worktree, write_paid_on_ref from classifying import read_rows from comparing import TARGET_TRIALS, trial_spent @@ -22,12 +23,12 @@ SIZES = ("targets", "guards", "canaries") -def kept_candidates(round_dir: str, entry: dict) -> List[dict]: +def kept_candidates(round_dir: str, entry: dict, adapter: Adapter) -> List[dict]: if entry.get("kept") is None: raise LoopError("stop 1 has no answer yet: record it with loop.py answer --stop 1") if not entry["kept"]: raise LoopError("the answer of stop 1 kept no candidate, so there is nothing to try") - return [dict(read_json(os.path.join(round_dir, "candidates", cid, "checks.json")), id=cid) for cid in entry["kept"]] + return [dict(load_checks(round_dir, cid, adapter), id=cid) for cid in entry["kept"]] def guards_of(kept: List[dict], rows_by_trial: Dict[str, Tuple[str, bool]], targets: List[str]) -> Tuple[List[str], Optional[str]]: @@ -84,7 +85,7 @@ def cmd_select(args: argparse.Namespace) -> None: record = read_input(round_dir) ledger = load_ledger(args.loop) entry = ledger_round(ledger, os.path.basename(round_dir)) - kept = kept_candidates(round_dir, entry) + kept = kept_candidates(round_dir, entry, adapter) ref = resolve(source_of(adapter, "select"), args.ref) for checks in kept: if not is_ancestor(adapter.checkout, checks["ref"], ref): From cdbd0ef9cb1f4451af69a8f62ce1d73bbc130476 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 21:50:32 +0100 Subject: [PATCH 85/92] test(run-to-fix): a trial run holds every selected task, and its repeats count over the round's trial jobs --- .../skills/run-to-fix/tests/test_decide.py | 70 +++++++++++++++++-- 1 file changed, 65 insertions(+), 5 deletions(-) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index 7f82c1d4..f317eff6 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -503,11 +503,6 @@ def test_a_missing_target_is_refused(self) -> None: self.assertNotEqual(r.code, 0) self.assertIn("bench/f2", r.err) - def test_a_task_with_another_count_of_trials_is_refused(self) -> None: - r = self.decide_on([trial(f"{t}__o", f"bench/{t}", 1.0) for t in ("f1", "f2", "p1")], self.repo.good) - self.assertNotEqual(r.code, 0) - self.assertIn("3 trials", r.err) - def test_another_commit_is_refused(self) -> None: r = self.decide_on(self.three_each(["f1", "f2", "p1"]), self.repo.bad) self.assertNotEqual(r.code, 0) @@ -521,6 +516,71 @@ def test_the_page_shows_the_commit_as_the_person_s_statement(self) -> None: "the person") +class SelectionCoverage(Flow): + """A trial job holds every selected task, each with one number of trials; the repeats count over the round's + trial jobs on the selected commit, and the targets are tested once every task has the selection's repeats.""" + + def job_of(self, name: str, tasks: List[str], repeat: int) -> str: + return write_job(self.jobs, name, "alpha", "harbor", + [trial(f"{t}-{k}__{name}", f"bench/{t}", 1.0) for t in tasks for k in range(repeat)]) + + def decide(self, job: str) -> Result: + return run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--code-ref", self.code_ref()]) + + def canaries_chosen(self) -> str: + self.kept() + r = self.select(["--size", "canaries", "--budget-usd", "1", "--text", "targets, guards and canaries"]) + self.assertEqual(r.code, 0, r.out + r.err) + canaries = load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["selection"]["canaries"] + self.assertEqual(len(canaries), 1) + return canaries[0] + + def test_a_missing_guard_is_refused(self) -> None: + self.chosen() + r = self.decide(self.job_of("noguard", ["f1", "f2"], 3)) + self.assertNotEqual(r.code, 0) + self.assertIn("bench/p1", r.err) + self.assertEqual(load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["trial_runs"], []) + + def test_a_missing_canary_is_refused(self) -> None: + canary = self.canaries_chosen() + r = self.decide(self.job_of("nocanary", ["f1", "f2", "p1"], 3)) + self.assertNotEqual(r.code, 0) + self.assertIn(canary, r.err) + + def test_tasks_with_unequal_trials_in_one_job_are_refused(self) -> None: + self.chosen() + trials = [trial(f"{t}-{k}__u", f"bench/{t}", 1.0) for t, n in (("f1", 3), ("f2", 1), ("p1", 3)) for k in range(n)] + r = self.decide(write_job(self.jobs, "uneven", "alpha", "harbor", trials)) + self.assertNotEqual(r.code, 0) + self.assertIn("bench/f2 (1)", r.err) + + def test_three_jobs_of_one_trial_each_make_a_complete_selection(self) -> None: + self.chosen() + for k, name in enumerate(("one", "two", "three")): + r = self.decide(self.job_of(name, ["f1", "f2", "p1"], 1)) + self.assertEqual(r.code, 0, r.out + r.err) + record = load(self.round_file("trial.json")) + self.assertEqual(record["trials_per_task"], k + 1) + self.assertEqual(record["complete"], k == 2) + page = self.page("decision-2.md") + self.assertIn(f"{k + 1} of the 3 trials per task", page) + if k < 2: + self.assertIsNone(record["targets"]["p"]) + self.assertIn("not complete", page) + else: + self.assertIsNotNone(record["targets"]["p"]) + self.assertEqual(record["targets"]["after"], {"trials": 6, "passed": 6}) + self.assertNotIn("not complete", page) + + def test_the_rerun_holds_every_selected_task(self) -> None: + canary = self.canaries_chosen() + r = self.decide(self.job_of("all", ["f1", "f2", "p1", canary.split("/", 1)[1]], 3)) + self.assertEqual(r.code, 0, r.out + r.err) + with open(self.round_file("rerun-tasks.txt"), encoding="utf-8") as fh: + self.assertIn(canary, fh.read().split()) + + class AmountsAreFinite(Flow): def test_select_refuses_a_budget_that_is_not_an_amount(self) -> None: self.kept() From 615762682fc78f8437a4d67c6c77ed0ac0ad03b4 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 21:57:00 +0100 Subject: [PATCH 86/92] fix(run-to-fix): a trial run holds every selected task, and its repeats count over the round's trial jobs decide --stop 2 refuses a job that lacks a guard or a canary, and a job whose tasks have unequal numbers of trials. It counts the trials per task over the round's trial jobs on the selected commit, so a runner that gives one trial per job is started three times. trial.json and the page say how many trials per task the jobs hold together and whether the selection is complete; the targets are tested only when it is. The rerun option runs every selected task. --- .claude/skills/run-to-fix/SKILL.md | 9 ++-- .../skills/run-to-fix/assets/decision-2.md | 3 +- .claude/skills/run-to-fix/references/files.md | 2 +- .claude/skills/run-to-fix/scripts/deciding.py | 43 +++++++++++++------ 4 files changed, 38 insertions(+), 19 deletions(-) diff --git a/.claude/skills/run-to-fix/SKILL.md b/.claude/skills/run-to-fix/SKILL.md index adbc1d7f..216c04c0 100644 --- a/.claude/skills/run-to-fix/SKILL.md +++ b/.claude/skills/run-to-fix/SKILL.md @@ -127,14 +127,17 @@ python3 L decide --loop DIR --stop 2 --trial-job JOB --code-ref COMMIT [--trial- `--code-ref` is the commit the person states the trial run ran. A job does not record its code; the page shows the commit as the person's statement, and `decide` refuses any commit other than the one `select` chose. The job must hold -only the selected tasks, every target, and the selection's count of trials for each task. When some of its trials +every selected task and no other, each with one number of trials. The trials per task count over this round's trial +jobs on that commit: a runner that gives each task one trial per job is started three times, and each job goes to +`decide --stop 2`. The page says how many trials per task the jobs hold together and whether the selection is +complete; the targets are tested only when it is. When some of its trials report no cost, `decide` refuses the run until the person gives its whole cost with `--trial-cost-usd`, at least what its trials record. **Stop 2.** Show the person `decision-2.md`. For a kept `report` candidate it says that this version does not read the agent's report in the new run. The page shows the build, the targets' trials before and in every trial run of this round on that build with Fisher's exact test, the guards and canaries, the money spent and left, and three options -with their price: stop, the targets and guards again, a full pass. The commands of the last two are written. Record +with their price: stop, the selected tasks again, a full pass. The commands of the last two are written. Record the choice: ```bash @@ -200,7 +203,7 @@ These follow `run-analysis`, with what the loop adds. of that wording in `run-analysis` changes what `read` refuses. - The classes read only shared files. What one agent writes reaches the loop through a found class or `commands.classes`. -- A trial run gives every task the same number of trials, because `run_tasks` takes one `{repeat}`. +- A trial job gives every task the same number of trials, because `run_tasks` takes one `{repeat}`. - A loop stopped with SIGTERM or Ctrl+C stops the adapter command it runs and removes its worktree. A loop killed with SIGKILL cannot: its worktree stays under `/worktrees/`, and `git worktree remove --force ` in the agent's checkout removes it. diff --git a/.claude/skills/run-to-fix/assets/decision-2.md b/.claude/skills/run-to-fix/assets/decision-2.md index 4dd6ff94..bd2f81c8 100644 --- a/.claude/skills/run-to-fix/assets/decision-2.md +++ b/.claude/skills/run-to-fix/assets/decision-2.md @@ -1,6 +1,7 @@ # Stop 2 of {{ROUND}}: what to run next -Build: {{BUILD}}. Code of the trial runs: {{CODE}}. Trial runs on it: {{TRIAL_JOB}}, {{TRIALS}} trials. Spent in this round: {{SPENT}}. Budget left: +Build: {{BUILD}}. Code of the trial runs: {{CODE}}. Trial runs on it: {{TRIAL_JOB}}, {{TRIALS}} trials. +{{COVERAGE}} Spent in this round: {{SPENT}}. Budget left: {{LEFT}} of {{BUDGET}}. | tasks | trials passed before | trials passed in the trial run | test | diff --git a/.claude/skills/run-to-fix/references/files.md b/.claude/skills/run-to-fix/references/files.md index d354407d..a1077fa4 100644 --- a/.claude/skills/run-to-fix/references/files.md +++ b/.claude/skills/run-to-fix/references/files.md @@ -43,7 +43,7 @@ round that has no `result.json`. | `trial-tasks.txt` | 4 | `select --size` | the tasks of the trial run, one per line (unit ids on a job of the standard), the `{units_file}` of `run_tasks` | | `build.json` | 4 | `select --size` | the commit tried, the path `commands.build` printed and its SHA-256 | | `jobs/` | 4 | the person's run | the job folders that `run_tasks` writes | -| `trial.json` | stop 2 | `decide --stop 2` | the round's trial runs, the results of targets, guards and canaries, the money spent and left, the options | +| `trial.json` | stop 2 | `decide --stop 2` | the round's trial runs, the trials per task they hold together and whether the selection is complete, the results of targets, guards and canaries, the money spent and left, the options | | `rerun-tasks.txt`, `full-tasks.txt` | stop 2 | `decide --stop 2` | the tasks of the two options that run something | | `decision-2.md` | stop 2 | `decide --stop 2` | the build, the results, the budget left and three options with their price | | `compare-metrics.json` | 5 | `compare` | `runs.py analyze` of the run compared against and the new run, labelled by their job folders, or `before` and `new` when the two names are equal | diff --git a/.claude/skills/run-to-fix/scripts/deciding.py b/.claude/skills/run-to-fix/scripts/deciding.py index 5d11beb3..98ae25b3 100644 --- a/.claude/skills/run-to-fix/scripts/deciding.py +++ b/.claude/skills/run-to-fix/scripts/deciding.py @@ -133,19 +133,18 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: if code_ref != selection["ref"]: raise LoopError(f"a trial run of this round runs the commit select chose, {selection['ref'][:12]}; --code-ref " f"names {code_ref[:12]}") - chosen = set(selection["targets"]) | set(selection["guards"]) | set(selection["canaries"]) + chosen: List[str] = selection["targets"] + selection["guards"] + selection["canaries"] per_task = collections.Counter(r.task for r in this_run) problems = [] - outside = sorted(set(per_task) - chosen) + outside = sorted(set(per_task) - set(chosen)) if outside: problems.append("tasks outside the selection: " + ", ".join(outside)) - absent = sorted(set(selection["targets"]) - set(per_task)) + absent = sorted(set(chosen) - set(per_task)) if absent: - problems.append("targets it lacks: " + ", ".join(absent)) - uneven = sorted(t for t, n in per_task.items() if n != selection["repeat"]) - if uneven: - problems.append(f"tasks without {selection['repeat']} trials: " - + ", ".join(f"{t} ({per_task[t]})" for t in uneven)) + problems.append("selected tasks it lacks: " + ", ".join(absent)) + if len(set(per_task.values())) > 1: + problems.append("tasks with unequal numbers of trials: " + + ", ".join(f"{t} ({n})" for t, n in sorted(per_task.items()))) if problems: raise LoopError("the trial job is not a trial run of the selection: " + "; ".join(problems)) costs = [r.cost_usd for r in this_run] @@ -171,12 +170,21 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: same = [t for t in runs if t["ref"] == selection["ref"]] rows: List[TrialRow] = [r for t in same for r in read_rows(t["job_dir"], threshold)] before = read_rows(record["run"]["job_dir"], threshold) + repeat_asked = selection["repeat"] + pooled = collections.Counter(r.task for r in rows) + per_task_trials = min(pooled[t] for t in chosen) + complete = per_task_trials >= repeat_asked + held = (f"{per_task_trials} of the {repeat_asked} trials per task" if per_task_trials <= repeat_asked + else f"{per_task_trials} trials per task, over the {repeat_asked} the selection asks") targets = targets_of(selection, before, rows) + if not complete: + targets = dict(targets, p=None, verdict=None, + text=f"not tested: the selection is not complete, the trial runs hold {held}") of = lambda tasks: passes([r for r in rows if r.task in set(tasks)]) spent = trial_spent(entry) sources = [task_costs(rows), task_costs(before)] left = float(record["budget_usd"]) - spent - rerun_tasks = selection["targets"] + selection["guards"] + rerun_tasks = chosen full_tasks = sorted({r.task for r in before}) options = [{"name": "stop", "what": "nothing", "usd": 0.0, "unpriced_tasks": 0, "text": usd(0.0), "command": None}] for name, tasks, repeat in (("rerun", rerun_tasks, TARGET_TRIALS), ("full", full_tasks, 1)): @@ -191,7 +199,7 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: "units_file": units, "repeat": str(repeat), "jobs_dir": os.path.join(round_dir, "jobs"), "budget_usd": f"{max(left, 0.0):.2f}", "build": build["path"] or "", "ref": selection["ref"]}, adapter.checkout) - what = (f"the {len(tasks)} targets and guards again, {repeat} trials each" if name == "rerun" + what = (f"the {len(tasks)} selected tasks again, {repeat} trials each" if name == "rerun" else f"a full pass of the {len(tasks)} tasks of the run, {repeat} trial each") price = price_of(tasks, repeat, sources) options.append({"name": name, "what": what, "usd": price.usd, "unpriced_tasks": price.unpriced_tasks, @@ -200,7 +208,8 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: trial_record = { "schema": "run-to-fix.trial/v1", "created": now(), "job_dir": job_dir, "counts": "cost_usd is every trial run of the round; trials and the results are the trial runs on this ref", - "trial_runs": runs, "trials": len(rows), "cost_usd": spent, + "trial_runs": runs, "trials": len(rows), "repeat": repeat_asked, "trials_per_task": per_task_trials, + "complete": complete, "cost_usd": spent, "cost_complete": all(t["cost_complete"] for t in runs), "budget_usd": float(record["budget_usd"]), "budget_left_usd": left, "targets": targets, "guards": of(selection["guards"]), "canaries": of(selection["canaries"]), "options": options, @@ -208,9 +217,12 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: } write_json(os.path.join(round_dir, "trial.json"), trial_record) share = lambda x: f"{x['passed']} of {x['trials']}" if x else "none" - results = [f"| targets ({targets['tasks']}) | {share(targets['before'])} | {share(targets['after'])} | " - f"{targets['verdict'] or 'none'}, p = {targets['p']:.3g} |" if targets["p"] is not None else - "| targets | none | none | none |"] + if targets["before"] is None: + results = ["| targets | none | none | none |"] + else: + test = (f"{targets['verdict']}, p = {targets['p']:.3g}" if targets["p"] is not None + else "not tested: the selection is not complete") + results = [f"| targets ({targets['tasks']}) | {share(targets['before'])} | {share(targets['after'])} | {test} |"] for name in ("guards", "canaries"): results.append(f"| {name} ({len(selection[name])}) | | {share(trial_record[name])} | |") option_rows = [f"| {o['name']} | {o['what']} | {o['text']} | " @@ -219,6 +231,9 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: "ROUND": os.path.basename(round_dir), "LOOP": args.loop, "BUILD": f"{build['path']} from {build['ref'][:12]}" if build["path"] else f"{build['reason']} ({build['ref'][:12]})", "TRIAL_JOB": ", ".join(t["job_dir"] for t in same), "TRIALS": str(len(rows)), + "COVERAGE": (f"Together they hold {held}: the selection is complete." if complete else + f"Together they hold {held}: the selection is not complete, so the targets are not tested. " + "Start the run of trial-tasks.txt again and pass its job to decide --stop 2."), "CODE": f"{selection['ref'][:12]}, as the person states; a trial job does not record the commit it ran", "SPENT": usd(spent) + "".join(f" (the person's figure for {t['job_dir']}, whose trials record " f"{usd(t['cost_recorded_usd'])})" for t in runs if t.get("cost_from") == "the person"), From 554c9b04849811c4233d1336cb012d0992980019 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 21:59:08 +0100 Subject: [PATCH 87/92] test(run-to-fix): refuse a selected task with no unit id or with two unit ids on a job of the standard --- .../skills/run-to-fix/tests/test_decide.py | 35 ++++++++++++++++++- 1 file changed, 34 insertions(+), 1 deletion(-) diff --git a/.claude/skills/run-to-fix/tests/test_decide.py b/.claude/skills/run-to-fix/tests/test_decide.py index f317eff6..234fb734 100644 --- a/.claude/skills/run-to-fix/tests/test_decide.py +++ b/.claude/skills/run-to-fix/tests/test_decide.py @@ -4,7 +4,7 @@ import os from typing import List -from agentrepo import RepoCase, recorded_trials +from agentrepo import REPLAY, RepoCase, recorded_trials from jobs import Result, Trial, load, run, trial, write, write_job, write_json KEEP_GOOD = "Keep good. bad breaks test_other." @@ -641,6 +641,39 @@ def test_the_task_lists_hold_unit_ids(self) -> None: self.assertEqual(self.lines("full-tasks.txt"), ["f1", "f2", "f3", "p1", "p2", "p3"]) +class StandardUnitIds(Flow): + """Each selected task of a job of the standard takes the one unit id its trials give.""" + + def set_unit(self, trial_name: str, unit: object) -> None: + path = os.path.join(self.run_dir, trial_name, "result.json") + result = load(path) + result["standard"]["unit"] = unit + write_json(path, result) + + def test_a_selected_task_without_a_unit_id_is_refused(self) -> None: + self.run_dir = write_job(self.jobs, "recorded-standard", "alpha", "standard", recorded_trials()) + self.set_unit("f1__r", None) + self.kept() + r = self.select(["--size", "guards", "--budget-usd", "0.5", "--text", "targets and guards"]) + self.assertNotEqual(r.code, 0) + self.assertIn("bench/f1", r.err) + self.assertFalse(os.path.exists(self.round_file("trial-tasks.txt"))) + + def test_a_task_with_two_unit_ids_is_refused(self) -> None: + extra = trial("p2b__r", "bench/p2", 1.0) + extra.cost = 0.01 + self.run_dir = write_job(self.jobs, "recorded-standard", "alpha", "standard", recorded_trials() + [extra]) + self.set_unit("p2b__r", "p2-other") + write(os.path.join(self.stubs, "replay.py"), + REPLAY.replace("labels[n]", 'labels.get(n, {"label": "done", "touched": False})')) + job = self.chosen() + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", job, "--code-ref", self.code_ref()]) + self.assertNotEqual(r.code, 0) + for text in ("bench/p2", "p2-other"): + self.assertIn(text, r.err) + self.assertEqual(load(os.path.join(self.loop, "ledger.json"))["rounds"][0]["trial_runs"], []) + + class RecordedAmounts(Flow): """A recorded cost or budget that is not a finite amount of 0 or more stops the loop before it is used.""" From c678a232ea903e05343d163f27b0dbbce7ea2a96 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:05:59 +0100 Subject: [PATCH 88/92] fix(run-to-fix): choose the unit id of each selected task, and refuse a task with none or with two run_names gives each task the one unit id its trials give on a job of the standard. A task whose trials give no unit id while other trials of the job give one, or two unit ids, is refused before trial-tasks.txt, the ledger or a command is written. A job of a Harbor recipe keeps its task names. --- .../skills/run-to-fix/references/adapter.md | 2 +- .claude/skills/run-to-fix/scripts/deciding.py | 9 ++++--- .../skills/run-to-fix/scripts/selecting.py | 3 ++- .claude/skills/run-to-fix/scripts/trials.py | 26 ++++++++++++++----- 4 files changed, 28 insertions(+), 12 deletions(-) diff --git a/.claude/skills/run-to-fix/references/adapter.md b/.claude/skills/run-to-fix/references/adapter.md index ce848480..d9b52611 100644 --- a/.claude/skills/run-to-fix/references/adapter.md +++ b/.claude/skills/run-to-fix/references/adapter.md @@ -45,7 +45,7 @@ Every key is present, and a value the agent does not have is null. `read` refuse | `{out_json}` | `replay`, `classes` | the file the command writes; the loop reads it after a zero exit | | `{base_ref}` | `replay`, optional | the full commit id of `source.base_ref` | | `{before_ref}` | `replay`, optional | the full commit id the replay is compared with: for the replay before a fix, its own `{ref}`; for the replay of the fix, the ref before it | -| `{units_file}` | `run_tasks` | a text file with one task per line: the unit id (`standard.unit`) when every trial of the run is a trial of the standard, else the task name | +| `{units_file}` | `run_tasks` | a text file with one task per line: on a job of the standard the unit id (`standard.unit`) its trials give, else the task name. A task whose trials give no unit id, or two, on a job where other trials give one is refused | | `{repeat}` | `run_tasks` | trials per task | | `{jobs_dir}` | `run_tasks` | where the run writes its job folder | | `{budget_usd}` | `run_tasks` | the budget the person named for this run, in US dollars | diff --git a/.claude/skills/run-to-fix/scripts/deciding.py b/.claude/skills/run-to-fix/scripts/deciding.py index 98ae25b3..c45814fc 100644 --- a/.claude/skills/run-to-fix/scripts/deciding.py +++ b/.claude/skills/run-to-fix/scripts/deciding.py @@ -129,6 +129,7 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: + ", ".join(repr(a) for a in checked.agents)) job_dir = checked.dir this_run: List[TrialRow] = read_rows(job_dir, threshold) + before = read_rows(record["run"]["job_dir"], threshold) code_ref = resolve(source_of(adapter, "decide --stop 2"), args.code_ref) if code_ref != selection["ref"]: raise LoopError(f"a trial run of this round runs the commit select chose, {selection['ref'][:12]}; --code-ref " @@ -158,6 +159,9 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: raise LoopError("every trial of the trial run reports its cost; --trial-cost-usd is for a run whose trials do not") if args.trial_cost_usd is not None and args.trial_cost_usd < recorded: raise LoopError(f"--trial-cost-usd {usd(args.trial_cost_usd)} is under the recorded {usd(recorded)}") + rerun_tasks = chosen + full_tasks = sorted({r.task for r in before}) + names = {"rerun": run_names(before, rerun_tasks), "full": run_names(before, full_tasks)} runs = [t for t in entry.setdefault("trial_runs", []) if t["job_dir"] != job_dir] runs.append({"job_dir": job_dir, "ref": selection["ref"], "trials": len(this_run), "cost_usd": args.trial_cost_usd if missing else recorded, "cost_recorded_usd": recorded, @@ -169,7 +173,6 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: # every trial run of this round on the same code adds its trials to the exact test of the targets same = [t for t in runs if t["ref"] == selection["ref"]] rows: List[TrialRow] = [r for t in same for r in read_rows(t["job_dir"], threshold)] - before = read_rows(record["run"]["job_dir"], threshold) repeat_asked = selection["repeat"] pooled = collections.Counter(r.task for r in rows) per_task_trials = min(pooled[t] for t in chosen) @@ -184,13 +187,11 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: spent = trial_spent(entry) sources = [task_costs(rows), task_costs(before)] left = float(record["budget_usd"]) - spent - rerun_tasks = chosen - full_tasks = sorted({r.task for r in before}) options = [{"name": "stop", "what": "nothing", "usd": 0.0, "unpriced_tasks": 0, "text": usd(0.0), "command": None}] for name, tasks, repeat in (("rerun", rerun_tasks, TARGET_TRIALS), ("full", full_tasks, 1)): units = os.path.join(round_dir, f"{name}-tasks.txt") with open(units, "w", encoding="utf-8", newline="\n") as fh: - fh.write("".join(t + "\n" for t in run_names(before, tasks))) + fh.write("".join(t + "\n" for t in names[name])) command = None line = adapter.commands["run_tasks"] if line is not None: diff --git a/.claude/skills/run-to-fix/scripts/selecting.py b/.claude/skills/run-to-fix/scripts/selecting.py index 770464fc..3db16bb2 100644 --- a/.claude/skills/run-to-fix/scripts/selecting.py +++ b/.claude/skills/run-to-fix/scripts/selecting.py @@ -126,9 +126,10 @@ def cmd_select(args: argparse.Namespace) -> None: if chosen.usd is not None and chosen.usd > args.budget_usd: raise LoopError(f"{names[args.size]} cost {describe(chosen)}, over the budget of {usd(args.budget_usd)}") tasks = sizes[args.size] + names = run_names(rows, tasks) units = os.path.join(round_dir, "trial-tasks.txt") with open(units, "w", encoding="utf-8", newline="\n") as fh: - fh.write("".join(t + "\n" for t in run_names(rows, tasks))) + fh.write("".join(t + "\n" for t in names)) build = build_on(adapter, args.loop, round_dir, ref, args.command_timeout_s) write_json(os.path.join(round_dir, "build.json"), build) entry["selection"] = {"ref": ref, "size": args.size, "targets": targets, diff --git a/.claude/skills/run-to-fix/scripts/trials.py b/.claude/skills/run-to-fix/scripts/trials.py index c0684fca..5f7065ff 100644 --- a/.claude/skills/run-to-fix/scripts/trials.py +++ b/.claude/skills/run-to-fix/scripts/trials.py @@ -7,7 +7,7 @@ import os import re from dataclasses import dataclass -from typing import Dict, List, Optional, Tuple +from typing import Dict, List, Optional, Set, Tuple from adapter import Limits @@ -116,11 +116,25 @@ def unit_of(result: dict) -> Optional[str]: def run_names(rows: List[TrialRow], tasks: List[str]) -> List[str]: - """The names a runner takes for `tasks`: the unit ids of a job of the standard, else the task names.""" - units = {r.task: r.unit for r in rows} - if rows and all(r.unit is not None for r in rows): - return [units[t] for t in tasks] - return list(tasks) + """The names a runner takes for `tasks`: on a job of the standard the one unit id of each task, else the task + names.""" + if all(r.unit is None for r in rows): + return list(tasks) + units: Dict[str, Set[Optional[str]]] = {} + for r in rows: + units.setdefault(r.task, set()).add(r.unit) + names = [] + for task in tasks: + given = units.get(task, {None}) + ids = sorted(u for u in given if u is not None) + if len(ids) < len(given) or not ids: + raise TrialError(f"trials of the job give unit ids, and a trial of {task} gives none, so the runner cannot " + "be given its unit id") + if len(ids) > 1: + raise TrialError(f"the trials of {task} give {len(ids)} unit ids, {', '.join(ids)}, so the runner cannot " + "be given one") + names.append(ids[0]) + return names def read_trial(trial_dir: str, pass_threshold: float) -> TrialRow: From e62ce955002fe915a74b21cc72b734a271b3c835 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:07:18 +0100 Subject: [PATCH 89/92] test(run-to-fix): the message for a round of another stop rule, a partial count of output tokens, and the checkout first at stop 2 --- .claude/skills/run-to-fix/tests/test_classify.py | 10 ++++++++++ .claude/skills/run-to-fix/tests/test_compare.py | 2 ++ .claude/skills/run-to-fix/tests/test_read.py | 10 ++++++++++ 3 files changed, 22 insertions(+) diff --git a/.claude/skills/run-to-fix/tests/test_classify.py b/.claude/skills/run-to-fix/tests/test_classify.py index cf2aff6c..0a31f13d 100644 --- a/.claude/skills/run-to-fix/tests/test_classify.py +++ b/.claude/skills/run-to-fix/tests/test_classify.py @@ -496,3 +496,13 @@ def test_a_set_limit_that_no_trial_records_is_named_as_switched_off(self) -> Non self.classify_ok() self.assertIn("output_limit: no trial records the tokens of one reply", load(self.round_file("groups.json"))["rules_off"]) + + def test_a_set_limit_that_some_trials_do_not_record_says_how_many(self) -> None: + t = trial("s1__s", "made-up-bench/s1", 0.0) + t.files = {"agent/trajectory.json": STEPS_WITHOUT_USAGE} + run_dir = write_job(self.jobs, "run", "alpha", "standard", [t, trial("p__s", "made-up-bench/p", 1.0)]) + self.set_adapter(adapter("alpha", {"output_token_limit": 4096}, {})) + self.read_ok(run_dir, None) + self.classify_ok() + self.assertIn("output_limit: 1 of 2 trials record no tokens of one reply, and output_limit is not checked on them", + load(self.round_file("groups.json"))["rules_off"]) diff --git a/.claude/skills/run-to-fix/tests/test_compare.py b/.claude/skills/run-to-fix/tests/test_compare.py index df91bceb..62212505 100644 --- a/.claude/skills/run-to-fix/tests/test_compare.py +++ b/.claude/skills/run-to-fix/tests/test_compare.py @@ -351,4 +351,6 @@ def test_compare_refuses_a_stop_rule_other_than_the_one_read_recorded(self) -> N r = run(["compare", "--loop", self.loop, "--new", new_dir, "--against", "run"]) self.assertNotEqual(r.code, 0) self.assertIn("stop rule", r.err) + self.assertIn("keep this loop folder as it is and start a new loop folder", r.err) + self.assertNotIn("loop.py read", r.err) self.assertFalse(os.path.exists(self.round_file("result.json"))) diff --git a/.claude/skills/run-to-fix/tests/test_read.py b/.claude/skills/run-to-fix/tests/test_read.py index 4e435d5a..5ec7e93d 100644 --- a/.claude/skills/run-to-fix/tests/test_read.py +++ b/.claude/skills/run-to-fix/tests/test_read.py @@ -318,6 +318,16 @@ def test_a_candidate_needs_the_checkout(self) -> None: self.assertNotEqual(r.code, 0) self.assertIn("source", r.err) + def test_stop_two_names_the_missing_checkout_before_it_reads_the_trial_job(self) -> None: + run_dir = healthy(self.jobs, "run", "alpha") + self.set_adapter(self.adapter_without_source({})) + self.read_ok(run_dir, None) + self.classify_ok() + r = run(["decide", "--loop", self.loop, "--stop", "2", "--trial-job", os.path.join(self.jobs, "no-such-job"), + "--code-ref", "HEAD"]) + self.assertNotEqual(r.code, 0) + self.assertIn("decide --stop 2 works on the agent's code, and adapter.json has no source", r.err) + def test_a_command_on_the_code_needs_the_checkout(self) -> None: run_dir = healthy(self.jobs, "run", "alpha") data = self.adapter_without_source({"test": "pytest {worktree}"}) From 8c15f1d8162715ca49240f6ed5227c3c9a476315 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:08:19 +0100 Subject: [PATCH 90/92] fix(run-to-fix): tell the person to keep the loop folder and start a new one for a round of another stop rule --- .claude/skills/run-to-fix/SKILL.md | 2 +- .claude/skills/run-to-fix/scripts/comparing.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.claude/skills/run-to-fix/SKILL.md b/.claude/skills/run-to-fix/SKILL.md index 216c04c0..d5a8b01f 100644 --- a/.claude/skills/run-to-fix/SKILL.md +++ b/.claude/skills/run-to-fix/SKILL.md @@ -159,7 +159,7 @@ writes `result.json`: the pass rates with their standard error; the tasks both r the targets' trials with Fisher's exact test; each prediction of stage 3 beside its result; and the stop rule. When some trials of the new run report no cost, the budget part of the stop rule is not checked, unless the person gives the run's cost with `--new-cost-usd`. `read` records the text of the stop rule, and `compare` refuses a round whose recorded -rule is not the rule of the skill it runs. It +rule is not the rule of the skill it runs: keep that loop folder as it is and start a new one. It closes the round and writes `report.html` with `render` of `run-analysis`; fill its prose slots as that skill describes (`runs.py render --slots`, then `runs.py check`). diff --git a/.claude/skills/run-to-fix/scripts/comparing.py b/.claude/skills/run-to-fix/scripts/comparing.py index e352dc43..ed090206 100644 --- a/.claude/skills/run-to-fix/scripts/comparing.py +++ b/.claude/skills/run-to-fix/scripts/comparing.py @@ -123,7 +123,7 @@ def cmd_compare(args: argparse.Namespace) -> None: record = read_input(round_dir) if record.get("stop_rule") != STOP_RULE: raise LoopError("the stop rule of this round, recorded in input.json when read started it, is not the stop " - "rule of this version of the skill; start a new round with loop.py read") + "rule of this version of the skill; keep this loop folder as it is and start a new loop folder") threshold = float(record["pass_threshold"]) against = record["run"] if args.against == "run" else record["baseline"] if against is None: From 9d4ab97f8433a487e9dda4433c9cf8931baf5110 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:08:19 +0100 Subject: [PATCH 91/92] fix(run-to-fix): count the trials that record no tokens of a reply, check the checkout first at stop 2, describe model_calls rules_off says how many trials record no tokens of one reply when the adapter sets an output limit. decide --stop 2 checks for the agent's checkout before it reads the trial job. classes.md and files.md say that model_calls in trials.csv is the standard's own count where a result holds one, and that the run-analysis page counts from the trajectory. --- .claude/skills/run-to-fix/references/classes.md | 12 +++++++----- .claude/skills/run-to-fix/references/files.md | 6 +++--- .claude/skills/run-to-fix/scripts/deciding.py | 3 ++- .claude/skills/run-to-fix/scripts/trials.py | 11 ++++++++--- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/.claude/skills/run-to-fix/references/classes.md b/.claude/skills/run-to-fix/references/classes.md index 4b79c4fb..583404f7 100644 --- a/.claude/skills/run-to-fix/references/classes.md +++ b/.claude/skills/run-to-fix/references/classes.md @@ -20,13 +20,15 @@ The first rule that holds gives the class. | `verifier_failed` | the verifier ran and the reward is under the threshold | never | With every limit null, each trial is `pass`, `time_limit`, `exception`, `no_score` or `verifier_failed`. -`groups.json` lists the rules a null limit switched off under `rules_off`, and `output_limit` with its reason when the -adapter sets the limit and no trial records the tokens of one reply, as on a job of the standard. +`groups.json` lists the rules a null limit switched off under `rules_off`. When the adapter sets +`limits.output_token_limit`, it also lists `output_limit` with the count of trials that record no tokens of one reply, +as on a job of the standard; `output_limit` is not checked on those trials. The primary reward follows the rule of `run-analysis`: `verifier_result.rewards.reward`, else the first numeric -reward. Model calls are `agent_result.n_model_calls` when `result.json` holds it, as a job of the standard does; else -they follow `run-analysis`: the per-call usage records (`extra.raw_usage` or steps with `metrics`, whichever is more), -else the agent's steps. The largest reply is the +reward. `model_calls` in `trials.csv` is the standard's own count, `agent_result.n_model_calls`, when `result.json` holds +it; else it follows `run-analysis`: the per-call usage records (`extra.raw_usage` or steps with `metrics`, whichever is +more), else the agent's steps. The page of `run-analysis` always counts from the trajectory, so on a job of the +standard its count of model calls can differ from `trials.csv`. The largest reply is the largest `metrics.completion_tokens` of a step or `output_tokens` of a usage record. `classify` stops when its count of trials or passes, or of the pairs with the baseline, differs from the count in `metrics.json`. diff --git a/.claude/skills/run-to-fix/references/files.md b/.claude/skills/run-to-fix/references/files.md index a1077fa4..488dbea4 100644 --- a/.claude/skills/run-to-fix/references/files.md +++ b/.claude/skills/run-to-fix/references/files.md @@ -29,7 +29,7 @@ round that has no `result.json`. | `input.json` | 1 | `read` | the run and baseline job folders, the adapter's SHA-256, the budget, the pass threshold, the `runs.py` used, the text of the stop rule; `refused` with its reason when the run is not analysable | | `health.txt` | 1 | `read` | `runs.py health` of the run and the baseline | | `metrics.json` | 1 | `read` | `runs.py analyze` of the baseline and the run, labels `baseline` and `run` | -| `trials.csv` | 2 | `classify` | one row per trial: `trial`, `task`, `class`, `signature`, `reward`, `pair`, `cost_usd`, `model_calls`, `agent_s`, `max_completion_tokens`, `exception_type`, then `found:`, `agent_class:`, `agent_flag:` | +| `trials.csv` | 2 | `classify` | one row per trial: `trial`, `task`, `class`, `signature`, `reward`, `pair`, `cost_usd`, `model_calls`, `agent_s`, `max_completion_tokens`, `exception_type`, then `found:`, `agent_class:`, `agent_flag:`. `model_calls` is the standard's own count where `result.json` holds one; the page of `run-analysis` counts from the trajectory | | `groups.json` | 2 | `classify` | class counts, the rules switched off, the groups with their warnings, the counts of found and agent classes | | `agent-classes.json` | 2 | `commands.classes` | the agent's own classes and flags per trial | | `causes/.md` | 2 | the assistant | one cause, below | @@ -110,8 +110,8 @@ of `trial.json` and `cheapest_option` of `result.json` hold such prices. | file | used for | |---|---| -| `/result.json` | task, trial name, reward, exception, agent phase, cost, agent name | -| `/agent/trajectory.json` | model calls and the largest reply | +| `/result.json` | task, trial name, reward, exception, agent phase, cost, agent name, unit id, and the standard's own count of model calls (`agent_result.n_model_calls`) where it holds one | +| `/agent/trajectory.json` | model calls where `result.json` holds no count of them, and the largest reply; the page of `run-analysis` counts model calls from it always | | `/verifier/ctrf.json`, `test-stdout.txt`, `score.json` | the detail of a `verifier_failed` signature | | any file a found class names | the found class's column | | everything `runs.py` reads | `health`, `analyze` and `render` (`run-analysis/references/files.md`) | diff --git a/.claude/skills/run-to-fix/scripts/deciding.py b/.claude/skills/run-to-fix/scripts/deciding.py index c45814fc..5b2ad9c4 100644 --- a/.claude/skills/run-to-fix/scripts/deciding.py +++ b/.claude/skills/run-to-fix/scripts/deciding.py @@ -115,6 +115,7 @@ def decide_one(args: argparse.Namespace, round_dir: str, record: dict) -> None: def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: adapter = adapter_of(args.loop) + checkout = source_of(adapter, "decide --stop 2") ledger = load_ledger(args.loop) entry = ledger_round(ledger, os.path.basename(round_dir)) selection = entry.get("selection") @@ -130,7 +131,7 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: job_dir = checked.dir this_run: List[TrialRow] = read_rows(job_dir, threshold) before = read_rows(record["run"]["job_dir"], threshold) - code_ref = resolve(source_of(adapter, "decide --stop 2"), args.code_ref) + code_ref = resolve(checkout, args.code_ref) if code_ref != selection["ref"]: raise LoopError(f"a trial run of this round runs the commit select chose, {selection['ref'][:12]}; --code-ref " f"names {code_ref[:12]}") diff --git a/.claude/skills/run-to-fix/scripts/trials.py b/.claude/skills/run-to-fix/scripts/trials.py index 5f7065ff..345b977f 100644 --- a/.claude/skills/run-to-fix/scripts/trials.py +++ b/.claude/skills/run-to-fix/scripts/trials.py @@ -185,7 +185,7 @@ def classify(row: TrialRow, limits: Limits) -> str: def rules_off(limits: Limits, rows: List[TrialRow]) -> List[str]: - """The rules a null limit switches off, and output_limit when no trial records the tokens of one reply.""" + """The rules a null limit switches off, and output_limit on the trials that record no tokens of one reply.""" off = [] if limits.time_limit_s is None: off.append("time_limit by the agent phase") @@ -193,8 +193,13 @@ def rules_off(limits: Limits, rows: List[TrialRow]) -> List[str]: off.append("step_limit") if limits.output_token_limit is None: off.append("output_limit") - elif all(r.max_completion_tokens is None for r in rows): - off.append("output_limit: no trial records the tokens of one reply") + else: + unread = sum(r.max_completion_tokens is None for r in rows) + if rows and unread == len(rows): + off.append("output_limit: no trial records the tokens of one reply") + elif unread: + off.append(f"output_limit: {unread} of {len(rows)} trials record no tokens of one reply, and output_limit " + "is not checked on them") return off From 75d16cd3cf0d4a9d6125468bce98f79a91daa5c5 Mon Sep 17 00:00:00 2001 From: Zakhar Tsvirka <186512928+opPO333@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:21:34 +0100 Subject: [PATCH 92/92] docs(run-to-fix): state the coverage and unit id messages and the model_calls note without a trailing so --- .claude/skills/run-to-fix/references/classes.md | 4 ++-- .claude/skills/run-to-fix/scripts/deciding.py | 2 +- .claude/skills/run-to-fix/scripts/trials.py | 8 ++++---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.claude/skills/run-to-fix/references/classes.md b/.claude/skills/run-to-fix/references/classes.md index 583404f7..5d8a6cf0 100644 --- a/.claude/skills/run-to-fix/references/classes.md +++ b/.claude/skills/run-to-fix/references/classes.md @@ -27,8 +27,8 @@ as on a job of the standard; `output_limit` is not checked on those trials. The primary reward follows the rule of `run-analysis`: `verifier_result.rewards.reward`, else the first numeric reward. `model_calls` in `trials.csv` is the standard's own count, `agent_result.n_model_calls`, when `result.json` holds it; else it follows `run-analysis`: the per-call usage records (`extra.raw_usage` or steps with `metrics`, whichever is -more), else the agent's steps. The page of `run-analysis` always counts from the trajectory, so on a job of the -standard its count of model calls can differ from `trials.csv`. The largest reply is the +more), else the agent's steps. On a job of the standard, the count of model calls on the page of +`run-analysis` can differ from `trials.csv`: that page always counts from the trajectory. The largest reply is the largest `metrics.completion_tokens` of a step or `output_tokens` of a usage record. `classify` stops when its count of trials or passes, or of the pairs with the baseline, differs from the count in `metrics.json`. diff --git a/.claude/skills/run-to-fix/scripts/deciding.py b/.claude/skills/run-to-fix/scripts/deciding.py index 5b2ad9c4..35c17573 100644 --- a/.claude/skills/run-to-fix/scripts/deciding.py +++ b/.claude/skills/run-to-fix/scripts/deciding.py @@ -234,7 +234,7 @@ def decide_two(args: argparse.Namespace, round_dir: str, record: dict) -> None: "BUILD": f"{build['path']} from {build['ref'][:12]}" if build["path"] else f"{build['reason']} ({build['ref'][:12]})", "TRIAL_JOB": ", ".join(t["job_dir"] for t in same), "TRIALS": str(len(rows)), "COVERAGE": (f"Together they hold {held}: the selection is complete." if complete else - f"Together they hold {held}: the selection is not complete, so the targets are not tested. " + f"Together they hold {held}: the selection is not complete. The targets are tested once it is. " "Start the run of trial-tasks.txt again and pass its job to decide --stop 2."), "CODE": f"{selection['ref'][:12]}, as the person states; a trial job does not record the commit it ran", "SPENT": usd(spent) + "".join(f" (the person's figure for {t['job_dir']}, whose trials record " diff --git a/.claude/skills/run-to-fix/scripts/trials.py b/.claude/skills/run-to-fix/scripts/trials.py index 345b977f..cbd242a9 100644 --- a/.claude/skills/run-to-fix/scripts/trials.py +++ b/.claude/skills/run-to-fix/scripts/trials.py @@ -128,11 +128,11 @@ def run_names(rows: List[TrialRow], tasks: List[str]) -> List[str]: given = units.get(task, {None}) ids = sorted(u for u in given if u is not None) if len(ids) < len(given) or not ids: - raise TrialError(f"trials of the job give unit ids, and a trial of {task} gives none, so the runner cannot " - "be given its unit id") + raise TrialError(f"a trial of {task} gives no unit id while other trials of the job give one; the runner " + "takes one unit id per task") if len(ids) > 1: - raise TrialError(f"the trials of {task} give {len(ids)} unit ids, {', '.join(ids)}, so the runner cannot " - "be given one") + raise TrialError(f"the trials of {task} give {len(ids)} unit ids, {', '.join(ids)}; the runner takes one " + "unit id per task") names.append(ids[0]) return names