diff --git a/.claude/commands/coder-eval-review.md b/.claude/commands/coder-eval-review.md index 7f53b0cd..d778b1f8 100644 --- a/.claude/commands/coder-eval-review.md +++ b/.claude/commands/coder-eval-review.md @@ -24,7 +24,8 @@ The run layout (`runs/////…`, `` a zero-p 1. Read `/run.json` if present (for context — `run_id`, `start_time`). 2. Glob `/*/*/*/task.json` and read each one. 3. Read `/analysis.md` if present — it already diagnoses many failures; lean on its findings rather than re-deriving them. -4. A task counts as **failed** if `final_status != "SUCCESS"` **or** `weighted_score < 0.9`. Skip passing tasks for now (we may extend to passing tasks later — the schema supports it). +4. Skip any task whose `final_status` is `"NOT_GRADED"` — `coder-eval execute` produced it, no criterion ran, and `weighted_score` is `null`. It is neither a pass nor a failure, and comparing `null < 0.9` would book every ungraded row as a failure to review. +5. Of the rest, a task counts as **failed** if `final_status != "SUCCESS"` **or** `weighted_score < 0.9`. Skip passing tasks for now (we may extend to passing tasks later — the schema supports it). If no `task.json` files exist, write an empty `review_index.json` (`{"reviews": []}`) and exit. diff --git a/.claude/shared/run-layout.md b/.claude/shared/run-layout.md index 57edc4a6..89f2bd10 100644 --- a/.claude/shared/run-layout.md +++ b/.claude/shared/run-layout.md @@ -12,6 +12,7 @@ runs/////{task.json, task.log, artifacts/} - `` — zero-padded replicate index (e.g. `00`, `01`). - `task.json` — the persisted per-replicate result (the consumer contract; carries the large `iterations` array — still accepted under its former name `turns` when reading, but not what current runs write). - `task.json.malformed` — present only on the docker degrade path: when an existing `task.json` fails to parse (schema skew from a stale `:latest` image, or a truncated/torn write), the docker runner moves the unparseable original aside to this sidecar and writes a synthetic `final_status=ERROR` `task.json` in its place. Diagnostic-only; `rglob("task.json")` consumers do not match it. +- `task.execute.json` — present only after a DETACHED grade (`coder-eval evaluate ` or `coder-eval run --resume` over a `NOT_GRADED` row). The pre-grade snapshot of `task.json`, written once and never overwritten by a later grade, so "this run was executed separately from grading" stays auditable. Diagnostic-only; `rglob("task.json")` consumers do not match it. - `task.log` — the human-readable task log; `artifacts/` — files the agent produced. **Scope-marker files** (used to detect what a given path represents): diff --git a/.github/workflows/verify-published-action.yml b/.github/workflows/verify-published-action.yml index 134dbeb1..914cbdc0 100644 --- a/.github/workflows/verify-published-action.yml +++ b/.github/workflows/verify-published-action.yml @@ -541,6 +541,18 @@ jobs: "uploaded run dir before re-running; this is an unattended paid job.") sys.exit(1) + # NOT_GRADED means a task ran but was never scored. This job invokes the + # published action, which runs `coder-eval run` (graded), so reaching it is + # impossible unless the action started dispatching `coder-eval execute` -- + # in which case every score gate below silently measures nothing and the job + # goes green having verified no verdict at all. Hard-fail, don't tolerate. + ungraded = [s for s in statuses if s == "NOT_GRADED"] + if ungraded: + print("::error::task(s) reported NOT_GRADED -- the published action ran without " + "grading. `coder-eval run` always grades, so the action is dispatching the " + "wrong command and every score gate in this job is measuring nothing.") + sys.exit(1) + # Exit-contract check, conditional on the model having actually performed. # Ignoring the step's exit code entirely (see the continue-on-error rationale # above) would also hide a REGRESSION in the action's own exit logic -- e.g. a diff --git a/CLAUDE.md b/CLAUDE.md index ac00d5ef..aba27c48 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,6 +27,13 @@ coder_eval/ ├── fs_permissions.py # set_permissions: stacked chmod window (via Sandbox.set_permissions) ├── pricing.py # Model pricing / cost calculation (ModelPricing, calculate_cost, register_pricing) ├── litellm_cost.py # Join proxy-captured ACTUAL per-call cost/cache onto turns (LiteLLM backend; apply_actual_cost) +├── reports_html.py # Single-file HTML report (the evalboard's static twin) +├── reports_stats.py # Shared report statistics + the ungraded rendering helpers (format_score, is_env_table_key) +├── formatting.py # Human-readable number/duration formatting shared by the renderers +├── invocation_log.py # JSON Lines CLI-invocation log the `cli_called` criterion reads +├── telemetry.py # App Insights / OpenTelemetry emission (CoderEval.Task.End et al.) +├── isolation/ # driver: docker — docker_runner.py builds, runs and reaps one container per task +├── optimize/ # Prompt/config optimization helpers ├── utils.py # Version info helpers │ ├── agents/ @@ -80,18 +87,24 @@ coder_eval/ │ └── timeout.py # Timeout handling (TurnTimeoutError carries optional partial TurnRecord) │ ├── orchestration/ # Batch execution utilities -│ ├── batch.py # Parallel task execution (run_batch + run_batch_resolved) +│ ├── batch.py # Parallel task execution (run_batch) + partition_for_resume/ResumePartition │ ├── config.py # Batch run configuration │ ├── early_stop.py # validate_early_stop guardrails + EarlyStopWatcher (armed live-verdict observer) │ ├── evaluation.py # Reference dir resolution + per-run private staging +│ ├── regrade.py # Grade an already-executed run in place — shared by `evaluate ` and `run --resume` │ ├── experiment.py # ExperimentRunner, resolve_task_for_variant, load_experiment │ └── task_loader.py # YAML task loading │ ├── cli/ # CLI commands (Typer + Rich) │ ├── __init__.py # Typer app setup (core commands) -│ ├── run_command.py # `coder-eval run` +│ ├── run_command.py # `coder-eval run` + `run_pipeline` (the body BOTH run and execute share) +│ ├── execute_command.py # `coder-eval execute` — Typer signature only; delegates to run_pipeline(grade=False) │ ├── plan_command.py # `coder-eval plan` +│ ├── evaluate_command.py # `coder-eval evaluate` (grade a dir, or re-grade a run dir) + `run_evaluation` +│ ├── evaluate_target.py # PURE shape detection for evaluate's positionals (run dir ⟺ holds task.json) │ ├── report_command.py # `coder-eval report` +│ ├── aggregate_command.py # `coder-eval aggregate` — rebuild run.json from the task.json rows on disk (the step right after `execute`) +│ ├── run_task_internal_command.py # `_run-task-internal` — the in-container entry point DockerRunner invokes; never called by a user │ ├── run_helpers.py # CLI helper functions │ ├── console.py # Rich console instance │ └── utils.py # CLI utilities @@ -146,6 +159,9 @@ action.yml # Published composite GitHub Action (coder-ev - **Reference solutions are directory-only, and shielded (partially) from the agent**: `task.reference` is a single required `directory:` (relative to the task YAML) — the inline `code:` / single-file `file:` forms are gone, because a directory is the only shape that can be permission-gated as a unit; a `model_validator(mode="before")` gives the removed forms a migration error. The orchestrator stages a **per-run private copy** (`orchestration/evaluation.py::stage_reference_dir`, symlinks stripped) into a tempdir, removed in `_cleanup` via `path_utils.rmtree_restrictive` (keyed on `_reference_staging_root`, recorded BEFORE the copy so a failed copy still cleans up; `rmtree(ignore_errors=True)` silently declines on a tree left at 000) and deliberately never preserved into `run_dir/artifacts`. That copy is held at mode `000` for the whole of every `agent.communicate` call via **`Sandbox.set_permissions`**, the driver-aware wrapper over `fs_permissions.py::set_permissions`. Windows **stack**: exiting restores the *enclosing* window's mode, only the outermost exit restores the pre-window mode — that is what makes a mid-turn re-grant (`mode=READ_ONLY_MODE`) expressible, and it covers two windows at the same mode so no refcount is needed. The window is enforced **only inside a docker container** (`Sandbox.enforces_permission_windows`) and is a no-op on the host, where the agent shares our uid. **That gate keys on the `CODER_EVAL_IN_CONTAINER` env var, NOT `sandbox.driver`** — `run_task_internal_command` rewrites `driver: docker` → `tempdir` before building the in-container Orchestrator, so a driver-based gate would silently disable the anti-cheat on exactly the path that needs it (regression-guarded by `TestSandboxDriverGate`); `resolve_reference_dir` gates its `/work/references` branch on the same var for the same reason. The task directory is **not** shielded (`:ro` mount → EROFS, and the same YAML is readable at `/work/input`). Criteria address reference files with the `$REFERENCE_DIR` token (same resolver as `$TASK_DIR`) and the `REFERENCE_DIR` env var for `run_command`; `reference_comparison` names one file via `reference_file`. Docker mounts a throwaway **read-write** copy at `/work/references` (a `:ro` mount cannot be chmod'd — EROFS), masks the in-task-dir original with an empty tmpfs, and drops `DAC_OVERRIDE`/`DAC_READ_SEARCH`. `FOWNER`/`CHOWN` are deliberately **NOT** dropped: the in-container orchestrator that applies the window is the same root process with the same caps, so dropping `FOWNER` breaks *the harness's own* chmod wherever the bind mount preserves a non-root owner (native Linux — verified: `chmod: Operation not permitted`), i.e. exactly where the drop would otherwise bite. A window that cannot be applied is now a hard error, not a warning: `Sandbox.set_permissions` passes `strict=True` whenever it enforces, so an unprotected run fails instead of producing a normal-looking score. **KNOWN GAP — this is defense-in-depth, not a boundary**: (a) `chmod(2)` is gated on owner-or-`CAP_FOWNER` and the container runs as root owning the copy, so a deliberate `chmod 755 /work/references` restores access; (b) the window spans `agent.communicate` only, and nothing reaps agent child processes at turn end, so a backgrounded read loop succeeds once the window closes. The **write** half of (b) is closed — `path_utils.digest_tree` hashes the tree at staging and `Orchestrator._verify_reference_integrity` re-checks before grading, raising `ReferenceTamperedError` (→ `FinalStatus.ERROR`) on a mismatch so an agent cannot overwrite the reference to drive `reference_comparison` to 1.0. Passive reads are blocked; an adversarial agent is not. Full containment requires running the agent as a non-root uid AND holding the window for the agent's whole lifetime — follow-up. `tasks/anti_cheat_reference` probes the passive-read half. - **Harness run-limit parity**: a shared `BaseAgentConfig` field must mean the same thing on every backend, so a divergence is either fixed or documented — never silent. **`run_limits.max_turns` on Codex/Antigravity counts VISIBLE turns** (resolved tool calls, read live off the shared `EventCollector.visible_turn_count`, the same list `TurnRecord.commands` holds) because one `communicate()` is a single SDK turn on both, so a native counter would clamp at 1; claude-code keeps its native SDK cap, whose unit (an agent-loop turn) absorbs arbitrarily many parallel calls — the same number is NOT the same budget across harnesses. OpenCode likewise keeps a native unit — the CLI streams a real multi-step loop per `communicate()` (`step_start`/`step_finish`), so `max_turns: N` allows N complete steps and cuts cleanly when step N+1 begins. The cap is enforced on the same loop boundary as the cooperative early stop and finalizes cleanly as `max_turns_exhausted` (no crash, no retry); on Antigravity that boundary lives in `_drain()`, so the background-work poll loop honors it too. Known unfixed divergences: `permission_mode` on Codex and Antigravity (both run unconfined — the sandbox driver is the isolation boundary), `disallowed_tools` on Codex (forwarded, not SDK-enforced), `allowed_tools`/`disallowed_tools` on Antigravity (not read at all), `turn_timeout` on Antigravity (bounded by an earlier internal poll deadline at 80% of it), `allowed_tools`/`disallowed_tools`/`system_prompt` on OpenCode (no CLI knob; warned at `start()`, not enforced), and **`agent.plugins[].path` depth** — claude-code REQUIRES a plugin root holding `skills/` and silently loads NOTHING from a bare skills directory, while Codex and Antigravity scan both depths and accept either. That is the costly direction: the wrong depth produces no error, every positive row of an activation suite scores 0, and the suite reports recall 0.0, which reads exactly like a skill that never triggers. Held to the plugin-root shape (for `SKILL_SOURCE_PATH` only) by lint rule CE045. `plugins` on OpenCode is **honored for skills**: each local plugin root is mapped to the `skills` dir its `.claude-plugin/plugin.json` declares (default `/skills`, never the root itself — `skills.paths` is scanned recursively and a plugin root can hold a self-referential symlink) and injected via `OPENCODE_CONFIG_CONTENT`, which `--pure` does not suppress; a plugin's agents/hooks/commands/MCP servers are still dropped. Full table + rationale: docs/agents/HARNESS_PARITY.md. - **sandbox isolation**: Tasks that don't need MCP servers should set `setting_sources: []` in their `agent:` block to isolate the sandbox from the host project's CLAUDE.md and settings. Without this, the host project's CLAUDE.md (often 20 KB+) is injected into every API call, inflating cache-creation tokens and cost significantly. +- **Execute vs. run (the grading switch)**: `coder-eval execute` is `coder-eval run` with grading removed — the agent runs and the full trajectory is captured, but no criterion is checked, `weighted_score` is `None` (never `0.0`, which would be indistinguishable from "graded and scored zero"), and the row finalizes as **`FinalStatus.NOT_GRADED`**, whose `category` is a **fourth** bucket, `"ungraded"`. Ungraded rows leave BOTH sides of every rate: `RunSummary.pass_rate` / `error_share` and `VariantAggregate.pass_rate` divide by `tasks_graded` (`tasks_run - tasks_not_graded`), and `tasks_not_graded` is part of the sum-to-`tasks_run` invariant, not a `tasks_failed` sub-counter. **Only SUCCESS/FAILURE collapse into it** — `ERROR`, `TIMEOUT`, `BUILD_FAILED`, `MAX_TURNS_EXHAUSTED` and the budget stops are facts about the *run*, not about grading, and still apply (so `execute` still exits non-zero on a crash). The switch is `BatchRunConfig.grade` → `Orchestrator(grade=...)` → the **four** grading call sites (single-shot, evaluate-only, the simulation dialog check, and post-failure diagnostics); it crosses the docker boundary in `context.json` (defaulting to `True` in-container, so a host predating `execute` keeps grading). It is **deliberately not a task-config field** — no 5-layer merge, no `-D` path — because a task YAML must never declare itself ungraded; only the invoking command decides. `run` and `execute` share one body (`run_command.run_pipeline`) and differ solely in that flag, so there is no third code path. Three things are refused rather than degraded: `--junit-xml` (a report of verdicts, and there are none — though `reports_junit` still emits `` for an ungraded row it encounters), `--allow-host-grading` (it decides how an ungraded row is GRADED, and `execute` grades nothing), and simulation tasks (their turn-continuation logic reads criteria results, so an ungraded dialog would silently change its own stopping behavior). `stop_early:` blocks are inert under `execute` for the same reason the kill switch exists: the full trajectory is the deliverable. Motivating consumer: an external harness (Harbor / Terminal-Bench 2.0) that builds its own container, calls coder-eval as the agent, and grades with its own tests. +- **Detached grading (`evaluate` over a run dir) + `Sandbox.adopt`**: `coder-eval evaluate` takes two shapes, told apart by a **pure** resolver (`cli/evaluate_target.py`) on one probe — a target holding `task.json` is a run directory. Run-dir mode rebuilds the task from the run's own `task_config.resolved`, **not** by re-loading the YAML: `resolved` is post-merge, so variant overrides / `-D` / dataset expansion are already baked in and re-loading the source would silently grade a *different* task (fallback to `source_file` only when `resolved` no longer validates, and loudly). It seeds the fresh result from the prior one via `Orchestrator(prior_result=...)` → `_seed_from_prior_result`, which carries the trajectory (every derived figure — tokens, cost, `command_stats`, `model_used` — recomputes from `iterations`), `iteration_count`, execution facts, and **`early_stop` — load-bearing, because gate selection is FIRED-ONLY**: dropping it re-grades a truncated trajectory under the full-run strict-AND gate and can flip the verdict. Carrying it is only half the fix — **both** grading paths select the gate through the single `Orchestrator._select_gate()`; the evaluate-only branch a detached grade actually takes originally called `all_criteria_passed` inline, so the seeded field was written and never read. `tests/test_seed_from_prior_result.py` partitions every `EvaluationResult` field as CARRIED or RECOMPUTED and fails closed on a new one, and asserts the two `_select_gate()` call sites. A prior status that `FinalStatus.is_execution_fact` (TIMEOUT / ERROR / BUILD_FAILED / the budget stops) is **preserved**, never overwritten: grading may only move `NOT_GRADED` to SUCCESS/FAILURE, since it neither repeated nor observed the agent phase. **`MAX_TURNS_EXHAUSTED` is deliberately NOT one of them under `execute`**: `_terminal_status` puts the `grade=False` arm ABOVE it, because on the graded path it is subordinate to the verdict — `run` returns SUCCESS for a max-turns trajectory whose criteria pass — so it is not knowable without grading. Consuming it first made it terminal AND permanent (the `is_execution_fact` arm then pinned it), so identical agent output scored SUCCESS/1.0 under `run` and MAX_TURNS_EXHAUSTED under `execute` → `evaluate`. The fact survives on `result.max_turns_exhausted`, which `_seed_from_prior_result` carries, so the detached grade walks the identical chain. The CLI must also branch on WHERE a status came from, not on its value: a preserved TIMEOUT exited 0 under "All criteria passed" (a CI wrapper reading the exit code went green on a row run.json counts as failed), and a preserved ERROR printed the ORIGINAL run's crash message as though grading had crashed, claimed the row was "left ungraded" (false — the restored record still read ERROR), and discarded a verdict just computed at 1.000. Grader-host `environment_info` is preserved as flat `graded_by_*` scalars rather than overwriting the run's (flat, not a nested sub-dict: `environment_info` is rendered as a flat map by the HTML report and typed as one by the evalboard, so a nested capture prints as a Python dict repr). The route recorder follows the same rule: on a detached grade it writes `graded_by_api_routing` / `graded_by_eval_routing` and leaves the run's `api_routing` alone — writing in place contradicted the "prior wins" contract and left a self-contradictory record (a direct route named beside the run's stale `aws_region`/`bedrock_model`). Two other parity fixes: `command_base_path` is now persisted into `environment_info` by `_sync_sandbox_command_path_with_agent` and restored in the evaluate-only branch (closing the PATH gap that method's docstring already named), and `_join_litellm_actual_cost` **skips** when `prior_result` is set (its join keys on a per-Orchestrator nonce the prior turns never carried, so it would clobber already-correct costs). The verdict is written back into the run's `task.json`, with the pre-grade record kept as `task.execute.json` — that in-place write is what makes plain `coder-eval aggregate ` rebuild a graded `run.json` with **zero** new code. **`Sandbox.adopt(workspace)`** is the grade-in-place primitive: it reuses `setup`'s adoption half but skips every *materializing* step (`_setup_template`, `_generate_cli_recorders`, venv/package installs, the destructive `$HOME` remediation), running only non-mutating derivation (mock-dir `+x`, venv *discovery*, plugin-tools pin); `_cleanup_on_exit` stays False so an adopted tree is never moved or deleted, and `Sandbox.was_adopted` is set — the Orchestrator reads it to SKIP the `pre_run`/`post_run` hooks (`run()` calls them unconditionally with `cwd = sandbox_dir`, and several in-tree tasks stage fixtures there with `cp -a /app/[!.]* "$PWD/"`, which would overwrite the agent's deliverables before the criteria read them) and to KEEP `sandbox_path` in the `PreservationMode.NONE` cleanup arm (an adopted tree survives cleanup, so the path is not stale). The hooks' recorded results are carried from the prior run instead. In-place is **more correct**, not merely faster: `_setup_template` filters the copy through `_should_ignore_template_file`, which drops `node_modules` / `dist` / `build` / `.venv` / `.git`, so on the copy path a criterion like `test -f dist/bundle.js` fails as a *copying artifact* rather than as a verdict (verified: 0.00 "does not exist" on copy vs 1.00 in place). Defaults: in-place for a run dir, copy for a bare work dir (criteria can mutate it and it is the user's own tree); `--in-place`/`--copy` override. `adopt` hard-errors on `driver: docker` (a container workspace is unreachable from the host), and grading a `driver: docker` task is REFUSED outright unless `--allow-host-grading` is passed — the earlier behavior silently rewrote the driver to `tempdir`, which ran a container task's criteria against a host filesystem lacking `/verifier` and the image's toolchain (FAILURE for a trajectory `run` scored 1.0, plus `rm -rf /verifier` unsandboxed on the grading machine) and neutralized `adopt`'s own docker guard; an opted-in row is stamped `graded_on_host` so it is never silently comparable with a container-graded one (lint rule CE051). A re-grade refuses on a `reference_digest` mismatch — the digest is persisted into `environment_info` at staging time by `_stage_reference` (it shipped once as a read with no writer anywhere, so the guard was dead code), and `verify_reference_unchanged` now takes the task file it resolves against and RAISES on a vanished or unresolvable reference instead of returning silently. That comparison digests a STAGED copy of the source, not the raw tree: the recorded digest is taken over the staged copy, which `stage_reference_dir` filters through `REFERENCE_COPY_IGNORE` (`.git`), so digesting the source directly compared two differently-filtered trees and reported a permanent false mismatch for any reference that is a git checkout — exactly the case the ignore list exists for. **The recorded config is untrusted input**: `evaluate ` rebuilds the task from a shareable artifact, so a rebuilt config that carries shell (`run_command` criteria, `agent_judge`, `llm_judge`, `uipath_eval`, and — only on the `--copy` path, since the in-place path skips them — `pre_run`/`post_run` **and the sandbox's own provisioning**) is REFUSED unless `--allow-recorded-commands` is passed. The provisioning half was the one the gate originally missed, and the worst: `grading_sandbox_config` carries the recorded `sandbox` block through untouched and the `--copy` branch calls `Sandbox.setup`, which reaches `uv pip install ` / `npm install` / `git clone ` — arbitrary code at install time — so a shared run dir whose criteria were all `file_exists` sailed through a scan that walked only `success_criteria`. `git clone` now passes `--` before the URL (argv position 2, so a value beginning with `-` was parsed as an option). `Sandbox.resolve_files` is containment-checked for the same reason: criterion paths were the one task-authored path skipping `_resolve_within_sandbox`, and `Path(root) / '/etc/passwd'` is `/etc/passwd`. A warning is not a control: it prints as the command is already being prepared. Passing the task file explicitly (`evaluate `) also bypasses it, since that config came from the operator. The workspace fallback `artifacts / prior.task_id` is containment-checked like its `sandbox_path` sibling (`task_id` is an unvalidated string, and `"../../.."` joins to a real directory `is_dir()` confirms), `_sanitize_restored_path` drops relative entries (they resolve against the grader's cwd) and anything inside the run dir rather than only the workspace, and `write_text_atomic` opens its temp file `O_EXCL|O_NOFOLLOW` — a pre-planted `task.json.tmp` symlink otherwise bypassed the write-back's destination symlink guard entirely. NOTE the Typer command is a thin wrapper over `run_evaluation(...)`, which has real Python defaults — calling a Typer command function in-process hands unspecified options an `OptionInfo` sentinel, which silently made `in_place=None` truthy. +- **`--resume` is command-relative**: `partition_for_resume(tasks, *, grade)` returns a four-way `ResumePartition` (`to_run` / `to_grade` / `prior_results` / `prior_resolved`), because **"finished" is not absolute — it depends on what the resuming command still owes the task**. A `NOT_GRADED` row carries a final status, so the original "has any final status" test called it complete: right for `execute --resume` (it finished executing), and wrong for `run --resume`, which was asked to grade and would instead report "already complete", grade nothing, and **exit 0**. The routing test is the row's **evidence** (`weighted_score is None and not success_criteria_results`), not its category: keying on `category == "ungraded"` missed every `execute` row that ALSO carries an execution fact — a TIMEOUT or budget stop aborts before grading, so it lands unscored with category `error`/`failed`, and resume filed it as complete while `evaluate ` graded the identical bytes happily. Under `grade=True` those rows route to `to_grade`, where `_grade_resumed_tasks` runs the criteria against the trajectory and workspace already on disk via `orchestration/regrade.py::regrade_in_place` — reusing the agent spend, which is the entire reason `execute` and `run` are separate. The carve-out is **only** for `NOT_GRADED`: `FAILURE`/`ERROR` stay complete under both commands (resume has never retried failures — delete the task.json), and `clear_rerun_artifacts` deliberately skips `to_grade`, whose artifacts are the very thing being graded. A per-task grading failure is warned, STAMPED onto the folded-back row's `error_message` (the console line alone is not durable), and folded back in with its ORIGINAL ungraded result, so one bad row neither aborts the resume nor vanishes from run.json — and the exit gate counts `tasks_not_graded` **when `grade` is True**, so a `run` that graded nothing exits non-zero instead of telling CI the suite is fine. Under `execute` an ungraded row is the expected outcome and never fails the command. `grade` is in `_FINGERPRINT_DIFF_EXEMPT` because `execute` → `run --resume` is a supported flow, not config drift — and the warning's "already-finalized tasks keep their original-config results" text is actively wrong for it. **`orchestration/regrade.py` is the single implementation** shared by that path and `evaluate`'s run-dir mode, which DELEGATES to `regrade_in_place` rather than restating it (it originally hand-built its own Sandbox + Orchestrator and had already drifted — hardcoding `replicate_index=0`, so every replicate but the first was relabelled — which is exactly how two copies become two verdicts for the same run); it raises plain `RegradeError`, which the CLI wraps, since `orchestration/` must not import the CLI layer (CE004). One fidelity rule it enforces: a re-graded row keeps the **agent run's** `started_at`/`duration_seconds`, not the grading pass's — a 10-minute run re-graded in 2s would otherwise report 2s into `average_duration`, the report tables and the evalboard; the grading cost is preserved separately as `environment_info["grading_duration_seconds"]`. - **Run-time caps (non-criterion enforcement)**: `TaskDefinition.run_limits` (`RunLimits` model) is the single namespace for all *task-level* run-time caps — `max_turns` / `task_timeout` / `turn_timeout` (structural) and `max_input_tokens` / `max_output_tokens` / `max_total_tokens` / `max_usd` (cumulative budget). Token/USD breaches abort with `FinalStatus.TOKEN_BUDGET_EXCEEDED` or `COST_BUDGET_EXCEEDED` (both `category == "failed"`). Structural caps are set from the CLI via `-D run_limits.max_turns=…` / `-D run_limits.task_timeout=…` / `-D run_limits.turn_timeout=…` (field-merged into `run_limits`); budget caps via `-D run_limits.max_usd=…` etc. or YAML. Layered config uses field-merge — a variant block overrides individual keys without replacing the task's block. The one *per-criterion* cap, `stop_early.decide_within`, deliberately lives on `LiveSuccessCriterion` instead (see below) — the watcher must attribute a decision-step timeout to a specific criterion, which `RunLimits` (task-scoped, criterion-agnostic) cannot express. - **Early stop on criterion (opt-in, per-criterion arming)**: a `stop_early:` block (`StopEarlyPolicy`) on a criterion ends a single-shot run early once the run's **armed** criteria decide the outcome, so a raised `max_turns` isn't wasted on the smoke flavor. The block's PRESENCE is the arming and alone activates the watcher — there is **no run-level master switch**: `run_limits.stop_early: false` is the run-level KILL SWITCH that force-disarms every block (the one-line experiment-variant/`-D` override for an authoritative full run), and `run_limits.stop_early: true` (the removed master arm) is a hard `EarlyStopConfigError` at resolution. The block exists on `LiveSuccessCriterion` only (currently `skill_triggered`, `command_executed` — so arming an unobservable criterion is unrepresentable, a pydantic extra-forbid error). Arming carries one implicit trigger (a native live-fail may fail-stop the run); its keys refine it: `on_pass: stop` (pass-stop the moment the criterion live-passes; default `continue` just latches) and `decide_within: N` (still undecided after N tool-call steps latches an **effective fail**, fed through the same fail-stop rule, reported as `decision_budget_exceeded` — an ordinary weighted fail, NOT a gate-bypassing force-fail; cumulative across retry attempts of the same turn). A trigger whose polarity the instance can't decide (per the abstract, checker-independent `live_decidable_polarities()`, a pure function of the criterion's own fields, paired with the checker's `live_verdict` override by lint rule CE025, a registry-based whole-tree check) is **inert by design** — one dataset-fanned YAML line serves both positive rows (pass/timeout live) and distractor rows (fail live). Verdicts **latch**: once a criterion decides, its `live_verdict` is never polled again. Stop rule is weighted, not strict-boolean: `run_limits.stop_early_gate_threshold` (default `1.0`, reproducing strict-AND behavior exactly) is the minimum weighted score (`Σ weight·score / Σ weight` over the armed subset) required to pass; a fail-stop fires once the armed set's **ceiling** (best case for everything still undecided) can no longer reach the threshold — so a low-weight fail or timeout that can't doom the gate is absorbed and the run continues — and is **deferred while any pass-capable armed criterion is undecided** (a distractor misfire never truncates a positive row's recall signal); a pass-stop fires once the `on_pass: stop` subset's **floor** (worst case) already meets the threshold, and is symmetrically **deferred while any pass-capable armed criterion outside the `on_pass: stop` subset is undecided** (so an early pass never freezes a sibling `on_pass: continue` criterion's signal out of the trajectory). A fail-stop is therefore verdict-preserving; a pass-stop can miss a *later* distractor misfire, so authoritative P/R/F1 comes from a kill-switched (`stop_early: false`) run. Driven by `orchestration/early_stop.py::EarlyStopWatcher` (built when `early_stop_active(task)`: ≥1 armed criterion, kill switch not thrown) through the agent's cooperative `should_stop` seam (tool-call granularity, no SIGKILL); live verdicts only *trigger* the stop — the standard `check_all_async` on the frozen trajectory is authoritative. Gating is **FIRED-ONLY**: a run the watcher actually cut gates on the **armed subset** via the weighted `EvaluationResult.armed_criteria_passed`; a run that completes naturally — armed or not — gates strict-AND via `all_criteria_passed`, so adding a block never changes the verdict of a run it didn't cut. Note the gate keys on the watcher having FIRED (`result.early_stop is not None`), not on confirmed truncation — an agent that ignores `should_stop`, or a stop firing on the final message, still gates armed-only. Every resolution-time guardrail violation is a hard error at resolution (plan *and* run); the one load-time case — a `stop_early:` block on a non-live criterion — is a pydantic schema error at task load, which the run surface reports as a skipped task like any other malformed task. A runtime verdict bug **fails open** to a full run. Surfaces: `EarlyStopInfo` (incl. `gate_threshold` at stop time), report notes/badges, `stopped_early` run.json rows, `EarlyStopped`/`EarlyStopReason` telemetry dims. Worked rationale: docs/TASK_DEFINITION_GUIDE.md § `stop_early`. No blocks anywhere ⇒ behavior byte-for-byte unchanged. @@ -216,7 +232,7 @@ make plugin-reference # the plugin's bundled criteria reference from the models Editing `src/coder_eval/pricing.py` means editing `evalboard/lib/pricing.ts` too — it is a hand-copied mirror, and `evalboard/lib/__tests__/pricing-parity.test.ts` fails the build on drift in either direction. -Recent additions, each traceable to a shipped defect: **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's). +Recent additions, each traceable to a shipped defect: **CE047** (an `environment_info` key that is READ must be WRITTEN somewhere in `src/` — the bag is `dict[str, Any]`, so nothing connects reader to writer, and the `reference_digest` anti-cheat guard shipped as a read with no writer anywhere: `.get()` returned `None`, the guard took its early return, and CLAUDE.md plus the user guide both described it as protection it never provided), **CE048** (never call a Typer command function in process — its parameter defaults are `OptionInfo` sentinels, not values, and the sentinel is TRUTHY, so `in_place=None` silently selected the wrong branch; the fix is the `run_pipeline` / `run_evaluation` / `run_plan` split, and this rule is the one that also scans `tests/`, since that is the only place the defect occurs), **CE049** (never coalesce a possibly-unmeasured score to a numeric literal — `score or 0.0` publishes "measured and scored zero" while meaning "never measured", which is how an ungraded night reached four unfiltered `avg(Score)` dashboards as a real zero), **CE050** (no untyped `getattr` probe for a discriminated-union field — pyright cannot see the string, so a rename degrades the guard to a permanent no-op; scoped to criterion-shaped receivers because `command`/`tool`/`prompt` are far too common to flag on their own), **CE051** (a sandbox driver may not be rewritten silently — the driver IS the isolation boundary, so a downgrade must be an explicit, stamped, operator-visible decision), **CE053** (no bare run-record filename literal outside `path_utils` — `TASK_JSON_FILENAME` shipped with a rename-safety rationale while twelve exact literals stayed unmigrated, including all three `rglob("task.json")` sites the constant's own comment cites as its reason to exist, so it created the second source of truth it argues against), **CE052** (an `os._exit` must sit inside a branch testing `CODER_EVAL_IN_CONTAINER` — it is the right primitive only for reaping the container's own disposable main process, and `run_task_internal_command` armed its heartbeat watchdog, a daemon thread whose whole authority is `os._exit(137)`, unconditionally: a test that invoked the command in-process left the pytest worker holding that thread, which exited the worker 40s later inside an unrelated test file, naming a different test on each run and on each platform with no traceback — and the dead worker's lost coverage data then failed the gate as `65.13 < 80.00`, naming neither the test nor the cause), **CE037** (no unreferenced module-level private helper in `src/` — a helper whose docstring documents a bug the live code still has is worse than none), **CE038** (in an `@asynccontextmanager`, the acquire must sit INSIDE the `try` whose `finally` releases it — `asyncio.shield` protects the inner task, NOT the await, so a cancel on `__aenter__` skips the unwind while the work completes), **CE039** (a criterion checker must not return a gating `score=0.0` from an `except OSError` over a path the *task author* named — that books an eval-config error as an agent failure; raise `CheckerMisuseError` instead, and `# noqa: CE039` the cases that really are the agent's). When fixing a bug, ask: *could a custom lint rule have prevented this?* If the root cause is a mechanically detectable pattern (e.g., "always import from `coder_eval.models`", "never call blocking IO in async"), add a rule to `tests/lint/rules/` following the CE001+ pattern and wire it up in `tests/lint/runner.py`. This turns a one-time fix into permanent enforcement. See `tests/test_custom_lint.py` for how rules are tested. (Doc-surface / whole-tree rules that reason over Markdown/YAML or the entire `src/` tree rather than one `.py` AST at a time — CE026–CE031, CE033–CE036 — are not `BaseRule`s in the runner; they are wired as dedicated `@pytest.mark.lint` test classes. CE036 enforces the `live_verdict` determinism + monotonicity contract (`criteria/base.py`) that `EarlyStopWatcher`'s latching, deferred fail-stop, and flip-attribution silently depend on: monotonicity over arbitrary Python is undecidable, so instead of a static check it REPLAYS each live criterion against every prefix of recorded trajectories (`tests/lint/live_verdict_contract.py::CASES`) — on the authored ordering AND under seeded shuffles (`permuted_violations`, which catch order-sensitive bugs the authored walk misses) — and asserts the property directly, plus registry-derived coverage — every `LiveSuccessCriterion` in the union must have cases, and every polarity its instances claim via `live_decidable_polarities()` must actually be reached by one (otherwise a single always-`undecided` fixture would "cover" a type while proving nothing). Adding a live criterion therefore means adding `ContractCase`s in the same change. CE035 resolves every `steps..outputs.` / `needs..outputs.` reference in `.github/workflows/**` to a writer that actually produces that key — GitHub expands an unwritten output to the empty string, so a typo degrades a gate silently and actionlint models `steps.*.outputs` as an open string map. CE034 scans `tasks/` and forces an armed, live-*passable* `command_executed` to set `require_success` — a crashed invocation would otherwise latch a live PASS, fire `on_pass: stop`, and let FIRED-ONLY armed gating report SUCCESS without ever consulting the unarmed criteria (negative assertions are fail-only and are exempt). CE033 keeps the plugin's bundled `reference/criteria.md` in parity with the `SuccessCriterion` union that generates it (`make plugin-reference` writes it; the rule re-renders and diffs — never hand-edit the file). CE031 guards against dead config: a behavior-driving field on `SimulationConfig`/`RunLimits`/`Dataset` that no code reads by name. CE026 keeps the GitHub Action's onboarding surfaces honest — `README.md`, `docs/CI_GATE.md`, `docs/tutorials/02-ci-pipeline.md`, and the plugin's `ci` skill, whose emitted workflow users copy into their own repos: a page's *first* Action snippet must show the agent-runtime prerequisite steps (pinned to the `action-dogfood` job that proves them in CI), a zero-install absolute next to such a snippet must name the channel it means, every `github.com/marketplace/actions/` link plus the shields badge label must match `action.yml`'s `name:`, and every `with:` key on a snippet's action step must be a real `action.yml` input (GitHub ignores unknown inputs, so a rename would silently degrade every copied workflow). Renaming an action input or changing its runtime prerequisites therefore means updating the skill too.) diff --git a/docs/REPORT_SCHEMA.md b/docs/REPORT_SCHEMA.md index f9d5554a..e85ef305 100644 --- a/docs/REPORT_SCHEMA.md +++ b/docs/REPORT_SCHEMA.md @@ -23,6 +23,7 @@ read). Times are ISO-8601. | --- | --- | --- | | `run.json` / `run.md` | `RunSummary` | Every run (and rebuildable via `coder-eval aggregate`) | | `///task.json` | `EvaluationResult` | One per replicate | +| `///task.execute.json` | `EvaluationResult` | Pre-grade snapshot, written once by a detached grade (`evaluate ` / `run --resume`). Deliberately **not** matched by `rglob("task.json")`, so it never enters an aggregation. | | `//suite.json` / `.md` | `SuiteRollup` | Dataset-backed suites only | | `experiment.json` / `.md` | `ExperimentResult` | Every run (experiment layer) | | `/variant.json` / `.md` | `VariantAggregate` | Per variant | @@ -44,7 +45,8 @@ run-level summary; full per-replicate detail lives in each `task.json`. | `start_time` / `end_time` | `datetime` | Run window. | | `total_duration_seconds` | `float` | Wall-clock. | | `tasks_run` | `int` | Total replicates executed. | -| `tasks_succeeded` / `tasks_failed` / `tasks_error` | `int` | Category counts. **Invariant:** the three sum to `tasks_run`. | +| `tasks_succeeded` / `tasks_failed` / `tasks_error` | `int` | Category counts. **Invariant:** these three plus `tasks_not_graded` sum to `tasks_run`. | +| `tasks_not_graded` | `int` | Tasks run by `coder-eval execute` — executed, deliberately unscored. Excluded from **both** sides of `pass_rate`. Defaults to `0`, so pre-`execute` `run.json` still parses. | | `tasks_token_budget_exceeded` / `tasks_cost_budget_exceeded` | `int` | Sub-counters of `tasks_failed` (not part of the invariant). | | `skipped_tasks` | `list[{path, reason}]` | Load failures / `skip: true` opt-outs. | | `max_parallel` | `int` | Concurrency used. | @@ -59,8 +61,9 @@ publishing different numbers for the same run. | Key | Type | Meaning | | --- | --- | --- | -| `pass_rate` | `float \| None` | `tasks_succeeded / tasks_run` — errors are in the denominator, counted as misses. `None` on an empty run (0/0 is unknown, not 0%). | -| `error_share` | `float \| None` | `tasks_error / tasks_run`. Diagnostic only; never adjusts the rate. | +| `pass_rate` | `float \| None` | `tasks_succeeded / tasks_graded` — errors are in the denominator, counted as misses; ungraded tasks are in neither. `None` when the run is empty, **or when no row produced a verdict at all** — an `execute` night whose only non-ungraded rows are crashes was never measured, and reporting `0.0%` there reads as a total failure. | +| `error_share` | `float \| None` | `tasks_error / tasks_graded`. Diagnostic only; never adjusts the rate. | +| `tasks_graded` | `int` | `tasks_run - tasks_not_graded`. The denominator of both rates above. | | `total_cost_usd` | `float \| None` | **The bill**: agent + judge + simulator, summed over the rows. `None` when nothing could be priced. | | `agent_cost_usd` | `float \| None` | Subject-agent spend alone. The harness-vs-harness comparison figure — judge spend is a property of the suite's criteria and identical across harnesses, so leaving it in would make two harnesses look closer than they are. | | `eval_overhead_cost_usd` | `float \| None` | Judge + simulator spend. The other half of `total_cost_usd`. | @@ -232,7 +235,9 @@ the same weighted armed gate as a native fail), ## `variant.json` — `VariantAggregate` A single aggregate (not wrapped): `variant_id`, `tasks_run`, `tasks_succeeded`, -`tasks_failed`, `tasks_error` (same sum-to-`tasks_run` invariant), `average_score`, +`tasks_failed`, `tasks_error`, `tasks_not_graded` (same sum-to-`tasks_run` invariant), +`average_score` (`float | None` — the mean over **measured** rows: an errored row counts +as `0.0`, an ungraded one leaves both sides; `null` when nothing was measured), `average_duration`, `total_tokens`, `replicate_count`, `tasks_token_budget_exceeded`, `tasks_cost_budget_exceeded`. @@ -246,6 +251,11 @@ The cross-variant summary: where each `VariantResult` carries `{variant_id, task_id, weighted_score, final_status, duration_seconds, total_tokens, iteration_count, total_assistant_turns, reference_similarity, replicate_index, replicate_count}`. + `weighted_score` is `float | None`: `null` only when every replicate was + **ungraded**. An **errored** replicate counts as `0.0` — same rule as + `VariantAggregate.average_score` above. That is deliberate: dropping errored + rows would let a nightly where one image build failed report a *higher* + headline score than a clean one. - `variant_aggregates: dict[str, VariantAggregate]` — keyed by variant id. - `total_duration_seconds`. - `per_replicate_scores: dict[variant_id -> dict[task_id -> list[float]]]`. @@ -262,8 +272,9 @@ Written for dataset-backed suites; its `passed` flag drives the CI exit code. | Key | Type | Meaning | | --- | --- | --- | | `suite_id` / `variant_id` | `str` | Identity. | -| `rows_total` / `rows_passed` / `rows_failed` / `rows_error` | `int` | Row counts. | -| `pass_rate` | `float` | `rows_passed / rows_total`. | +| `rows_total` / `rows_passed` / `rows_failed` / `rows_error` / `rows_not_graded` | `int` | Row counts. **Invariant:** the four category counts sum to `rows_total`. `rows_not_graded` defaults to `0`. | +| `pass_rate` | `float \| null` | `rows_passed / rows_graded` — ungraded rows leave both sides, matching `RunSummary.pass_rate`. `null` when nothing was graded (0/0 is unknown, not 0%). | +| `rows_graded` | `int` | `rows_total - rows_not_graded`. The denominator above, serialized so a consumer never has to re-derive it. | | `average_weighted_score` | `float \| null` | Mean row score. | | `criterion_stats` | `list[{criterion_type, rows_evaluated, average_score, error_count}]` | Per-criterion summary. | | `failed_samples` | `list[FailedRowSummary]` | Capped at 20 (`{row_id, task_id, final_status, weighted_score, failure_reasons, error_message, task_json_relpath, replicate_index}`). | @@ -308,10 +319,20 @@ String enum values and their reporting category: | `COST_BUDGET_EXCEEDED` | failed | `$` | | `ERROR` | error | `!` | | `BUILD_FAILED` | error | `B` | +| `NOT_GRADED` | ungraded | `?` | > **Gotcha:** `BUILD_FAILED` (a failed Docker image build) categorizes as **error**, > not failed — easy to miscount downstream. +`NOT_GRADED` is produced only by [`coder-eval execute`](USER_GUIDE.md#coder-eval-execute--run-without-grading): +the task ran and its full trajectory was captured, but no criterion was checked, so +`weighted_score` is `None` (**not** `0.0` — that would be indistinguishable from a task +that was graded and scored zero). `ungraded` is a fourth reporting category, not a fold +into one of the other three: counting it as failed would depress every pass rate, and +counting it as succeeded would invent a verdict. Execution facts still win over it — a +crash, timeout, or budget breach under `execute` reports `ERROR` / `TIMEOUT` / +`TOKEN_BUDGET_EXCEEDED` as usual. + `TOKEN_BUDGET_EXCEEDED` and `COST_BUDGET_EXCEEDED` are produced by the cumulative budget caps under `run_limits:` (`max_input_tokens` / `max_output_tokens` / `max_total_tokens`, and `max_usd` respectively), checked after each completed agent turn — see diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index 1676690d..24980077 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -41,7 +41,8 @@ coder-eval run tasks/hello_date.yaml --stream full # live LLM output | `--driver` | Shorthand alias for `-D sandbox.driver=…` (`tempdir` or `docker`) | | `--type, -T` | Override agent type for all tasks (`claude-code`, `codex`, `antigravity`, `opencode`, or a plugin kind). | | `--repeats` | Run each `(task, variant)` N times (≥1); overrides experiment/variant `repeats:`. See [Replicates](#replicates). | -| `--resume` | Resume an interrupted run: skip tasks already finalized in `--run-dir` and run the rest, folding prior results into `run.json`. Requires `--run-dir`. A task with *any* final status (incl. FAILED/ERROR) counts as finalized, so resume does **not** retry failures — delete a task's `task.json` to force a re-run. A config mismatch is warned, not refused. | +| `--resume` | Resume an interrupted run: skip tasks already finalized in `--run-dir` and run the rest, folding prior results into `run.json`. Requires `--run-dir`. See [Resuming a run](#resuming-a-run). | +| `--allow-host-grading` | `--resume` only. Grade an executed-but-ungraded `driver: docker` row on this host instead of refusing; the row is stamped `graded_on_host`. Rejected without `--resume`, since a fresh `run` grades inside the driver the task asks for. | | `--sample N` | For dataset-backed tasks, run a fixed-seed random N-row sample (reproducible; cheap smoke test). See [Bring Your Own Dataset](DATASETS.md). | | `--sample-per-stratum N` | For dataset-backed tasks, keep up to N rows per stratum (`stratify_field`). Overridden by `--sample`. Nondeterministic unless `dataset.sample_seed` is set — see [Bring Your Own Dataset](DATASETS.md). | | `--include-skipped` | Also run tasks marked `skip: true` in their YAML (off by default so CI keeps excluding them). | @@ -58,6 +59,92 @@ flags of their own. They live under `run_limits:` in the task YAML, or on the co `-D run_limits.=` (e.g. `-D run_limits.max_usd=2.50`). The complete field reference is in the [Task Definition Guide](TASK_DEFINITION_GUIDE.md#run-limits). +### `coder-eval execute` — run without grading + +```bash +coder-eval execute tasks/hello_date.yaml # run, capture, score nothing +coder-eval execute tasks/*.yaml --run-dir ./my-run -j 3 # every `run` flag but two +``` + +Identical to `coder-eval run` except that no success criterion is checked. Each task +executes normally and its full trajectory lands in `task.json` as usual, but +`weighted_score` stays `null` and the row finalizes as `NOT_GRADED` — a reporting +category of its own, excluded from both sides of every pass rate. The two commands +share one implementation, so they cannot drift apart. + +Use it when something *else* owns the verdict — an external harness that builds its own +container and runs its own tests — or to separate one expensive agent run from grading +you want to iterate on afterwards. Grade the results later with +[`coder-eval evaluate`](#coder-eval-evaluate--grade-without-running-an-agent). + +**Only the verdict is withheld, never the facts of the run.** A crash, timeout, or +budget breach still reports `ERROR` / `TIMEOUT` / `TOKEN_BUDGET_EXCEEDED` and still +exits non-zero, exactly as under `run`. + +Exhausting `max_turns` is the one fact that does *not* become a status here. Under +`run` it decides the outcome only when the criteria fail — a max-turns trajectory +whose criteria pass is `SUCCESS` — so it is not knowable without grading. `execute` +records `max_turns_exhausted: true` on the row and finalizes `NOT_GRADED`; the later +grade reads the flag and reaches exactly the status `run` would have. Rows like this +are picked up by `run --resume`, which owes a grade to anything executed but never +scored — including a row that also timed out or tripped a budget. + +Every `run` flag is available except three things, each refused rather than quietly +degraded: + +| Not supported | Why | +| --- | --- | +| `--junit-xml` | A JUnit report reports verdicts, and there are none. | +| `--allow-host-grading` | It decides how an executed-but-ungraded row is *graded*, and `execute` grades nothing. | +| Simulation tasks | The dialog loop reads criteria results to decide whether to keep talking, so an ungraded dialog would silently change its own stopping behavior. Rejected by name at startup. | + +`stop_early:` blocks are also inert here: early stop exists to cut a run once the +criteria decide the outcome, and under `execute` the full trajectory is the deliverable. + +`--resume` **is** supported, and `run --resume` pairs with it (see below). + +### Resuming a run + +`--resume` continues an interrupted run without re-paying for finished work. It +requires `--run-dir` (an auto-generated directory is always fresh). + +What it owes each task depends on what it finds in that task's `task.json`: + +| On disk | `run --resume` | `execute --resume` | +| --- | --- | --- | +| No `task.json`, unreadable, or no `final_status` | re-run | re-run | +| `NOT_GRADED` | **grade in place** | already complete | +| Any other status, **including `FAILURE` / `ERROR`** | already complete | already complete | + +**"Finished" is relative to the resuming command.** A `NOT_GRADED` row owes +`execute` nothing — it finished executing — but owes `run` a grade. So +`run --resume` runs the criteria against the trajectory and workspace already on +disk instead of re-running the agent, which is the whole reason to split the two +commands: + +```bash +coder-eval execute tasks/*.yaml --run-dir ./r # expensive half +coder-eval run tasks/*.yaml --run-dir ./r --resume # grades what execute left +``` + +**Resume never retries failures.** `FAILURE` and `ERROR` count as complete under +both commands — delete a task's `task.json` to force a re-run. A task about to +re-run has its stale `artifacts/` cleared first, so leftover files from +a killed container cannot satisfy a file-based criterion. + +A run-config mismatch is **warned, not refused**: resumed tasks keep their +original-config results, so the run genuinely mixes configs. The `grade` flag is +exempt from that warning, because `execute` → `run --resume` is a supported flow +rather than a config mistake. + +**A dataset task must pin its sample to be resumable.** Stratified sampling +(`--sample-per-stratum` / `dataset.sample_per_stratum`) re-draws on every +invocation, and each row is its own task (`/`) with its own run +directory. A resume therefore draws a *different* row set, finds no `task.json` +for it, and pays for the agent a second time while the executed rows sit +orphaned in the run dir. Set `dataset.sample_seed`, or use `--sample N` (which is +seeded), before splitting a dataset run across `execute` and `run --resume`. + ### `coder-eval plan` — validate tasks ```bash @@ -71,23 +158,77 @@ Checks task syntax, required CLI tools, API keys, and schema validity without ex | --- | --- | | `--experiment, -e` | Experiment definition YAML to resolve variants against (default: `experiments/default.yaml`). | -### `coder-eval evaluate` — test criteria without an agent +### `coder-eval evaluate` — grade without running an agent + +Two shapes, told apart by whether the target holds a `task.json`: ```bash -coder-eval evaluate tasks/hello_date.yaml ./my_solution # evaluate a directory -coder-eval evaluate tasks/hello_date.yaml ./my_solution --preserve # keep the sandbox +# 1. Grade a directory against a task +coder-eval evaluate tasks/hello_date.yaml ./my_solution + +# 2. Re-grade a finished run — including one left NOT_GRADED by `execute` +coder-eval execute tasks/hello_date.yaml --run-dir ./r +coder-eval evaluate ./r/default/hello_date/00 +coder-eval aggregate ./r # run.json now reports the verdict +``` + +**Run-directory mode** rebuilds the task from the run's own recorded +`task_config.resolved`, not by re-reading the YAML. That is what makes the grade +describe the run that happened: variant overrides, `-D` flags and dataset row +expansion are already baked into `resolved`, so re-loading the source would +silently grade a *different* task. The run's trajectory is restored too, so +criteria that read the agent's tool calls (`command_executed`, `skill_triggered`, +judges with trajectory) score exactly as they would have during the run. + +It writes the verdict back into the run's `task.json` and keeps the pre-grade +record beside it as `task.execute.json`. Writing back in place is what makes +`aggregate` free — no new flag, no second copy of the results. If grading itself +crashes, the ungraded record is put back: `ERROR` counts as complete for both +commands, so an errored row could never be graded again. + +**A run directory is untrusted input.** It is a shareable artifact — the whole +point of the detached flow is that one machine executes and another grades — and +rebuilding the task from it means the run dir decides what runs on your host, +with your environment. So two things are refused rather than assumed: + +- A recorded config that carries shell (`run_command` criteria, `agent_judge`, + `uipath_eval`, and on the `--copy` path `pre_run`/`post_run`) needs + `--allow-recorded-commands`. The message names every command first. Passing the + task file explicitly bypasses this — that config came from you. +- A run made with `driver: docker` needs `--allow-host-grading`. Grading cannot + start a container, and such a task's criteria address container paths and + toolchains; on your host they score `0.0` for a run that passed. An opted-in + row is stamped `graded_on_host` in `environment_info` so it is never silently + compared with a container-graded one. + +Passing a task file **over** a run directory re-grades it with different +criteria, reusing the trajectory and workspace of a run you already paid for: + +```bash +coder-eval evaluate tasks/hello_date.edited.yaml ./r/default/hello_date/00 ``` -Runs a task's success criteria against a directory without an agent — useful for -testing criterion definitions, validating task configs, or scoring code that was -already written. +**In-place vs. copy.** The two-argument form copies your directory into a fresh +sandbox (criteria can mutate the target, and it is your own tree). Run-directory +mode grades **in place**, because copying filters build output — `node_modules`, +`dist`, `build`, `.venv`, `.git` are all on the default ignore list, so a +criterion like `test -f dist/bundle.js` would fail as a *copying artifact* +rather than as a verdict. Override either default with `--in-place` / `--copy`. | Flag | Description | | --- | --- | -| `--preserve / --no-preserve` | Preserve sandbox after evaluation (default: preserve) | -| `--run-dir` | Custom run directory (default: auto-generated timestamped dir in `runs/`). | +| `--workspace` | Grade this directory instead of the run's own artifacts (run-directory mode only). | +| `--in-place / --copy` | Grade where the files are, or copy first. Default: in-place for a run directory, copy for a plain work directory. | +| `--preserve / --no-preserve` | Preserve sandbox after evaluation (default: preserve). Ignored when grading in place — an adopted directory is never moved or deleted. | +| `--run-dir` | Where the graded `task.json` lands (default: auto-generated timestamped dir in `runs/`). | +| `--allow-recorded-commands` | Accept a rebuilt config that would run shell (`run_command` criteria, judges, `pre_run`/`post_run`) or install packages on this host. Refused by default — a run directory is a shareable artifact, so its recorded config is untrusted input. | +| `--allow-host-grading` | Grade a `driver: docker` task on this host instead of refusing. The row is stamped `graded_on_host` so it is never silently compared with a container-graded one. | | `--verbose, -v` | DEBUG-level logging | +A re-grade refuses to run if the task's `reference:` directory changed since the +run (digest mismatch) — grading then would score the agent's old work against a +new answer key. + ### `coder-eval report` — view results ```bash diff --git a/docs/tutorials/02-ci-pipeline.md b/docs/tutorials/02-ci-pipeline.md index f6cb76c9..2c8c6839 100644 --- a/docs/tutorials/02-ci-pipeline.md +++ b/docs/tutorials/02-ci-pipeline.md @@ -119,9 +119,14 @@ jobs: for p in sorted(pathlib.Path(sys.argv[1]).rglob("task.json"))] for r in rows: print(f"- {r.get('task_id','?')}: {r.get('final_status','?')}") - ok = sum(1 for r in rows if r.get("final_status") == "SUCCESS") - print(f"\n**{ok}/{len(rows)} PASS**") - sys.exit(0 if rows and ok == len(rows) else 1) + # NOT_GRADED is a fourth category, not a failure: `coder-eval execute` + # runs the agent and deliberately scores nothing. Counting such a row + # as a miss fails a pipeline that has not measured anything yet. + graded = [r for r in rows if r.get("final_status") != "NOT_GRADED"] + ok = sum(1 for r in graded if r.get("final_status") == "SUCCESS") + print(f"\n**{ok}/{len(graded)} PASS**" + + (f" ({len(rows) - len(graded)} not graded)" if len(graded) != len(rows) else "")) + sys.exit(0 if graded and ok == len(graded) else 1) PY - name: Upload run reports diff --git a/evalboard/app/_overview/window-summary.tsx b/evalboard/app/_overview/window-summary.tsx index 29c06245..e67ace75 100644 --- a/evalboard/app/_overview/window-summary.tsx +++ b/evalboard/app/_overview/window-summary.tsx @@ -54,8 +54,8 @@ export function WindowSummary({ isFiltered: boolean; }) { const pct = - totals.tasksRun > 0 - ? (totals.tasksSucceeded / totals.tasksRun) * 100 + totals.tasksGraded > 0 + ? (totals.tasksSucceeded / totals.tasksGraded) * 100 : null; const scope = isFiltered ? `matching · last ${window}` diff --git a/evalboard/app/page.tsx b/evalboard/app/page.tsx index 8537c1ac..7a66f245 100644 --- a/evalboard/app/page.tsx +++ b/evalboard/app/page.tsx @@ -379,8 +379,12 @@ export default async function Page({ {listing.rows.map((r) => { const total = r.tasksRun; - const pct = total - ? (r.tasksSucceeded / total) * 100 + // Ungraded rows (`coder-eval execute`) leave BOTH + // sides: the rate divides by tasksGraded, while the + // Tasks column still reports everything that ran. + const graded = r.tasksGraded; + const pct = graded + ? (r.tasksSucceeded / graded) * 100 : null; return ( - {r.tasksSucceeded}/{total} + {r.tasksSucceeded}/{graded} @@ -517,8 +521,9 @@ export default async function Page({ {adhoc.rows.map((r) => { const total = r.tasksRun; - const pct = total - ? (r.tasksSucceeded / total) * 100 + const graded = r.tasksGraded; + const pct = graded + ? (r.tasksSucceeded / graded) * 100 : null; return ( - {r.tasksSucceeded}/{total} + {r.tasksSucceeded}/{graded} diff --git a/evalboard/app/path-to-ga/__tests__/task-table.test.tsx b/evalboard/app/path-to-ga/__tests__/task-table.test.tsx index c67284a6..179100a2 100644 --- a/evalboard/app/path-to-ga/__tests__/task-table.test.tsx +++ b/evalboard/app/path-to-ga/__tests__/task-table.test.tsx @@ -12,6 +12,7 @@ function row(overrides: Partial = {}): TagTaskRow { skill: "uipath-maestro-flow", appearances: 20, matureSkips: 0, + ungraded: 0, executed: 20, passRate: 90, latestStatus: "SUCCESS", diff --git a/evalboard/app/runs/[id]/__tests__/task-grid.test.tsx b/evalboard/app/runs/[id]/__tests__/task-grid.test.tsx index c840715f..27953ed3 100644 --- a/evalboard/app/runs/[id]/__tests__/task-grid.test.tsx +++ b/evalboard/app/runs/[id]/__tests__/task-grid.test.tsx @@ -408,7 +408,7 @@ describe("TaskGrid — replicates", () => { expect(reptaskLinks[0]).toHaveAttribute("href", "/runs/r1/reptask?r=0"); // k/N ✓ badge: all 3 replicates passed → "3 of 3", GREEN. solo (single // run) shows no badge. - const badge = within(table).getByTitle(/3 of 3 replicates passed/i); + const badge = within(table).getByTitle(/3 of 3 graded replicates passed/i); expect(badge.textContent?.replace(/\s/g, "")).toBe("3/3✓"); expect(badge.className).toContain("text-green-700"); }); @@ -426,7 +426,7 @@ describe("TaskGrid — replicates", () => { ); const table = screen.getByRole("table"); // 1 of 3 replicates passed → shows the pass count, AMBER (partial). - const badge = within(table).getByTitle(/1 of 3 replicates passed/i); + const badge = within(table).getByTitle(/1 of 3 graded replicates passed/i); expect(badge.textContent?.replace(/\s/g, "")).toBe("1/3✓"); expect(badge.className).toContain("text-amber-700"); }); @@ -442,7 +442,7 @@ describe("TaskGrid — replicates", () => { />, ); const table = screen.getByRole("table"); - const badge = within(table).getByTitle(/0 of 2 replicates passed/i); + const badge = within(table).getByTitle(/0 of 2 graded replicates passed/i); expect(badge.textContent?.replace(/\s/g, "")).toBe("0/2✓"); expect(badge.className).toContain("text-red-700"); }); @@ -469,7 +469,7 @@ describe("TaskGrid — replicates", () => { expect(link).toHaveAttribute("href", "/runs/r1/t?r=1"); // Badge still shows the true ratio. expect( - within(tr).getByTitle(/1 of 2 replicates passed/i).textContent?.replace(/\s/g, ""), + within(tr).getByTitle(/1 of 2 graded replicates passed/i).textContent?.replace(/\s/g, ""), ).toBe("1/2✓"); }); }); diff --git a/evalboard/app/runs/[id]/__tests__/ungraded-golden.test.ts b/evalboard/app/runs/[id]/__tests__/ungraded-golden.test.ts new file mode 100644 index 00000000..3cd06210 --- /dev/null +++ b/evalboard/app/runs/[id]/__tests__/ungraded-golden.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, test } from "vitest"; +import type { TaskResultSummary } from "@/lib/runs"; +import { perTaskGradedCounts, perTaskPassCounts } from "@/lib/status"; +import { taskVariantKey } from "@/lib/variants"; +import { computeRunMetrics, computeVariantMetrics } from "../run-view"; + +// One invariant, asserted through every rate surface a `coder-eval execute` run +// touches: an UNGRADED row leaves BOTH sides of the rate. +// +// This is arithmetic over a data shape, not a code pattern, so no lint rule +// catches it — and the first ungraded sweep proved a correct guard can still be +// applied to the wrong denominator: `pct` divided by the graded count while the +// LABEL beside it read `passed / total`, so 8 graded passes next to 2 ungraded +// rows rendered "80%" beside "8 / 12", and a plain 12-task execute run rendered +// a red "0% 0 / 12". + +function row( + taskId: string, + extra: Partial = {}, +): TaskResultSummary { + return { + taskId, + variantId: null, + replicateIndex: null, + status: "SUCCESS", + weightedScore: 1.0, + durationSeconds: 1.0, + totalCostUsd: 0.1, + actualCommands: null, + totalTurns: null, + expectedTurns: null, + expectedSeconds: null, + hasFinalReply: false, + inputTokens: null, + outputTokens: null, + cacheCreationTokens: null, + cacheReadTokens: null, + model: null, + tags: [], + skill: null, + matureSkipped: false, + ...extra, + }; +} + +const ungraded = (taskId: string, extra: Partial = {}) => + row(taskId, { status: "NOT_GRADED", weightedScore: null, ...extra }); + +describe("a fully ungraded run reports 0 of 0, never 0%", () => { + const tasks = Array.from({ length: 12 }, (_, i) => ungraded(`t${i}`)); + const m = computeRunMetrics(tasks); + + test("every row is bucketed as ungraded, none as failed", () => { + expect(m.total).toBe(12); + expect(m.ungraded).toBe(12); + expect(m.failed).toBe(0); + expect(m.errored).toBe(0); + expect(m.passed).toBe(0); + }); + + test("the denominator the tile renders is zero, so the tile reads neutral", () => { + // `graded`, not `total`. The tile's tone is null when this is 0, which is + // what stops a clean execute run rendering as a red measured 0%. + expect(m.graded).toBe(0); + }); + + test("the per-task rollup does not score them as failures", () => { + // 0 of 0 tasks, not 0 of 12: an all-ungraded task is absent from the + // rollup entirely rather than appearing as a task that failed. + expect(m.taskTotal).toBe(0); + expect(m.taskPassed).toBe(0); + expect(m.taskFailed).toBe(0); + }); +}); + +describe("a mixed run divides by the graded rows on both halves of the tile", () => { + const tasks = [ + ...Array.from({ length: 8 }, (_, i) => row(`p${i}`)), + row("f0", { status: "FAILURE", weightedScore: 0 }), + row("f1", { status: "FAILURE", weightedScore: 0 }), + ungraded("u0"), + ungraded("u1"), + ]; + const m = computeRunMetrics(tasks); + + test("pct and its label describe the same sample", () => { + expect(m.total).toBe(12); + expect(m.graded).toBe(10); + expect(m.passed).toBe(8); + expect(m.pct).toBeCloseTo(80); + // The pair the tile renders: "80% 8 / 10". Dividing by `total` here is + // the bug this asserts against — it produced "80% 8 / 12". + expect(m.passed / m.graded).toBeCloseTo(m.pct / 100); + }); + + test("ungraded rows are not counted as failures", () => { + expect(m.failed).toBe(2); + expect(m.failedTotal).toBe(2); + }); +}); + +describe("repeats: an ungraded replicate leaves both sides of the k/N badge", () => { + const tasks = [ + row("t", { replicateIndex: 0 }), + ungraded("t", { replicateIndex: 1 }), + ungraded("u", { replicateIndex: 0 }), + ungraded("u", { replicateIndex: 1 }), + ]; + + test("the badge reads 1/1, not 1/2", () => { + const passes = perTaskPassCounts(tasks); + const graded = perTaskGradedCounts(tasks); + const key = taskVariantKey({ taskId: "t", variantId: null }); + expect(passes.get(key)).toBe(1); + expect(graded.get(key)).toBe(1); + }); + + test("a task whose every replicate was ungraded is absent, not 0/2", () => { + const graded = perTaskGradedCounts(tasks); + expect(graded.has(taskVariantKey({ taskId: "u", variantId: null }))).toBe( + false, + ); + }); +}); + +describe("variants: each arm's rate excludes its own ungraded rows", () => { + const tasks = [ + row("t0", { variantId: "a" }), + ungraded("t1", { variantId: "a" }), + row("t0", { variantId: "b", status: "FAILURE", weightedScore: 0 }), + ungraded("t1", { variantId: "b" }), + ]; + const rows = computeVariantMetrics(tasks); + + test("arm a is 100% of one graded task, not 50% of two", () => { + const a = rows.find((r) => r.variantId === "a")!.metrics; + expect(a.graded).toBe(1); + expect(a.taskTotal).toBe(1); + expect(a.taskPassed).toBe(1); + }); + + test("arm b is a measured 0% — an ungraded row must not soften a real failure", () => { + const b = rows.find((r) => r.variantId === "b")!.metrics; + expect(b.graded).toBe(1); + expect(b.taskTotal).toBe(1); + expect(b.taskPassed).toBe(0); + }); +}); diff --git a/evalboard/app/runs/[id]/run-view.tsx b/evalboard/app/runs/[id]/run-view.tsx index ea1626e9..68398baf 100644 --- a/evalboard/app/runs/[id]/run-view.tsx +++ b/evalboard/app/runs/[id]/run-view.tsx @@ -6,7 +6,7 @@ import type { ActivationScore, TaskResultSummary } from "@/lib/runs"; import type { ReviewIndexEntry } from "@/lib/reviews-types"; import { fmtDuration, humanizeTaskId } from "@/lib/format"; import { passBarClass, passClass } from "@/lib/pass-rate"; -import { perTaskPassCounts, statusCategory } from "@/lib/status"; +import { assertNever, perTaskPassCounts, statusCategory } from "@/lib/status"; import { DEFAULT_VARIANT_ID, taskVariantKey, @@ -38,6 +38,14 @@ export interface RunMetrics { passed: number; failed: number; errored: number; + // Rows that ran but were never scored (`coder-eval execute`). Excluded from + // both sides of `pct`, so a fully ungraded run reports 0 of 0, not 0%. + ungraded: number; + // `total - ungraded`: the pass-rate denominator. Carried on the metrics + // rather than re-derived at each tile — the tile used to divide by `graded` + // while LABELLING the result `passed / total`, so 8 of 10 graded beside 2 + // ungraded rendered "80% 8 / 12". + graded: number; failedTotal: number; pct: number; // Per-task view of pass rate for repeated runs: distinct task_ids, and how @@ -72,15 +80,36 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics { let passed = 0; let failed = 0; let errored = 0; + let ungraded = 0; let cost = 0; let durationSum = 0; const costSamples: number[] = []; const durSamples: number[] = []; for (const t of tasks) { + // A `switch` with an assertNever default, not an if/else chain with a + // catch-all `else failed++`. That chain is what made adding "ungraded" + // a SILENT change: anything not a pass or an error was counted as a + // failure and kept in the denominator, and no compiler saw it. With + // this shape a fifth StatusCategory fails `tsc --noEmit` right here. const cat = statusCategory(t.status); - if (cat === "passed") passed++; - else if (cat === "error") errored++; - else failed++; + switch (cat) { + case "passed": + passed++; + break; + case "error": + errored++; + break; + case "ungraded": + // Never scored, so it leaves BOTH sides of the rate. + ungraded++; + break; + case "failed": + case "unknown": + failed++; + break; + default: + assertNever(cat); + } if (t.matureSkipped) continue; if (t.totalCostUsd != null) { cost += t.totalCostUsd; @@ -91,16 +120,22 @@ export function computeRunMetrics(tasks: TaskResultSummary[]): RunMetrics { durSamples.push(t.durationSeconds); } } + const graded = total - ungraded; return { total, passed, failed, errored, + ungraded, + graded, failedTotal: failed + errored, - pct: total ? (passed / total) * 100 : 0, + pct: graded ? (passed / graded) * 100 : 0, ...(() => { // Per-task rollup (any replicate passed → task passed) via the shared // helper, so the run tile and the grid badge apply the same rule. + // perTaskPassCounts already drops ungraded replicates, so a task + // whose replicates were all ungraded is absent from the map — which + // is what keeps it out of BOTH taskTotal and taskPassed. const perTask = perTaskPassCounts(tasks); const taskPassed = [...perTask.values()].filter((n) => n > 0).length; return { @@ -544,11 +579,15 @@ export function RunView({ const passed = hasRepeats ? metrics.taskPassed : metrics.passed; + // metrics.graded, NOT metrics.total: `pct` divides by + // the graded count, so labelling it with every row that + // ran renders a mismatched pair (and a red 0% next to + // "0 / 12" for a clean `coder-eval execute` run). const totalN = hasRepeats ? metrics.taskTotal - : metrics.total; - // null when nothing ran, so an empty run reads neutral - // rather than as a measured 0%. + : metrics.graded; + // null when nothing was MEASURED, so an empty or fully + // ungraded run reads neutral rather than as a 0%. const tone = totalN > 0 ? pct : null; return ( <> diff --git a/evalboard/app/runs/[id]/task-grid.tsx b/evalboard/app/runs/[id]/task-grid.tsx index 4209feb2..6d3fb7b9 100644 --- a/evalboard/app/runs/[id]/task-grid.tsx +++ b/evalboard/app/runs/[id]/task-grid.tsx @@ -14,7 +14,7 @@ import { MaturePill, StatusPill, } from "@/lib/pills"; -import { isPassStatus, perTaskPassCounts, statusSortRank } from "@/lib/status"; +import { isPassStatus, perTaskGradedCounts, perTaskPassCounts, statusSortRank } from "@/lib/status"; import { DEFAULT_VARIANT_ID, taskVariantKey, variantsOf } from "@/lib/variants"; import { displayedTurns, @@ -182,6 +182,7 @@ function TaskIdCell({ className, matureSourceRuns, replicateCount = 1, + replicateGradedCount = 0, replicatePassCount = 0, sourceId, }: { @@ -194,6 +195,8 @@ function TaskIdCell({ // >1 the row collapses to a single entry with a k/N ✓ badge; the per-run // detail is reachable from the task page's run selector (?r=NN). replicateCount?: number; + // How many of those replicates were graded — the badge's denominator. + replicateGradedCount?: number; // How many of those replicates passed — shown as k/N and color-coded. replicatePassCount?: number; }) { @@ -238,12 +241,12 @@ function TaskIdCell({ return ( {humanizeTaskId(t.taskId)} - {replicateCount > 1 && ( + {replicateCount > 1 && replicateGradedCount > 0 && ( - {replicatePassCount}/{replicateCount} ✓ + {replicatePassCount}/{replicateGradedCount} ✓ )} @@ -565,6 +568,13 @@ export function TaskGrid({ // page's pass-rate tile all apply the same "any replicate passed" rule. const replicatePassCounts = useMemo(() => perTaskPassCounts(tasks), [tasks]); + // The badge's DENOMINATOR. Not `replicateCounts`, which counts every + // replicate: an ungraded replicate (`coder-eval execute`) was never scored, + // so counting it there rendered a red "0/2 ✓" with the tooltip "0 of 2 + // replicates passed" for a run nothing was wrong with. Zero (every + // replicate ungraded) suppresses the badge entirely. + const replicateGradedCounts = useMemo(() => perTaskGradedCounts(tasks), [tasks]); + // Collapse replicates to one row per (variant, task): repeated runs share a // taskId, so the grid shows a single entry with a k/N ✓ badge; the per-run // detail is selectable on the task page. The representative is chosen so its @@ -752,6 +762,9 @@ export function TaskGrid({ replicateCount={ replicateCounts.get(taskVariantKey(t)) ?? 1 } + replicateGradedCount={ + replicateGradedCounts.get(taskVariantKey(t)) ?? 0 + } replicatePassCount={ replicatePassCounts.get(taskVariantKey(t)) ?? 0 } @@ -913,6 +926,9 @@ export function TaskGrid({ replicateCount={ replicateCounts.get(taskVariantKey(t)) ?? 1 } + replicateGradedCount={ + replicateGradedCounts.get(taskVariantKey(t)) ?? 0 + } replicatePassCount={ replicatePassCounts.get(taskVariantKey(t)) ?? 0 } diff --git a/evalboard/app/scribe/run-table.tsx b/evalboard/app/scribe/run-table.tsx index 22fc0984..d3e767eb 100644 --- a/evalboard/app/scribe/run-table.tsx +++ b/evalboard/app/scribe/run-table.tsx @@ -11,8 +11,10 @@ import { TableScroll } from "../_components/scroll-table"; type ScribeRow = RunListingRow & { title?: string | null }; function passPct(row: ScribeRow): number | null { - if (row.tasksRun === 0) return null; - return (row.tasksSucceeded / row.tasksRun) * 100; + // tasksGraded, not tasksRun: an ungraded row (`coder-eval execute`) was + // never scored, so it leaves both sides of the rate. + if (row.tasksGraded === 0) return null; + return (row.tasksSucceeded / row.tasksGraded) * 100; } export function ScribeRunTable({ @@ -120,7 +122,7 @@ export function ScribeRunTable({ : "—"} - {row.tasksSucceeded}/{row.tasksRun} + {row.tasksSucceeded}/{row.tasksGraded} {fmtDuration(row.taskDurationSeconds)} diff --git a/evalboard/app/trends/trends-view.tsx b/evalboard/app/trends/trends-view.tsx index 611b7025..b1e7eb4a 100644 --- a/evalboard/app/trends/trends-view.tsx +++ b/evalboard/app/trends/trends-view.tsx @@ -32,6 +32,7 @@ import { VersionChip } from "@/app/_components/version-list"; import { HarnessSelector } from "@/app/_components/harness-selector"; import { harnessShortLabel } from "@/app/_components/harness-badge"; import { fetchTaskHistoryAction } from "./actions"; +import { isGraded, isPassStatus } from "@/lib/status"; function fmtUsd(c: number | null): string { if (c == null) return "—"; @@ -146,9 +147,13 @@ function sortTasks( function statusFill(status: string | null): string { if (status == null) return "bg-gray-300"; - if (status === "SUCCESS") return "bg-green-500"; - // All non-success outcomes share one red — the trends view treats every - // failure mode as equally bad rather than ranking FAILED vs ERROR. + if (isPassStatus(status)) return "bg-green-500"; + // An ungraded row was never measured, so it is not a failure. Painting it + // red contradicted lib/trends.ts, which had just excluded those very rows + // from the pass rate feeding this same timeline. + if (!isGraded(status)) return "bg-gray-300"; + // All non-success graded outcomes share one red — the trends view treats + // every failure mode as equally bad rather than ranking FAILED vs ERROR. return "bg-red-500"; } diff --git a/evalboard/lib/__tests__/overview.test.ts b/evalboard/lib/__tests__/overview.test.ts index d012d39d..92774063 100644 --- a/evalboard/lib/__tests__/overview.test.ts +++ b/evalboard/lib/__tests__/overview.test.ts @@ -44,6 +44,7 @@ describe("summarizeListing", () => { id: "r", tasksSucceeded: 0, tasksRun: 0, + tasksGraded: 0, totalCostUsd: null, taskDurationSeconds: null, ...overrides, @@ -57,6 +58,7 @@ describe("summarizeListing", () => { costPartial: false, tasksSucceeded: 0, tasksRun: 0, + tasksGraded: 0, durationSeconds: null, durationPartial: false, }); @@ -1256,3 +1258,82 @@ describe("adhocRunDate", () => { ]); }); }); + +describe("ungraded rows leave both sides of every rate", () => { + // The three denominators this PR changed had no assertions at all. Each + // case below is the arithmetic invariant, not the code shape: a row that + // was never measured must leave the numerator AND the denominator, or a + // `coder-eval execute` night publishes a measured-looking 0%. + function ungradedPerRun(id: string, tasks: RunOverviewTask[]): PerRun { + return { + id, + overview: { + id, + tasks, + totalCostUsd: null, + taskDurationSeconds: null, + componentShas: [], + }, + reviewTagCounts: {}, + reviewTagsByTask: {}, + adhoc: false, + title: null, + }; + } + + test("buildTagTaskRows: executed = appearances - matureSkips - ungraded", () => { + const TAG = "path-to-ga"; + const rows = buildTagTaskRows( + [ + ungradedPerRun("r1", [ + task({ taskId: "a", tags: [TAG], status: "SUCCESS" }), + ]), + ungradedPerRun("r2", [ + task({ taskId: "a", tags: [TAG], status: "NOT_GRADED" }), + ]), + ], + TAG, + ); + + expect(rows[0].appearances).toBe(2); + expect(rows[0].ungraded).toBe(1); + expect(rows[0].executed).toBe(1); + expect(rows[0].passRate).toBe(100); + }); + + test("buildTagTaskRows: an all-ungraded task has no pass rate at all", () => { + const TAG = "path-to-ga"; + const rows = buildTagTaskRows( + [ + ungradedPerRun("r1", [ + task({ taskId: "a", tags: [TAG], status: "NOT_GRADED" }), + ]), + ], + TAG, + ); + + expect(rows[0].executed).toBe(0); + expect(rows[0].passRate).toBeNull(); + }); + + test("turnBudgetRateForTasks: an ungraded budgeted row is not a budget miss", () => { + // The confirmed miss. `status` is a free string, so neither tsc nor + // assertNever could see that a raw `!== "SUCCESS"` books the fourth + // category as a failure — it entered `eligible` and never + // `withinBudget`, publishing 0%. + const withUngraded = turnBudgetRateForTasks([ + task({ status: "SUCCESS", expectedTurns: 10, visibleTurns: 5 }), + task({ status: "NOT_GRADED", expectedTurns: 10, visibleTurns: 99 }), + ]); + + expect(withUngraded).toBe(100); + }); + + test("turnBudgetRateForTasks: an all-ungraded scope reports null, not 0%", () => { + expect( + turnBudgetRateForTasks([ + task({ status: "NOT_GRADED", expectedTurns: 10, visibleTurns: 4 }), + ]), + ).toBeNull(); + }); +}); diff --git a/evalboard/lib/__tests__/status-parity.test.ts b/evalboard/lib/__tests__/status-parity.test.ts new file mode 100644 index 00000000..0ade2830 --- /dev/null +++ b/evalboard/lib/__tests__/status-parity.test.ts @@ -0,0 +1,77 @@ +import { readFileSync } from "node:fs"; +import { dirname, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { describe, expect, test } from "vitest"; +import { statusCategory, type StatusCategory } from "../status"; + +// Drift guard: lib/status.ts is a hand-copied mirror of coder_eval's +// `FinalStatus.category` (src/coder_eval/models/enums.py). Python's side is +// guarded there by `assert set(_STATUS_CATEGORIES) == set(FinalStatus)`; this +// mirror was not, which is how NOT_GRADED came to be categorized as "unknown" +// and then counted as a failure by every rate helper in the app. +// +// status.test.ts covers the mapping, but it iterates a HAND-MAINTAINED record — +// so it can never fail when Python adds a tenth member. This file parses the +// Python table instead, following the lib/__tests__/pricing-parity.test.ts +// precedent, so the next status added upstream breaks the build here rather +// than silently rendering as grey "unknown" and inflating a denominator. + +const here = dirname(fileURLToPath(import.meta.url)); +const PY_PATH = resolve(here, "../../../src/coder_eval/models/enums.py"); + +// Match: `FinalStatus.SUCCESS: "succeeded",` inside _STATUS_CATEGORIES. +const ROW_RE = /FinalStatus\.([A-Z_]+):\s*"(succeeded|failed|error|ungraded)"/g; + +// Python's four buckets -> the TS union. The names differ on ONE member +// ("succeeded" vs "passed"), deliberately: "passed" is the word the UI uses. +// Spelled out here so a rename on either side is a failure, not a silent +// fall-through to "unknown". +const PY_TO_TS: Record = { + succeeded: "passed", + failed: "failed", + error: "error", + ungraded: "ungraded", +}; + +function parsePythonCategories(): Record { + const src = readFileSync(PY_PATH, "utf8"); + const start = src.indexOf("_STATUS_CATEGORIES"); + expect(start, "_STATUS_CATEGORIES not found in enums.py").toBeGreaterThan( + -1, + ); + // Bound the scan to that dict so _EXECUTION_FACT_STATUSES and the docstrings + // below it cannot contribute phantom rows. + const end = src.indexOf("\n}", start); + const table = src.slice(start, end === -1 ? undefined : end); + + const out: Record = {}; + for (const m of table.matchAll(ROW_RE)) { + out[m[1]] = PY_TO_TS[m[2]]; + } + return out; +} + +describe("status.ts mirrors coder_eval FinalStatus.category", () => { + const py = parsePythonCategories(); + + test("the Python table was actually parsed", () => { + // A regex that silently matches nothing would make every assertion + // below vacuous — the exact failure mode this file exists to prevent. + expect(Object.keys(py).length).toBeGreaterThanOrEqual(9); + expect(py.SUCCESS).toBe("passed"); + expect(py.NOT_GRADED).toBe("ungraded"); + }); + + test.each(Object.entries(parsePythonCategories()))( + "%s categorizes as %s on both sides", + (status, expected) => { + expect(statusCategory(status)).toBe(expected); + }, + ); + + test("no member falls through to unknown", () => { + for (const status of Object.keys(py)) { + expect(statusCategory(status)).not.toBe("unknown"); + } + }); +}); diff --git a/evalboard/lib/__tests__/status.test.ts b/evalboard/lib/__tests__/status.test.ts new file mode 100644 index 00000000..a91db0a6 --- /dev/null +++ b/evalboard/lib/__tests__/status.test.ts @@ -0,0 +1,70 @@ +import { describe, expect, it } from "vitest"; + +import { + isGraded, + isPassStatus, + statusCategory, + statusSortRank, + type StatusCategory, +} from "../status"; + +// This module mirrors coder_eval's FinalStatus.category (models/enums.py), which +// is guarded there by `assert set(_STATUS_CATEGORIES) == set(FinalStatus)`. The +// mirror had no test at all, which is how NOT_GRADED came to be categorized as +// "unknown" and then counted as a failure by every rate helper downstream. +const EVERY_FINAL_STATUS: Record = { + SUCCESS: "passed", + FAILURE: "failed", + ERROR: "error", + BUILD_FAILED: "error", + TIMEOUT: "failed", + MAX_TURNS_EXHAUSTED: "failed", + TOKEN_BUDGET_EXCEEDED: "failed", + COST_BUDGET_EXCEEDED: "failed", + NOT_GRADED: "ungraded", +}; + +describe("statusCategory", () => { + it.each(Object.entries(EVERY_FINAL_STATUS))( + "maps %s to %s", + (status, expected) => { + expect(statusCategory(status)).toBe(expected); + }, + ); + + it("treats a missing status as unknown, distinct from ungraded", () => { + expect(statusCategory(null)).toBe("unknown"); + expect(statusCategory(null)).not.toBe(statusCategory("NOT_GRADED")); + }); + + it("does not classify an ungraded row as a pass or a failure", () => { + // The whole point of the fourth category: folding it into either side + // of a rate misreports a run that was never scored. + expect(statusCategory("NOT_GRADED")).not.toBe("passed"); + expect(statusCategory("NOT_GRADED")).not.toBe("failed"); + expect(isPassStatus("NOT_GRADED")).toBe(false); + }); +}); + +describe("isGraded", () => { + it("is false only for an ungraded row", () => { + expect(isGraded("NOT_GRADED")).toBe(false); + for (const status of Object.keys(EVERY_FINAL_STATUS)) { + if (status === "NOT_GRADED") continue; + expect(isGraded(status)).toBe(true); + } + // A null status is "no row here", not "ran but unscored". + expect(isGraded(null)).toBe(true); + }); +}); + +describe("statusSortRank", () => { + it("sorts failures first, ungraded in the middle, passes last", () => { + expect(statusSortRank("FAILURE")).toBeLessThan( + statusSortRank("NOT_GRADED"), + ); + expect(statusSortRank("NOT_GRADED")).toBeLessThan( + statusSortRank("SUCCESS"), + ); + }); +}); diff --git a/evalboard/lib/__tests__/trends.test.ts b/evalboard/lib/__tests__/trends.test.ts index 1a039d95..79e310a9 100644 --- a/evalboard/lib/__tests__/trends.test.ts +++ b/evalboard/lib/__tests__/trends.test.ts @@ -225,3 +225,30 @@ describe("historyForTaskInner", () => { expect(byRun.r2).toBe(false); }); }); + +describe("aggregate — ungraded rows", () => { + // The parallel of the "mature skip" block above, for the fourth status + // category. The denominator changed here with nothing asserting it: an + // ungraded row must leave BOTH sides, or a `coder-eval execute` run drags + // every task's trend down as though it had failed. + test("leave both sides of the pass rate", () => { + const { trends } = aggregate([ + perRun("r1", [task({ status: "SUCCESS" })]), + perRun("r2", [task({ status: "NOT_GRADED" })]), + ]); + + expect(trends[0].totalRuns).toBe(1); + expect(trends[0].successRuns).toBe(1); + expect(trends[0].passRate).toBe(1); + }); + + test("a fully ungraded task reports no runs rather than a 0% pass rate", () => { + const { trends } = aggregate([ + perRun("r1", [task({ status: "NOT_GRADED" })]), + perRun("r2", [task({ status: "NOT_GRADED" })]), + ]); + + expect(trends[0].totalRuns).toBe(0); + expect(trends[0].successRuns).toBe(0); + }); +}); diff --git a/evalboard/lib/__tests__/watchlist.test.ts b/evalboard/lib/__tests__/watchlist.test.ts index c1614d39..5355f023 100644 --- a/evalboard/lib/__tests__/watchlist.test.ts +++ b/evalboard/lib/__tests__/watchlist.test.ts @@ -299,3 +299,35 @@ describe("empty window", () => { expect(data.windowSize).toBe(0); }); }); + +describe("ungraded rows", () => { + // The denominators in leaderboard() and attention() gained an `isGraded` + // filter with nothing asserting it. An ungraded row must leave BOTH sides: + // counted only in the denominator it reads as a failure, which is exactly + // what would put a `coder-eval execute` night at the top of the watchlist. + test("leave both sides of a skill's pass rate", () => { + const data = buildWatchlist([ + perRun("2026-01-02", [ + task({ taskId: "a", skill: "alpha", status: "SUCCESS" }), + task({ taskId: "b", skill: "alpha", status: "NOT_GRADED" }), + ]), + ]); + + expect(data.leaderboard).toEqual([ + { skill: "alpha", passRate: 1, outcomes: 1 }, + ]); + }); + + test("an all-ungraded skill raises no attention at all", () => { + const runs = Array.from({ length: 4 }, (_, i) => + perRun(`2026-01-0${4 - i}`, [ + task({ taskId: "a", skill: "unmeasured", status: "NOT_GRADED" }), + ]), + ); + + const { topAttention, leaderboard } = buildWatchlist(runs); + + expect(topAttention).toHaveLength(0); + expect(leaderboard).toEqual([]); + }); +}); diff --git a/evalboard/lib/overview.ts b/evalboard/lib/overview.ts index b3343b62..c28c7704 100644 --- a/evalboard/lib/overview.ts +++ b/evalboard/lib/overview.ts @@ -18,7 +18,7 @@ import { withinTurnBudget } from "./turns"; import { humanizeTaskId } from "./format"; import { mapWithConcurrency } from "./concurrency"; import { DEFAULT_HARNESS, normalizeHarness, orderHarnesses } from "./harness"; -import { isPassStatus } from "./status"; +import { isGraded, isPassStatus } from "./status"; import { taskCarriesRepoTag } from "./tags"; import type { Window } from "./reviews-types"; @@ -81,7 +81,14 @@ export function turnBudgetRateForTasks(tasks: RunOverviewTask[]): number | null for (const t of tasks) { const hasBudget = t.expectedTurns != null && t.expectedTurns >= 1; if (!hasBudget) continue; // unbudgeted tasks never count, success or fail - if (t.status !== "SUCCESS") { + // An ungraded row leaves BOTH sides, like every other rate in this app. + // A raw `!== "SUCCESS"` here put it in `eligible` and never in + // `withinBudget`, i.e. booked it as a budget MISS — so one + // `coder-eval execute` run published a measured-looking Turn Budget of + // 0%. `status` is a free string, so neither tsc nor assertNever could + // see it; the typed helper is the only thing that can. + if (!isGraded(t.status)) continue; + if (!isPassStatus(t.status)) { // A budgeted task that didn't succeed never stayed within budget. eligible += 1; continue; @@ -111,7 +118,10 @@ export function withinExpectedTimeRateForTasks( let eligible = 0; let within = 0; for (const t of tasks) { - if (t.status !== "SUCCESS" || t.matureSkipped) continue; + // Behaviourally identical to the old raw literal (a non-pass already + // left both sides here), converted so this file has ONE rule for what + // "passed" means rather than two spellings of it. + if (!isPassStatus(t.status) || t.matureSkipped) continue; const verdict = withinExpectedTime(t.durationSeconds, t.expectedSeconds); if (verdict === null) continue; // unscored, or no duration → can't judge eligible += 1; @@ -135,7 +145,7 @@ export function timePerPassedTaskForTasks( tasks: RunOverviewTask[], ): number | null { const executed = tasks.filter((t) => !t.matureSkipped); - const passed = executed.filter((t) => t.status === "SUCCESS").length; + const passed = executed.filter((t) => isPassStatus(t.status)).length; if (!passed) return null; const total = executed.reduce((a, t) => a + (t.durationSeconds ?? 0), 0); return total > 0 ? total / passed : null; @@ -171,6 +181,10 @@ export interface RunListingRow { // Unfiltered, they are whole-run totals. tasksSucceeded: number; tasksRun: number; + // Rows that were actually scored. An ungraded row (`coder-eval execute`) + // leaves BOTH sides of every rate, so this — not tasksRun — is the pass-rate + // denominator. Equals tasksRun on any graded run. + tasksGraded: number; totalCostUsd: number | null; taskDurationSeconds: number | null; // Run-level harness (coder-eval AgentKind) for the Harness column; null on @@ -189,6 +203,7 @@ export interface RunListingTotals { costPartial: boolean; // some matched runs had no cost (sum understates) tasksSucceeded: number; tasksRun: number; + tasksGraded: number; // the pass-rate denominator; see RunListingRow.tasksGraded durationSeconds: number | null; // null when no matched run recorded a duration durationPartial: boolean; } @@ -201,11 +216,13 @@ export function summarizeListing(rows: RunListingRow[]): RunListingTotals { let costRuns = 0; let tasksSucceeded = 0; let tasksRun = 0; + let tasksGraded = 0; let durationSeconds = 0; let durationRuns = 0; for (const r of rows) { tasksSucceeded += r.tasksSucceeded; tasksRun += r.tasksRun; + tasksGraded += r.tasksGraded; if (r.totalCostUsd != null) { costUsd += r.totalCostUsd; costRuns += 1; @@ -220,6 +237,7 @@ export function summarizeListing(rows: RunListingRow[]): RunListingTotals { costPartial: costRuns > 0 && costRuns < rows.length, tasksSucceeded, tasksRun, + tasksGraded, durationSeconds: durationRuns > 0 ? durationSeconds : null, durationPartial: durationRuns > 0 && durationRuns < rows.length, }; @@ -786,7 +804,11 @@ export async function getOverview( runId: id, timestamp: date.getTime(), harness: runHarness, - successRate: (row.tasksSucceeded / row.tasksRun) * 100, + // Ungraded rows leave both sides; a fully ungraded run contributes + // no point rather than a fabricated 0%. + successRate: row.tasksGraded + ? (row.tasksSucceeded / row.tasksGraded) * 100 + : null, turnBudgetRate: turnBudgetRateForTasks(scoped.tasks), withinExpectedTimeRate: withinExpectedTimeRateForTasks( scoped.tasks, @@ -830,11 +852,16 @@ export interface TagTaskRow { appearances: number; // Of `appearances`, how many were mature carry-forwards (not executed). matureSkips: number; - // `appearances - matureSkips`: the denominator behind passRate, carried on - // the row rather than re-derived by the renderer so the percentage and the - // caption that names its sample size can never describe different rules. + // Of `appearances`, how many ran but were never scored (`coder-eval + // execute`). Like a mature skip, such a row leaves BOTH sides of the rate. + ungraded: number; + // `appearances - matureSkips - ungraded`: the denominator behind passRate, + // carried on the row rather than re-derived by the renderer so the + // percentage and the caption that names its sample size can never describe + // different rules. executed: number; - // 0-100 over EXECUTED appearances only (appearances - matureSkips). + // 0-100 over MEASURED appearances only + // (appearances - matureSkips - ungraded). // null when nothing in the window actually ran, so the UI shows "—" // rather than a measured-looking 0% or 100%. passRate: number | null; @@ -893,6 +920,7 @@ export function buildTagTaskRows(perRun: PerRun[], tag: string): TagTaskRow[] { skill: string | null; appearances: number; matureSkips: number; + ungraded: number; executedPasses: number; latestRunId: string; latestStatus: string | null; @@ -939,6 +967,7 @@ export function buildTagTaskRows(perRun: PerRun[], tag: string): TagTaskRow[] { skill: t.skill, appearances: 0, matureSkips: 0, + ungraded: 0, executedPasses: 0, latestRunId: id, latestStatus: t.status, @@ -950,6 +979,11 @@ export function buildTagTaskRows(perRun: PerRun[], tag: string): TagTaskRow[] { entry.appearances += 1; if (t.matureSkipped) { entry.matureSkips += 1; + } else if (!isGraded(t.status)) { + // Never scored, so it is neither a pass nor a miss. Counted like + // a mature skip: out of BOTH sides of the rate below, which is + // computed as appearances - matureSkips - ungraded. + entry.ungraded += 1; } else if (isPassStatus(t.status)) { // lib/status.ts, not a raw "SUCCESS" literal: `status` is an // untyped string, and this page's pass rate must move with @@ -967,12 +1001,13 @@ export function buildTagTaskRows(perRun: PerRun[], tag: string): TagTaskRow[] { // `?? true` only satisfies Map.get's `| undefined`; it is not a real // "unknown ⇒ keep" case.) if (!(newestTagged.get(taskId) ?? true)) continue; - const executed = e.appearances - e.matureSkips; + const executed = e.appearances - e.matureSkips - e.ungraded; rows.push({ taskId, skill: e.skill, appearances: e.appearances, matureSkips: e.matureSkips, + ungraded: e.ungraded, executed, passRate: executed > 0 ? (e.executedPasses / executed) * 100 : null, latestStatus: e.latestStatus, @@ -1092,9 +1127,10 @@ function rowFromScoped( ): RunListingRow { return { id, - tasksSucceeded: scoped.tasks.filter((t) => t.status === "SUCCESS") + tasksSucceeded: scoped.tasks.filter((t) => isPassStatus(t.status)) .length, tasksRun: scoped.tasks.length, + tasksGraded: scoped.tasks.filter((t) => isGraded(t.status)).length, totalCostUsd: scoped.totalCostUsd, taskDurationSeconds: scoped.taskDurationSeconds, harness: harness ?? null, @@ -1204,9 +1240,11 @@ export function buildAdhocRows( id, title: title ?? null, startedAt: overview.startedAt ?? null, - tasksSucceeded: overview.tasks.filter((t) => t.status === "SUCCESS") - .length, + tasksSucceeded: overview.tasks.filter((t) => + isPassStatus(t.status), + ).length, tasksRun: overview.tasks.length, + tasksGraded: overview.tasks.filter((t) => isGraded(t.status)).length, totalCostUsd: overview.totalCostUsd, taskDurationSeconds: overview.taskDurationSeconds, // Harness is a main-table-only (internal) column; ad-hoc rows omit it. diff --git a/evalboard/lib/pills.tsx b/evalboard/lib/pills.tsx index 1ab93e04..25cd715d 100644 --- a/evalboard/lib/pills.tsx +++ b/evalboard/lib/pills.tsx @@ -1,4 +1,4 @@ -import { statusCategory } from "./status"; +import { assertNever, statusCategory, type StatusCategory } from "./status"; // Shown on the green "Mature" status pill for a task that was skipped this run // (5 consecutive passes → re-validated only on its weekly slot) and carried @@ -51,6 +51,23 @@ export function MaturePill() { ); } +// A switch with an assertNever default, so a new StatusCategory fails +// `tsc --noEmit` here instead of quietly falling into whichever branch the +// boolean expression happened to leave it in. +function isFailureCategory(cat: StatusCategory): boolean { + switch (cat) { + case "failed": + case "error": + return true; + case "passed": + case "ungraded": + case "unknown": + return false; + default: + return assertNever(cat); + } +} + export function StatusPill({ status, relabel = false, @@ -66,7 +83,7 @@ export function StatusPill({ // Flow-execution failures (Faulted/Failed) land in statusCategory's "failed" // bucket too. Only null/unknown stays grey. const cat = statusCategory(status); - const isFailure = !ok && (cat === "failed" || cat === "error"); + const isFailure = !ok && isFailureCategory(cat); // Narrower list drives the relabel-to-"Failed" text so specific statuses // (e.g. MAX_TURNS_EXHAUSTED) keep their raw label while still showing red. const fail = @@ -82,14 +99,20 @@ export function StatusPill({ ? "bg-red-50 text-red-700 border-red-200" : "bg-gray-50 text-gray-600 border-gray-200"; const raw = status ?? "—"; + // NOT_GRADED gets a human label like every other status. Without it the grid + // rendered the raw enum token while its neighbours read "Passed"/"Failed" — + // and `isFailure` is already false for it, so the pill was correctly grey + // and incorrectly labelled. const label = relabel && ok ? "Passed" : relabel && status === "TIMEOUT" ? "Timed out" - : relabel && fail - ? "Failed" - : raw; + : relabel && cat === "ungraded" + ? "Not graded" + : relabel && fail + ? "Failed" + : raw; return ( passed // ERROR / BUILD_FAILED -> error (BUILD_FAILED is an environment/setup failure) +// NOT_GRADED -> ungraded (`coder-eval execute`: ran, deliberately unscored) // anything else (FAILURE, TIMEOUT, MAX_TURNS_EXHAUSTED, …) -> failed // +// "ungraded" is its OWN member rather than being folded into "unknown". Folding +// it there looks safe — an ungraded row genuinely has no verdict — but every +// rate helper in this app is written as `if passed … else if error … else +// failed++`, so anything that is not a pass or an error is counted as a failure +// AND kept in the denominator. A clean `execute` run then renders as 0% pass, N +// failed. +// +// Widening this union does NOT by itself produce a type error at the consumers: +// most of them go through `isPassStatus` / `isGraded` rather than switching on +// StatusCategory, and the two that do switch used if/else chains with a final +// `else`. That is exactly why the first ungraded sweep missed lib/overview.ts +// entirely. `assertNever` below is the fix: a consumer that must handle every +// category exhaustively uses a `switch` with `default: return assertNever(c)`, +// and adding a member then fails `tsc --noEmit` at that site. +// // Note: this only categorizes coder_eval task statuses. UI status display // (e.g. StatusPill) also handles flow execution statuses like "Completed" // and "Faulted" and uses its own logic. import { taskVariantKey } from "./variants"; -export type StatusCategory = "passed" | "failed" | "error" | "unknown"; +export type StatusCategory = "passed" | "failed" | "error" | "ungraded" | "unknown"; export function statusCategory(status: string | null): StatusCategory { if (!status) return "unknown"; if (status === "SUCCESS") return "passed"; if (status === "ERROR" || status === "BUILD_FAILED") return "error"; + if (status === "NOT_GRADED") return "ungraded"; return "failed"; } +// Whether a row was measured at all. An ungraded row must leave BOTH sides of +// every rate — it is not a pass and not a failure, so counting it either way +// (or keeping it in a denominator) misreports a run that was never scored. +export function isGraded(status: string | null): boolean { + return statusCategory(status) !== "ungraded"; +} + // Whether a status is a pass (SUCCESS). The single predicate behind the // "a task passes if any replicate passed" rule. export function isPassStatus(status: string | null): boolean { @@ -35,16 +59,60 @@ export function perTaskPassCounts< >(rows: readonly T[]): Map { const m = new Map(); for (const r of rows) { + // Ungraded replicates leave BOTH sides, exactly as they do in every + // per-row rate: they are excluded from the count AND from the map, so a + // task whose replicates were all ungraded does not appear at all rather + // than appearing as "0 of N passed". Without this an + // `execute --repeats 2` run renders a red "0/2 ✓". + if (!isGraded(r.status)) continue; const k = taskVariantKey(r); m.set(k, (m.get(k) ?? 0) + (isPassStatus(r.status) ? 1 : 0)); } return m; } -// Default table sort: failures and errors first, unknowns next, passes last. +// Per (variant, task): how many replicates were GRADED. The denominator paired +// with perTaskPassCounts, so a badge reading "k/N ✓" never divides a graded +// numerator by an all-rows N. +export function perTaskGradedCounts< + T extends { taskId: string; variantId?: string | null; status: string | null }, +>(rows: readonly T[]): Map { + const m = new Map(); + for (const r of rows) { + if (!isGraded(r.status)) continue; + const k = taskVariantKey(r); + m.set(k, (m.get(k) ?? 0) + 1); + } + return m; +} + +// Compile-time exhaustiveness guard. Call it from a `switch`'s `default` arm +// over a StatusCategory: a new member then makes the argument non-`never` and +// `tsc --noEmit` fails at that site, which is what forces every consumer to be +// revisited instead of silently falling through an `else`. +export function assertNever(x: never): never { + throw new Error(`Unhandled status category: ${String(x)}`); +} + +// Default table sort: failures and errors first, ungraded/unknown next, passes +// last. A `switch` with `assertNever`, not the if/else-with-catch-all this +// file's own header calls out: the catch-all sorted the new "ungraded" bucket +// into the "unknown" rank by fall-through rather than by decision, and a sixth +// category would land there just as silently. export function statusSortRank(status: string | null): number { const c = statusCategory(status); - if (c === "failed" || c === "error") return 0; - if (c === "passed") return 2; - return 1; + switch (c) { + case "failed": + case "error": + return 0; + case "passed": + return 2; + case "ungraded": + case "unknown": + // Between the two: an ungraded row is not a failure to rank first, + // and not a pass to bury last. + return 1; + default: + return assertNever(c); + } } diff --git a/evalboard/lib/trends.ts b/evalboard/lib/trends.ts index 24128a8e..2145f735 100644 --- a/evalboard/lib/trends.ts +++ b/evalboard/lib/trends.ts @@ -12,6 +12,7 @@ import { } from "./overview"; import { DEFAULT_HARNESS } from "./harness"; import { DEFAULT_SOURCE, type Source } from "./sources"; +import { isGraded, isPassStatus } from "./status"; import { taskCarriesRepoTag } from "./tags"; import type { ComponentSha } from "./runs"; @@ -151,14 +152,17 @@ export function aggregate(perRun: PerRun[]): TrendsData { } if (!b.skill && t.skill) b.skill = t.skill; for (const tg of t.tags) b.tagSet.add(tg); - b.totalCount += 1; + // An ungraded row (`coder-eval execute`) was never scored, so it + // enters neither side of the pass rate. Counting it in totalCount + // alone would drag a task's trend down as if it had failed. + if (isGraded(t.status)) b.totalCount += 1; if (t.matureSkipped) b.matureSkips += 1; b.statuses.push({ runId: id, status: t.status, matureSkipped: t.matureSkipped ?? false, }); - if (t.status === "SUCCESS") { + if (isPassStatus(t.status)) { b.successCount += 1; // A mature task the nightly skipped still counts as a pass, but // it wasn't executed — its row carries 0 cost / 0 duration and diff --git a/evalboard/lib/watchlist.ts b/evalboard/lib/watchlist.ts index ae215881..f231ba12 100644 --- a/evalboard/lib/watchlist.ts +++ b/evalboard/lib/watchlist.ts @@ -11,6 +11,7 @@ import type { PerRun } from "./overview"; import type { RunOverviewTask } from "./runs"; +import { isGraded, isPassStatus } from "./status"; import { timeRatio } from "./timing"; import { turnRatio } from "./turns"; @@ -98,7 +99,10 @@ function runsNewestFirst(perRun: PerRun[]): LoadedRun[] { .map((r) => ({ id: r.id, tasks: r.overview!.tasks })); } -const isPass = (status: string | null) => status === "SUCCESS"; +// Delegates rather than restating `=== "SUCCESS"`: this file already had to +// learn about the ungraded bucket, and a second private spelling of "passed" +// is how one surface ends up disagreeing with the rest. +const isPass = (status: string | null) => isPassStatus(status); const clamp01 = (n: number) => (n < 0 ? 0 : n > 1 ? 1 : n); const mean = (xs: number[]) => @@ -116,7 +120,9 @@ function stdev(xs: number[]): number { function skillPassSeq(runs: LoadedRun[], skill: string): number[] { const seq: number[] = []; for (const run of runs) { - const ts = run.tasks.filter((t) => t.skill === skill); + const ts = run.tasks.filter( + (t) => t.skill === skill && isGraded(t.status), + ); if (ts.length === 0) continue; seq.push(ts.filter((t) => isPass(t.status)).length / ts.length); } @@ -137,6 +143,8 @@ export function leaderboard(runs: LoadedRun[]): LeaderboardRow[] { for (const run of runs) { for (const t of run.tasks) { if (!t.skill) continue; + // Ungraded rows leave both sides of the rate — see isGraded. + if (!isGraded(t.status)) continue; total.set(t.skill, (total.get(t.skill) ?? 0) + 1); if (isPass(t.status)) passed.set(t.skill, (passed.get(t.skill) ?? 0) + 1); @@ -202,6 +210,8 @@ export function attention(runs: LoadedRun[]): AttentionRow[] { const ts = run.tasks.filter((t) => t.skill === skill); if (ts.length > 0) appeared++; for (const t of ts) { + // Ungraded rows leave both sides of the rate — see isGraded. + if (!isGraded(t.status)) continue; outcomes++; taskIds.add(t.taskId); if (isPass(t.status)) passes++; @@ -210,6 +220,14 @@ export function attention(runs: LoadedRun[]): AttentionRow[] { } } if (appeared < floor) continue; + // Nothing measured ⇒ nothing to say. Without this, a skill whose rows + // were all ungraded fell into `outcomes ? … : 0` below and scored + // passRate 0 / failRate 1 — the maximum FAIL_WEIGHT — so a + // `coder-eval execute` night would put its most-run skill at the TOP of + // an exec-triage hero, described as chronically failing. The same + // "never measured" vs "measured and scored zero" confusion CE049 exists + // to prevent on the Python side. + if (outcomes === 0) continue; const passRate = outcomes ? passes / outcomes : 0; const failRate = 1 - passRate; diff --git a/plugins/coder-eval/reference/run-layout.md b/plugins/coder-eval/reference/run-layout.md index a080dedb..2d4a5e65 100644 --- a/plugins/coder-eval/reference/run-layout.md +++ b/plugins/coder-eval/reference/run-layout.md @@ -11,6 +11,7 @@ runs/////{task.json, task.log, artifacts/} - `` — zero-padded replicate index (e.g. `00`, `01`). - `task.json` — the persisted per-replicate result (the consumer contract; carries the large `iterations` array — still accepted under its former name `turns` when reading, but not what current runs write). - `task.json.malformed` — present only on the docker degrade path: when an existing `task.json` fails to parse (schema skew from a stale `:latest` image, or a truncated/torn write), the docker runner moves the unparseable original aside to this sidecar and writes a synthetic `final_status=ERROR` `task.json` in its place. Diagnostic-only; `rglob("task.json")` consumers do not match it. +- `task.execute.json` — present only after a DETACHED grade (`coder-eval evaluate ` or `coder-eval run --resume` over a `NOT_GRADED` row). The pre-grade snapshot of `task.json`, written once and never overwritten by a later grade, so "this run was executed separately from grading" stays auditable. Diagnostic-only; `rglob("task.json")` consumers do not match it. - `task.log` — the human-readable task log; `artifacts/` — files the agent produced. **Scope-marker files** (used to detect what a given path represents): diff --git a/plugins/coder-eval/skills/analyze/SKILL.md b/plugins/coder-eval/skills/analyze/SKILL.md index 39e548ac..56b365b3 100644 --- a/plugins/coder-eval/skills/analyze/SKILL.md +++ b/plugins/coder-eval/skills/analyze/SKILL.md @@ -93,6 +93,17 @@ record is truncated or synthetic (the docker degrade path writes a `final_status for it. Note that `iterations` present but empty is a legitimate zero-turn record and not the same thing — test `has("iterations")`, never truthiness. +**A `final_status` of `NOT_GRADED` is not a failure — it is a row nothing measured.** +`coder-eval execute` runs the agent and deliberately skips every criterion, so such a +record has `weighted_score: null` and an EMPTY `success_criteria_results`. The recipe +above then reports `all_criteria_perfect: false` (the `length > 0` guard) with no +`failed_criteria`, which reads as "uniformly imperfect" for a run that was never scored. +Partition the rows first: exclude `NOT_GRADED` from BOTH sides of any pass rate or mean +score, count them separately, and say so in the report. If EVERY row is `NOT_GRADED`, +the answer is "this run was executed but not graded — grade it with `coder-eval run + --run-dir --resume` or `coder-eval evaluate ///00`", +not a table of zeros. + `error_excerpt` = the first ~200 characters of each failing criterion's `error`, falling back to `details`. Those are the only two free-text fields a criterion result carries, and which one is populated depends on the failure: `error` holds an exception, `details` diff --git a/pyproject.toml b/pyproject.toml index 5eea61f9..da637243 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -220,12 +220,42 @@ external = [ "CE011", "CE012", "CE013", + "CE014", + "CE015", + "CE016", + "CE017", "CE018", + "CE019", + "CE020", + "CE021", + "CE022", + "CE023", + "CE024", + "CE025", + "CE026", + "CE027", + "CE028", + "CE029", + "CE030", + "CE031", + "CE032", "CE033", + "CE034", "CE035", + "CE036", "CE037", "CE038", "CE039", + "CE043", + "CE045", + "CE046", + "CE047", + "CE048", + "CE049", + "CE050", + "CE051", + "CE052", + "CE053", ] # custom architectural lint rules (tests/lint/) [tool.ruff.lint.pylint] diff --git a/src/coder_eval/cli/__init__.py b/src/coder_eval/cli/__init__.py index 40eab01d..66eaddae 100644 --- a/src/coder_eval/cli/__init__.py +++ b/src/coder_eval/cli/__init__.py @@ -7,6 +7,7 @@ from .aggregate_command import aggregate_command from .console import console from .evaluate_command import evaluate_command +from .execute_command import execute_command from .plan_command import plan_command from .report_command import report_command from .run_command import run_command @@ -46,9 +47,10 @@ def main( Run 'coder-eval COMMAND --help' for help on a specific command. Available commands: - - run: Execute evaluation tasks + - run: Execute evaluation tasks and grade them + - execute: Execute evaluation tasks WITHOUT grading them - plan: Validate task files (dry-run) - - evaluate: Run criteria against a directory without an agent + - evaluate: Grade a directory against a task, or re-grade a finished run - report: Display or export evaluation reports - aggregate: Rebuild run.json/run.md from finalized task.json files """ @@ -75,6 +77,7 @@ def main( # emits a CoderEval.Cli. event (Status/DurationMs/ErrorType) on completion; # functools.wraps preserves the signature so Typer still parses each command's flags. app.command(name="run")(track_command("run")(run_command)) +app.command(name="execute")(track_command("execute")(execute_command)) app.command(name="plan")(track_command("plan")(plan_command)) app.command(name="evaluate")(track_command("evaluate")(evaluate_command)) app.command(name="report")(track_command("report")(report_command)) diff --git a/src/coder_eval/cli/aggregate_command.py b/src/coder_eval/cli/aggregate_command.py index 9e3e1187..942c6f0e 100644 --- a/src/coder_eval/cli/aggregate_command.py +++ b/src/coder_eval/cli/aggregate_command.py @@ -78,11 +78,15 @@ def aggregate_command( skipped_tasks=skipped, ) write_run_summary(summary, out_dir) - console.print( - f"[green][OK][/green] Aggregated {summary.tasks_run} task(s) " - + f"({summary.tasks_succeeded} ok / {summary.tasks_failed} fail / {summary.tasks_error} err) " - + f"→ {out_dir / 'run.json'}" - ) + # The fourth bucket is named here too. `coder-eval aggregate ` is the + # step right after `coder-eval execute`, so an ungraded run is the FIRST + # thing this line renders — and without the term it reads + # "Aggregated 12 task(s) (0 ok / 0 fail / 0 err)", four numbers that no + # longer sum to tasks_run with nothing on screen to say where the rest went. + counts = f"{summary.tasks_succeeded} ok / {summary.tasks_failed} fail / {summary.tasks_error} err" + if summary.tasks_not_graded: + counts += f" / {summary.tasks_not_graded} not graded" + console.print(f"[green][OK][/green] Aggregated {summary.tasks_run} task(s) ({counts}) → {out_dir / 'run.json'}") console.print( "[dim]Note: run-level summary only — per-suite (suite.json/suite.md) and " + "experiment (experiment.json/experiment.md) rollups are not rebuilt.[/dim]" diff --git a/src/coder_eval/cli/evaluate_command.py b/src/coder_eval/cli/evaluate_command.py index 1e62d886..d2603f7f 100644 --- a/src/coder_eval/cli/evaluate_command.py +++ b/src/coder_eval/cli/evaluate_command.py @@ -1,38 +1,258 @@ -"""Evaluate command - run criteria against a directory without an agent.""" +"""Evaluate command - run criteria against a directory or re-grade a finished run.""" + +from __future__ import annotations import asyncio +import logging +from dataclasses import dataclass from pathlib import Path import typer +from ..evaluation.judge_persistence import TASK_JSON_TRANSCRIPT_EXCLUDE from ..logging_config import setup_logging from ..models import ( AgentKind, EvaluationResult, FinalStatus, PreservationMode, + TaskDefinition, TemplateDirSource, parse_agent_config, ) +from ..orchestration.regrade import ( + RegradeError, + back_up_pre_grade_record, + default_workspace, + grading_sandbox_config, + load_prior_result, + regrade_in_place, + restore_pre_grade_record, + stamp_host_grading, + task_from_prior, + verify_reference_unchanged, +) from ..orchestration.task_loader import load_task from ..orchestrator import Orchestrator +from ..path_utils import PRE_GRADE_JSON_FILENAME, TASK_JSON_FILENAME, write_text_atomic from ..sandbox import Sandbox from .console import console +from .evaluate_target import ( + EvaluateMode, + EvaluateTarget, + EvaluateTargetError, + as_work_dir, + resolve_evaluate_target, +) from .run_helpers import prepare_run_directory +logger = logging.getLogger(__name__) + + +def resolve_grade_in_place(target: EvaluateTarget, in_place: bool | None) -> bool: + """Whether this grade runs in the target directory or in a copy of it. + + In-place is the default for a run directory: that workspace is the run's own + output, and copying it filters build artifacts (``node_modules``, ``dist``, + ``.venv``) out of the grade, so a criterion reading them fails as a copying + artifact rather than as a verdict. A plain work directory defaults to + copying, because criteria can mutate the target and it is the user's own + tree. + + A function rather than an expression because two places need the answer and + one of them — the recorded-shell refusal — changes what the command is + willing to execute. A restated copy of the rule silently stops matching the + moment the default moves. + """ + return in_place if in_place is not None else (target.mode is EvaluateMode.RUN_DIR) + + +@dataclass(frozen=True) +class _ResolvedInputs: + """Everything the two positionals + ``--workspace`` decide, resolved once.""" + + target: EvaluateTarget + task: TaskDefinition + source_yaml: str + work_dir: Path + task_file: Path | None + prior: EvaluationResult | None + + +def _resolve_inputs( + task_or_run_dir: Path, + work_dir: Path | None, + workspace: Path | None, + *, + allow_recorded_commands: bool, + in_place: bool | None, +) -> _ResolvedInputs: + """Turn the CLI positionals into a task, a workspace, and (maybe) a prior run. + + Split out of the command because it is where both shapes converge: after this + the rest of ``evaluate`` is one code path regardless of which form was used. + """ + try: + target = resolve_evaluate_target(task_or_run_dir, work_dir) + except EvaluateTargetError as e: + raise typer.BadParameter(str(e)) from e + + if workspace is not None and target.mode is not EvaluateMode.RUN_DIR: + raise typer.BadParameter( + "--workspace applies to a run directory only; in the two-argument form the " + + "directory to grade is already the second argument." + ) + + try: + return _resolve_run_dir_or_work_dir( + target, workspace, allow_recorded_commands=allow_recorded_commands, in_place=in_place + ) + except RegradeError as e: + # The shared core raises a plain exception (orchestration/ must not + # depend on the CLI layer, CE004); surface it as a CLI error here. + raise typer.BadParameter(str(e)) from e + + +def _resolve_run_dir_or_work_dir( + target: EvaluateTarget, + workspace: Path | None, + *, + allow_recorded_commands: bool, + in_place: bool | None, +) -> _ResolvedInputs: + """The mode-specific half of :func:`_resolve_inputs`.""" + prior: EvaluationResult | None = None + if target.mode is EvaluateMode.RUN_DIR and target.task_file is not None: + # `is_run_dir` is a filename probe, so a plain work directory holding an + # unrelated file called task.json lands here. That would abort the + # pre-existing `evaluate ` form on a pydantic wall the + # user can only escape by renaming their own file. The task file is + # already in hand, so fall back to the shape they asked for. + try: + load_prior_result(target.target) + except RegradeError as e: + logger.warning( + "%s holds a %s that is not a readable run record (%s); grading it as a plain " + "work directory.", + target.target, + TASK_JSON_FILENAME, + e, + ) + target = as_work_dir(target) + + if target.mode is EvaluateMode.RUN_DIR: + prior = load_prior_result(target.target) + if target.task_file is not None: + task, source_yaml = load_task(target.task_file) + console.print(f"[dim]Grading with {target.task_file} (overrides the run's recorded config).[/dim]") + else: + # pre_run/post_run and the sandbox's installers both run only on the + # --copy path (an adopted workspace must not have its hooks re-run + # over the agent's deliverables, and `adopt` installs nothing), so + # in place they are not capabilities the run dir can reach. + # + # Derived through the SAME function `run_evaluation` uses, not + # restated. This value decides whether recorded shell is refused, so + # a second copy of the rule would keep answering the old question if + # the default ever moved — and silently stop covering commands that + # then do run. + setup_will_run = not resolve_grade_in_place(target, in_place) + task, source_yaml = task_from_prior( + prior, + target.target, + allow_recorded_commands=allow_recorded_commands, + include_setup_phase=setup_will_run, + ) + work_dir = workspace or default_workspace(target.target, prior) + recorded_source = prior.task_config.source_file if prior.task_config else None + task_file = target.task_file or (Path(recorded_source) if recorded_source else None) + else: + assert target.task_file is not None # guaranteed by resolve_evaluate_target + task_file = target.task_file + try: + task, source_yaml = load_task(task_file) + except Exception as e: + console.print(f"[red]✗ Failed to load task:[/red] {e}") + raise typer.Exit(1) from e + work_dir = target.target + + if not work_dir.is_dir(): + console.print(f"[red]✗ Work directory is not a directory:[/red] {work_dir}") + raise typer.Exit(1) + + # Evaluate-only mode bypasses experiment resolution + CLI overrides, so + # `agent` may be None or `agent.type` may be unset for tasks that defer + # those to the experiment / CLI layers. The orchestrator only uses + # `agent.type` for result labeling here (no agent is created), so a + # default is safe. + if task.agent is None: + task.agent = parse_agent_config(type=AgentKind.CLAUDE_CODE) + elif task.agent.type is None: + task.agent = parse_agent_config(**{**task.agent.model_dump(exclude_unset=True), "type": AgentKind.CLAUDE_CODE}) + + if prior is not None: + verify_reference_unchanged(prior, task, task_file) + # Snapshot the ungraded record BEFORE anything grades. Taking it inside + # _write_back instead would capture an ALREADY-GRADED record whenever + # --run-dir points at the target run dir (the orchestrator writes there + # first), destroying the very evidence the copy exists to preserve. + back_up_pre_grade_record(target.target) + + return _ResolvedInputs( + target=target, + task=task, + source_yaml=source_yaml, + work_dir=work_dir, + task_file=task_file, + prior=prior, + ) + + +def _replicate_index_of(run_dir: Path) -> int: + """Recover the replicate index a run directory encodes in its leaf name. + + Preservation lays runs out as ``///``. Hardcoding 0 + would relabel every replicate but the first as replicate 0. + """ + try: + return int(run_dir.name) + except ValueError: + return 0 + + def evaluate_command( - task_file: Path = typer.Argument( # noqa: B008 + task_or_run_dir: Path = typer.Argument( # noqa: B008 ..., - help="Path to task YAML file", + # One metavar per positional, so the usage line reads as Click renders + # it. A composite metavar on the first ("[TASK_FILE] TARGET") plus an + # empty one on the second produced `[TASK_FILE] TARGET []`. + metavar="TASK_FILE_OR_RUN_DIR", + help="Task YAML file, or (when it is the only argument) a finished run directory.", exists=True, ), - work_dir: Path = typer.Argument( # noqa: B008 - ..., - help="Directory containing the code to evaluate", - exists=True, - file_okay=False, - dir_okay=True, + work_dir: Path | None = typer.Argument( # noqa: B008 + None, + # No metavar="" here: an empty one leaks a bare `[]` into both the usage + # line and the arguments table. The first positional's metavar already + # spells out the two shapes. + help="Directory containing the code to evaluate. Omit when TASK_FILE is a run directory.", + ), + workspace: Path | None = typer.Option( # noqa: B008 + None, + "--workspace", + help=( + "Grade this directory instead of the run's own artifacts. Run-directory mode only (e.g. a verifier's /app)." + ), + ), + in_place: bool | None = typer.Option( + None, + "--in-place/--copy", + help=( + "Grade the workspace where it is, or copy it into a fresh sandbox first. " + "Default: in-place for a run directory, copy for a plain work directory. " + "Copying filters build output (node_modules, dist, build, .venv), so a " + "criterion that reads those needs --in-place." + ), ), verbose: bool = typer.Option( False, @@ -44,48 +264,108 @@ def evaluate_command( True, "--preserve/--no-preserve", "-p/-P", - help="Move sandbox artifacts to run directory (default: preserve). The temp sandbox is always removed.", + help=( + "Move sandbox artifacts to run directory (default: preserve). The temp sandbox is " + "always removed. Ignored when grading in place (the default for a run directory) — " + "an adopted directory is never moved or deleted." + ), + ), + allow_recorded_commands: bool = typer.Option( + False, + "--allow-recorded-commands", + help=( + "Accept shell commands (run_command criteria, pre_run/post_run) rebuilt from the run " + "directory's own task.json. A run directory is a shareable artifact, so its recorded " + "config is untrusted input; without this, grading refuses rather than running it here." + ), + ), + allow_host_grading: bool = typer.Option( + False, + "--allow-host-grading", + help=( + "Grade a `driver: docker` run on this host. Grading cannot start a container, so the " + "criteria run against a filesystem that lacks the container's paths and toolchain — " + "scores may differ from the run. Such rows are stamped graded_on_host." + ), ), run_dir: Path | None = typer.Option( # noqa: B008 None, "--run-dir", - help="Custom run directory (default: auto-generated timestamped directory in runs/)", + help="Where the graded task.json lands (default: auto-generated timestamped directory in runs/)", ), ) -> None: - """Evaluate criteria against a directory without running an agent. + """Evaluate criteria against a directory, or re-grade a finished run. - Runs the success criteria defined in a task against a work directory. - Artifacts are saved to a run directory when --preserve is used. + Two shapes, told apart by whether the target holds a task.json: - Examples: + \b + Grade a directory against a task (no agent runs): coder-eval evaluate tasks/hello.yaml ./my_solution - coder-eval evaluate tasks/test.yaml /path/to/code --preserve - coder-eval evaluate tasks/test.yaml /path/to/code --run-dir ./my_eval_run + + \b + Re-grade a finished run — including one produced by `coder-eval execute`, + which leaves every task NOT_GRADED. The run's own task.json supplies the + resolved config AND the trajectory, so criteria that read the agent's tool + calls score exactly as they would have during the run: + coder-eval execute tasks/hello.yaml --run-dir ./r + coder-eval evaluate ./r/default/hello/00 + + \b + Iterate on criteria against a run you already paid for, by passing a task + file over a run directory (its trajectory and workspace are still used): + coder-eval evaluate tasks/hello.edited.yaml ./r/default/hello/00 """ - setup_logging(verbose=verbose) + run_evaluation( + task_or_run_dir=task_or_run_dir, + work_dir=work_dir, + workspace=workspace, + in_place=in_place, + verbose=verbose, + preserve=preserve, + allow_recorded_commands=allow_recorded_commands, + allow_host_grading=allow_host_grading, + run_dir=run_dir, + ) - console.print("\n[bold]Evaluating Criteria[/bold]\n") - try: - task, source_yaml = load_task(task_file) - except Exception as e: - console.print(f"[red]✗ Failed to load task:[/red] {e}") - raise typer.Exit(1) from e +def run_evaluation( + *, + task_or_run_dir: Path, + work_dir: Path | None = None, + workspace: Path | None = None, + in_place: bool | None = None, + verbose: bool = False, + preserve: bool = True, + allow_recorded_commands: bool = False, + allow_host_grading: bool = False, + run_dir: Path | None = None, +) -> None: + """The body of ``coder-eval evaluate``, with real Python defaults. - # Evaluate-only mode bypasses experiment resolution + CLI overrides, so - # `agent` may be None or `agent.type` may be unset for tasks that defer - # those to the experiment / CLI layers. The orchestrator only uses - # `agent.type` for result labeling here (no agent is created), so a - # default is safe. + Split from the Typer signature so it is directly callable: invoking a Typer + command function in-process hands every unspecified option an ``OptionInfo`` + sentinel rather than its default, which silently turns ``in_place=None`` into + a truthy object. Callers (tests, and any library use) call this instead. + """ + setup_logging(verbose=verbose) - if task.agent is None: - task.agent = parse_agent_config(type=AgentKind.CLAUDE_CODE) - elif task.agent.type is None: - task.agent = parse_agent_config(**{**task.agent.model_dump(exclude_unset=True), "type": AgentKind.CLAUDE_CODE}) + console.print("\n[bold]Evaluating Criteria[/bold]\n") - if not work_dir.is_dir(): - console.print(f"[red]✗ Work directory is not a directory:[/red] {work_dir}") - raise typer.Exit(1) + inputs = _resolve_inputs( + task_or_run_dir, + work_dir, + workspace, + allow_recorded_commands=allow_recorded_commands, + in_place=in_place, + ) + task = inputs.task + source_yaml = inputs.source_yaml + graded_dir = inputs.work_dir + task_file = inputs.task_file + prior = inputs.prior + target = inputs.target + + grade_in_place = resolve_grade_in_place(target, in_place) try: prepared_run_dir = prepare_run_directory(run_dir) @@ -93,31 +373,120 @@ def evaluate_command( console.print(f"[red]✗ Failed to prepare run directory:[/red] {e}") raise typer.Exit(1) from e - # Build a sandbox pre-loaded with the work_dir contents, then run evaluate-only - sandbox_config = task.sandbox.model_copy(deep=True) - template_source = TemplateDirSource(path=str(work_dir.resolve())) - if sandbox_config.template_sources: - sandbox_config.template_sources = [template_source, *sandbox_config.template_sources] - else: - sandbox_config.template_sources = [template_source] + try: + sandbox_config = grading_sandbox_config(task, allow_host_grading=allow_host_grading) + except RegradeError as e: + console.print(f"[red]✗ {e}[/red]") + raise typer.Exit(1) from e + if not grade_in_place: + # Copy path: preload the sandbox with the work dir as a template source. + template_source = TemplateDirSource(path=str(graded_dir.resolve())) + sandbox_config.template_sources = [template_source, *(sandbox_config.template_sources or [])] - task_dir = task_file.parent.resolve() + task_dir = task_file.parent.resolve() if task_file is not None else None sandbox = Sandbox(sandbox_config, task_id=task.task_id, task_dir=task_dir) async def _setup_and_run() -> EvaluationResult: - await asyncio.to_thread(sandbox.setup) + if grade_in_place and prior is not None: + # Delegate to the shared re-grade core. Restating its body here is + # how this path and `run --resume` came to differ (replicate_index, + # error semantics) while CLAUDE.md called regrade.py the single + # implementation — two copies of "how to re-grade" drift into two + # verdicts for the same run. + return await regrade_in_place( + task=task, + prior=prior, + workspace=graded_dir, + run_dir=prepared_run_dir, + task_file=task_file, + source_yaml=source_yaml, + variant_id=prior.variant_id, + replicate_index=_replicate_index_of(target.target), + allow_host_grading=allow_host_grading, + ) + if grade_in_place: + await asyncio.to_thread(sandbox.adopt, graded_dir) + else: + await asyncio.to_thread(sandbox.setup) orchestrator = Orchestrator( task=task, run_dir=prepared_run_dir, - preservation_mode=PreservationMode.MOVE_ON_WRITE if preserve else PreservationMode.NONE, + # An adopted directory is the caller's; never move or delete it. + preservation_mode=( + PreservationMode.NONE + if grade_in_place + else (PreservationMode.MOVE_ON_WRITE if preserve else PreservationMode.NONE) + ), task_file=task_file, sandbox=sandbox, - variant_id="evaluate", + variant_id=prior.variant_id if prior is not None else "evaluate", + replicate_index=_replicate_index_of(target.target), source_yaml=source_yaml, + prior_result=prior, ) - return await orchestrator.run() + graded = await orchestrator.run() + # Same stamp the delegating branch gets from `regrade_in_place`. Line 357 + # above accepted the docker→host downgrade for THIS branch too, and + # CLAUDE.md, the user guide and CE051's own noqa all state the stamp as + # unconditional — so `evaluate --copy --allow-host-grading` + # was writing an unstamped host verdict that nothing downstream could + # tell apart from a container-graded one. + stamp_host_grading(graded, task) + return graded result = asyncio.run(_setup_and_run()) + _report_and_exit(result, task=task, prior=prior, target=target, prepared_run_dir=prepared_run_dir) + + +def _report_and_exit( + result: EvaluationResult, + *, + task: TaskDefinition, + prior: EvaluationResult | None, + target: EvaluateTarget, + prepared_run_dir: Path, +) -> None: + """Render the graded row, write it back, and choose the exit code. + + Split out of ``run_evaluation`` because it answers a different question — + what to TELL the operator about a result that already exists — and because + the two together grew past the function-size bound the moment the inherited + status handling landed. Always raises ``typer.Exit``. + """ + + # BEFORE the count guard below. A grading crash returns a populated ERROR + # result with an EMPTY criteria list (Orchestrator.run() converts internal + # failures into a result rather than raising), so the count check fires + # first and the user is told only "Result count mismatch: got 0, expected 2" + # — the real error is never printed, and the "still re-gradeable" notice is + # unreachable on exactly the path it was written for. + # Whether the terminal status describes THIS pass or was carried over from + # the run being graded. `Orchestrator._terminal_status` preserves a prior + # execution fact (TIMEOUT / ERROR / BUILD_FAILED / a budget stop) because + # grading may not overturn it — so reading `result.final_status` as this + # pass's own outcome misreports both arms below. It made a preserved ERROR + # print the ORIGINAL run's crash message as though grading had crashed, + # claim the row was "left ungraded" (it was not — the restored record still + # reads ERROR), and throw away a verdict that had just been computed at + # 1.000; and it made a preserved TIMEOUT exit 0 under "All criteria passed", + # so a CI wrapper reading the exit code goes green on a row run.json counts + # as failed. + inherited = prior is not None and prior.final_status.is_execution_fact + + if result.final_status is FinalStatus.ERROR and not inherited: + console.print(f"\n[red]✗ Evaluation error: {result.error_message}[/red]") + if prior is not None: + # A grading-time crash (a failing checker, an unreachable judge) is + # not a verdict about the run. Leaving ERROR on disk would replace a + # perfectly re-gradeable NOT_GRADED row with one BOTH commands treat + # as permanently complete, so the run could never be graded again + # without hand-restoring task.execute.json. + restore_pre_grade_record(target.target) + console.print( + f"[yellow]⚠[/] Grading errored; {target.target / TASK_JSON_FILENAME} is left " + + "ungraded so the run stays re-gradeable." + ) + raise typer.Exit(1) # Display results console.print("[bold]Criteria Results:[/bold]\n") @@ -162,12 +531,61 @@ async def _setup_and_run() -> EvaluationResult: if result.sandbox_path: console.print(f"[dim]Artifacts: {result.sandbox_path}[/dim]") - if result.final_status == FinalStatus.ERROR: - console.print(f"\n[red]✗ Evaluation error: {result.error_message}[/red]") + if prior is not None: + console.print( + f"[dim]Re-graded {prior.final_status.value} → {result.final_status.value} " + + f"over {len(result.iterations)} recorded turn(s).[/dim]" + ) + _write_back(target.target, result) + + if result.final_status.is_execution_fact: + # The criteria tally is real and worth printing — it is why the table + # above still renders — but it is not the row's outcome. run.json will + # count this row under its preserved status, and the exit code must + # agree with run.json rather than with the tally. + console.print( + f"\n[red]Criteria: {passed}/{total} passed, but the run itself ended as " + + f"{result.final_status.value} — grading cannot overturn that.[/red]" + ) raise typer.Exit(1) - elif failed == 0: + if failed == 0: console.print("\n[green]All criteria passed! ✓[/green]") raise typer.Exit(0) else: console.print(f"\n[red]{failed} criterion/criteria failed.[/red]") raise typer.Exit(1) + + +def _write_back(run_dir: Path, result: EvaluationResult) -> None: + """Replace the graded run's ``task.json`` with the verdict, keeping a copy of the original. + + Updating in place is what makes the rest of the toolchain free: plain + ``coder-eval aggregate `` then rebuilds ``run.json`` from these rows + with no new code, and every report and evalboard view reads the graded row. + + The pre-grade original is kept alongside as ``task.execute.json`` so the + ungraded record is auditable — the write is not a silent overwrite of the + only evidence that the run was executed separately. + """ + target = run_dir / TASK_JSON_FILENAME + backup = run_dir / PRE_GRADE_JSON_FILENAME + if target.is_symlink(): + # A run directory is a shareable artifact, so its task.json is untrusted + # input. Following a symlink here turns `evaluate ` into an + # arbitrary-file-overwrite primitive on the grader's host. + console.print(f"[yellow]⚠[/] {target} is a symlink; refusing to write through it.") + return + try: + # Atomic, matching the orchestrator's own task.json writer: a torn write + # here makes the row parse as malformed, which a later --resume reads as + # "not complete" and re-pays for the agent. + write_text_atomic(target, result.model_dump_json(indent=2, exclude=TASK_JSON_TRANSCRIPT_EXCLUDE)) + except OSError as e: + # Never fail the grade over the write-back: the verdict was computed and + # already printed, and the fresh run dir holds its own task.json. + console.print(f"[yellow]⚠[/] Could not update {target}: {e}") + return + console.print( + f"[dim]Updated {target} (original kept as {backup.name}); " + + "run `coder-eval aggregate` to refresh run.json.[/dim]" + ) diff --git a/src/coder_eval/cli/evaluate_target.py b/src/coder_eval/cli/evaluate_target.py new file mode 100644 index 00000000..b16e8857 --- /dev/null +++ b/src/coder_eval/cli/evaluate_target.py @@ -0,0 +1,120 @@ +"""Shape detection for ``coder-eval evaluate``'s positional arguments. + +``evaluate`` accepts two shapes that look alike on the command line: + + coder-eval evaluate tasks/hello.yaml ./my_solution # grade a directory + coder-eval evaluate runs/latest/default/hello/00 # re-grade a finished run + +Both are "a task and a place", but the second carries its own task config and +trajectory inside ``task.json``, so nothing needs to be supplied twice. Rather +than adding a ``--run-dir-mode`` flag the caller has to remember, the shape is +detected from the target: a directory holding ``task.json`` is a run directory. + +The logic lives here, apart from the Typer command, because it is pure — it does +one ``is_file`` probe and otherwise just maps arguments to a decision — so it can +be tested exhaustively without building sandboxes or invoking a CLI runner. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import StrEnum +from pathlib import Path + +from ..path_utils import TASK_JSON_FILENAME + + +class EvaluateMode(StrEnum): + """Which of the two shapes the caller asked for.""" + + WORK_DIR = "work_dir" + """Grade a plain directory against a task file. The original behavior.""" + + RUN_DIR = "run_dir" + """Re-grade a finished run: its task.json supplies config and trajectory.""" + + +@dataclass(frozen=True) +class EvaluateTarget: + """The resolved intent behind ``evaluate``'s positional arguments.""" + + mode: EvaluateMode + target: Path + """The run directory (RUN_DIR) or the directory to grade (WORK_DIR).""" + + task_file: Path | None + """Explicit task YAML. Required in WORK_DIR mode; an optional override in RUN_DIR mode.""" + + +class EvaluateTargetError(ValueError): + """The two positionals do not describe either supported shape.""" + + +def is_run_dir(path: Path) -> bool: + """Whether ``path`` is a finished task run directory (it holds ``task.json``).""" + return (path / TASK_JSON_FILENAME).is_file() + + +def resolve_evaluate_target(first: Path, second: Path | None) -> EvaluateTarget: + """Map ``evaluate``'s one-or-two positionals onto a mode + target. + + Args: + first: The first positional — a task file, or a run directory when it is + the only argument. + second: The second positional (the directory to grade), or None. + + Returns: + The resolved target. + + Raises: + EvaluateTargetError: If the arguments match neither shape. The message + always names what was passed and what to pass instead: a caller who + gets this wrong is one keystroke from the right command, and a bare + "invalid arguments" would not tell them which one. + """ + if second is None: + # One argument: only the run-dir shape is unambiguous. A lone task file + # names no place to grade, and a lone plain directory names no criteria. + if not first.is_dir(): + raise EvaluateTargetError( + f"{first} is not a directory. With a single argument, pass a finished run " + + f"directory (one containing {TASK_JSON_FILENAME}). To grade a directory against a " + + "task, pass both: coder-eval evaluate " + ) + if not is_run_dir(first): + raise EvaluateTargetError( + f"{first} holds no {TASK_JSON_FILENAME}, so it is not a run directory. Pass the task " + + f"file too: coder-eval evaluate {first}" + ) + return EvaluateTarget(mode=EvaluateMode.RUN_DIR, target=first, task_file=None) + + # Two arguments. The second is the place; the first is the task file. When + # that place turns out to be a run directory the caller is re-grading it with + # a DIFFERENT task file than the one it ran with — the "iterate on my + # criteria against an expensive run I already paid for" case, which is the + # main reason to keep `execute` and `evaluate` separate at all. Allow it, and + # let the caller be told which config won. + # + # This probe is a filename test, so a plain work directory that merely + # happens to contain a file called `task.json` is read as a run directory — + # and the pre-existing two-argument form would abort on a pydantic wall with + # no way to override it. It is not repaired here (this function is pure and + # cannot tell a real record from a namesake); the caller re-reads the record + # and falls back to WORK_DIR when it does not parse. See + # ``evaluate_command._resolve_run_dir_or_work_dir``. + mode = EvaluateMode.RUN_DIR if second.is_dir() and is_run_dir(second) else EvaluateMode.WORK_DIR + return EvaluateTarget(mode=mode, target=second, task_file=first) + + +def as_work_dir(target: EvaluateTarget) -> EvaluateTarget: + """Re-read a two-argument target as the plain work-directory shape. + + The escape hatch for the namesake ``task.json`` above. Only valid when a task + file was supplied, which is exactly the two-argument form. + """ + if target.task_file is None: + raise EvaluateTargetError( + f"{target.target} holds a {TASK_JSON_FILENAME} that is not a readable run record, and no " + + f"task file was given. Pass one: coder-eval evaluate {target.target}" + ) + return EvaluateTarget(mode=EvaluateMode.WORK_DIR, target=target.target, task_file=target.task_file) diff --git a/src/coder_eval/cli/execute_command.py b/src/coder_eval/cli/execute_command.py new file mode 100644 index 00000000..15bbdc2a --- /dev/null +++ b/src/coder_eval/cli/execute_command.py @@ -0,0 +1,236 @@ +"""Execute command - run evaluation tasks WITHOUT grading them. + +``coder-eval execute`` is ``coder-eval run`` with the grading half removed: the +sandbox is built, the agent runs, and the full trajectory is captured into the +usual ``task.json`` / ``run.json`` layout — but no success criterion is checked, +``weighted_score`` stays ``None``, and each row finalizes as +``FinalStatus.NOT_GRADED``. + +It exists so an *external* harness can own the verdict. The motivating case is +Harbor (Terminal-Bench 2.0), which builds its own container, calls coder-eval as +the agent, and grades with its own ``tests/test.sh``. Grading twice there would +be worse than not grading at all: coder-eval's verdict would be reported +alongside Harbor's without being the one that counts. + +Every flag on ``run`` is available here except ``--junit-xml``, which is a report +of verdicts and there are none. + +``--resume`` IS supported, because ``partition_for_resume`` now takes the +resuming command into account: a ``NOT_GRADED`` row owes ``execute`` nothing (it +finished executing) but owes ``run`` a grade, so ``run --resume`` grades those +rows in place rather than skipping them as "already complete". + +The command shares ``run``'s entire body (``run_command.run_pipeline``); only the +Typer signature is restated, because Typer builds its parser from the signature. +``tests/test_execute_command.py`` asserts the two signatures stay in step. +""" + +from pathlib import Path + +import click +import typer + +from ..models import PreservationMode +from .run_command import run_pipeline + + +def execute_command( + task_files: list[Path] | None = typer.Argument( # noqa: B008 + None, + help="Path(s) to task YAML file(s). Defaults to all tasks/ recursively.", + ), + preservation_mode: PreservationMode | None = typer.Option( # noqa: B008 + None, + "--preservation-mode", + help=( + "How to persist each task's sandbox: NONE (delete), MOVE_ON_WRITE " + "(run in a tempdir, move into run_dir/artifacts), or DIRECT_WRITE " + "(run directly in run_dir/artifacts). Default is driver-derived — " + "docker → DIRECT_WRITE, else MOVE_ON_WRITE. Explicit value always wins." + ), + ), + run_dir: Path | None = typer.Option( # noqa: B008 + None, + "--run-dir", + help="Custom run directory (default: auto-generated timestamped directory in runs/)", + ), + resume: bool = typer.Option( + False, + "--resume", + help=( + "Resume an interrupted execute: skip tasks already executed in --run-dir " + "and run only the rest. Requires --run-dir. A NOT_GRADED row counts as " + "done here (it finished executing); a later `coder-eval run --resume` on " + "the same directory grades those rows instead of skipping them." + ), + ), + max_parallel: int = typer.Option( + 1, + "--max-parallel", + "-j", + help="Maximum number of tasks to run concurrently (default: 1 = sequential)", + min=1, + ), + verbose: bool = typer.Option( + False, + "--verbose", + "-v", + help="Enable verbose (DEBUG level) logging", + ), + log_file: Path | None = typer.Option( # noqa: B008 + None, + "--log-file", + help="Log to file in addition to console", + ), + tags: str | None = typer.Option( + None, + "--tags", + "-t", + help="Only run tasks matching any of these tags (comma-separated, e.g., 'smoke,golden')", + ), + exclude_tags: str | None = typer.Option( + None, + "--exclude-tags", + help="Skip tasks matching any of these tags (comma-separated, e.g., 'example,integration')", + ), + include_skipped: bool = typer.Option( + False, + "--include-skipped", + help=( + "Also run tasks marked `skip: true` in their YAML. Off by default so the " + "nightly/CI keep excluding them; use for on-demand / local runs of " + "quarantined or opt-in tasks." + ), + ), + agent_type: str | None = typer.Option( + None, + "--type", + "-T", + help="Override agent type for all tasks (e.g. 'claude-code', 'codex', or a plugin kind)", + ), + model: str | None = typer.Option( + None, + "--model", + "-m", + help="Override agent model for all tasks (e.g., claude-sonnet-4-20250514)", + ), + stream: str | None = typer.Option( + None, + "--stream", + "-s", + click_type=click.Choice(["full", "minimal"], case_sensitive=False), + help="Stream LLM events to terminal: 'full' or 'minimal' (turn-level only). Disables progress bar.", + ), + backend: str | None = typer.Option( + None, + "--backend", + "-b", + click_type=click.Choice(["direct", "bedrock", "litellm"], case_sensitive=False), + help="API backend (default: from API_BACKEND env var)", + ), + experiment: Path | None = typer.Option( # noqa: B008 + None, + "--experiment", + "-e", + help="Experiment definition YAML (default: experiments/default.yaml)", + ), + sample: int | None = typer.Option( + None, + "--sample", + help=( + "For dataset-backed tasks, use a random N-row sample " + "(fixed seed: reproducible, unbiased across paths). Cheap dataset smoke-test." + ), + min=1, + ), + sample_per_stratum: int | None = typer.Option( + None, + "--sample-per-stratum", + help=( + "For dataset-backed tasks, keep up to N rows per stratum (stratify_field, " + "default expected_skill) — a stratified sample that overrides the task's " + "dataset.sample_per_stratum without editing the YAML. Ignored when --sample is set. " + "Nondeterministic (re-draws each run) unless the task sets dataset.sample_seed." + ), + min=1, + ), + repeats: int | None = typer.Option( + None, + "--repeats", + help="Run each (task, variant) N times. Overrides experiment/variant `repeats:`. Must be >=1.", + min=1, + ), + driver: str | None = typer.Option( + None, + "--driver", + click_type=click.Choice(["tempdir", "docker"], case_sensitive=False), + help="Override sandbox driver for all tasks. 'docker' runs each task in a fresh container.", + ), + set_overrides: list[str] = typer.Option( # noqa: B008 + [], + "--set", + "-D", + metavar="PATH=VALUE", + help=( + "Override any resolved task-config field under agent/run_limits/sandbox, " + "e.g. -D run_limits.max_turns=30 -D agent.permission_mode=plan " + "-D agent.sdk_options.effort=high -D sandbox.docker.network=none. " + "Repeatable. Validated against the schema. A path set by both an alias " + "and -D is an error; values are YAML-parsed (on/off/yes/no stay strings). " + "(--model and --driver are shorthand aliases for -D agent.model / " + "-D sandbox.driver.)" + ), + ), +) -> None: + """Run evaluation tasks WITHOUT checking their success criteria. + + Identical to `coder-eval run` except that nothing is graded: each task + executes, its full trajectory is captured to task.json, and the row + finalizes as NOT_GRADED with no weighted_score. Use it when an external + harness owns the verdict, or to separate an expensive agent run from + grading you want to iterate on afterwards. + + Grade the results afterwards with `coder-eval evaluate`. + + Execution failures still fail: a crash, timeout, or budget breach reports + ERROR / TIMEOUT / TOKEN_BUDGET_EXCEEDED and exits non-zero exactly as under + `run`. Only the verdict is withheld, never the facts of the run. + + Not supported here: --junit-xml (no verdicts to report), --allow-host-grading + (nothing is graded, so there is no host-grading decision to make — pass it to + `coder-eval run --resume` or `coder-eval evaluate` instead), and simulation + tasks (their turn-continuation logic reads criteria results). + + Examples: + + coder-eval execute tasks/hello_date.yaml + + coder-eval execute tasks/*.yaml --run-dir ./my-run --max-parallel 3 + """ + run_pipeline( + grade=False, + task_files=task_files, + preservation_mode=preservation_mode, + run_dir=run_dir, + resume=resume, + # Not exposed as a flag — a JUnit report reports verdicts, and there are none. + junit_xml=None, + # Same reason: nothing is graded here, so there is no host-grading choice. + allow_host_grading=False, + max_parallel=max_parallel, + verbose=verbose, + log_file=log_file, + tags=tags, + exclude_tags=exclude_tags, + include_skipped=include_skipped, + agent_type=agent_type, + model=model, + stream=stream, + backend=backend, + experiment=experiment, + sample=sample, + sample_per_stratum=sample_per_stratum, + repeats=repeats, + driver=driver, + set_overrides=set_overrides, + ) diff --git a/src/coder_eval/cli/plan_command.py b/src/coder_eval/cli/plan_command.py index 25318717..61d2da8b 100644 --- a/src/coder_eval/cli/plan_command.py +++ b/src/coder_eval/cli/plan_command.py @@ -47,6 +47,19 @@ def plan_command( coder-eval plan tasks/*.yaml coder-eval plan tasks/*.yaml -e experiments/model-comparison.yaml """ + run_plan(task_files=task_files, experiment=experiment) + + +def run_plan(*, task_files: list[Path] | None = None, experiment: Path | None = None) -> None: + """The body of ``coder-eval plan``, with real Python defaults. + + Split from the Typer signature for the same reason as ``run_pipeline`` / + ``run_evaluation``: calling a Typer command function in process hands every + unspecified option an ``OptionInfo`` sentinel rather than its default, and + the sentinel is truthy. The `isinstance(experiment, Path)` guard this + function used to need was that bug being papered over rather than fixed. + Callers (tests, library use) call this. Enforced by lint rule CE048. + """ # Default to discovering all tasks under tasks/ when none provided resolved_task_files = task_files if task_files else discover_default_tasks() @@ -64,7 +77,7 @@ def plan_command( from ..orchestration.run_limits import validate_run_limits # Always load experiment (defaults to experiments/default.yaml) - exp_path = experiment if isinstance(experiment, Path) else DEFAULT_EXPERIMENT_PATH + exp_path = experiment or DEFAULT_EXPERIMENT_PATH try: exp_def = load_experiment(exp_path) if exp_path == DEFAULT_EXPERIMENT_PATH: diff --git a/src/coder_eval/cli/report_command.py b/src/coder_eval/cli/report_command.py index 6c5fcfc4..a64d0cf7 100644 --- a/src/coder_eval/cli/report_command.py +++ b/src/coder_eval/cli/report_command.py @@ -6,6 +6,7 @@ from rich.markdown import Markdown from ..models import EvaluationResult +from ..path_utils import TASK_JSON_FILENAME from ..reports import ReportGenerator from ..reports_html import write_task_html from .console import console @@ -96,7 +97,7 @@ def _regenerate_html_reports(run_dir: Path, output_file: Path | None) -> None: report is written to that file. Otherwise each task.html is written next to its task.json. """ - task_json_paths = sorted(run_dir.rglob("task.json")) + task_json_paths = sorted(run_dir.rglob(TASK_JSON_FILENAME)) if not task_json_paths: console.print(f"[red]Error: no task.json files found under {run_dir}[/red]") raise typer.Exit(1) diff --git a/src/coder_eval/cli/run_command.py b/src/coder_eval/cli/run_command.py index 9b22929c..d3d718e8 100644 --- a/src/coder_eval/cli/run_command.py +++ b/src/coder_eval/cli/run_command.py @@ -8,6 +8,7 @@ import urllib.parse import urllib.request from collections.abc import Callable +from datetime import datetime from pathlib import Path from typing import Any @@ -17,7 +18,15 @@ from ..config import Settings, settings from ..logging_config import setup_logging -from ..models import PreservationMode, ResolvedTask, RunSummary, TaskResult +from ..models import ( + AgentKind, + EvaluationResult, + FinalStatus, + PreservationMode, + ResolvedTask, + RunSummary, + TaskResult, +) from ..orchestration.config import BatchRunConfig from ..path_utils import create_latest_symlink, format_task_log_id from ..streaming.callbacks import CompositeStreamCallback @@ -192,10 +201,27 @@ def run_command( "Resume an interrupted run: skip tasks already finalized in --run-dir and " "run only the rest, folding prior results into run.json. A task counts as " "finalized once it has ANY final status — including FAILED/ERROR — so resume " - "does NOT retry failures (delete a task's task.json to force a re-run). " + "does NOT retry failures (delete a task's task.json to force a re-run). The " + "one exception is a NOT_GRADED row left by `coder-eval execute`: `run` was " + "asked for a verdict, so those rows are GRADED in place against the " + "trajectory and workspace already on disk, without re-running the agent. " "Requires --run-dir. A config mismatch (model/backend/flags) is warned, not " "refused — the resumed tasks keep their original-config results, so the run " - "mixes configs; use a fresh --run-dir to keep configs separate." + "mixes configs; use a fresh --run-dir to keep configs separate. A dataset " + "task using nondeterministic stratified sampling needs dataset.sample_seed " + "(or --sample) to be resumable — otherwise the resume draws a different row " + "set and pays for the agent twice." + ), + ), + allow_host_grading: bool = typer.Option( + False, + "--allow-host-grading", + help=( + "When --resume grades a `driver: docker` row, grade it on this host anyway. " + "Grading cannot start a container, so such criteria run against a filesystem " + "lacking the container's paths and toolchain and may score differently than " + "the run did; those rows are stamped graded_on_host. Without this they are " + "refused and stay ungraded." ), ), max_parallel: int = typer.Option( @@ -354,9 +380,80 @@ def run_command( coder-eval run tasks/*.yaml --tags golden,basic --exclude-tags example """ + run_pipeline( + grade=True, + task_files=task_files, + preservation_mode=preservation_mode, + run_dir=run_dir, + resume=resume, + allow_host_grading=allow_host_grading, + max_parallel=max_parallel, + verbose=verbose, + log_file=log_file, + junit_xml=junit_xml, + tags=tags, + exclude_tags=exclude_tags, + include_skipped=include_skipped, + agent_type=agent_type, + model=model, + stream=stream, + backend=backend, + experiment=experiment, + sample=sample, + sample_per_stratum=sample_per_stratum, + repeats=repeats, + driver=driver, + set_overrides=set_overrides, + ) + + +def run_pipeline( + *, + grade: bool, + task_files: list[Path] | None, + preservation_mode: PreservationMode | None, + run_dir: Path | None, + resume: bool, + allow_host_grading: bool, + max_parallel: int, + verbose: bool, + log_file: Path | None, + junit_xml: Path | None, + tags: str | None, + exclude_tags: str | None, + include_skipped: bool, + agent_type: str | None, + model: str | None, + stream: str | None, + backend: str | None, + experiment: Path | None, + sample: int | None, + sample_per_stratum: int | None, + repeats: int | None, + driver: str | None, + set_overrides: list[str], +) -> None: + """The shared body of ``coder-eval run`` and ``coder-eval execute``. + + Everything below the Typer signature is identical for both commands; the only + difference is ``grade``, which decides whether success criteria are checked + (``run``) or the trajectory is captured and left unscored (``execute``). Both + commands are pure flag-parsing wrappers over this function, so a behavior + change can never apply to one and miss the other. + """ # --resume needs an explicit run dir to resume into (auto-generated dirs are always fresh). if resume and run_dir is None: raise typer.BadParameter("--resume requires --run-dir pointing at the run to continue.") + # --allow-host-grading only reaches anything from inside the `if resume:` + # branch below, so without --resume it parsed, was accepted, and did nothing + # at all — no warning, no error. The sibling mode-scoped flag on the same + # feature (`evaluate --workspace`) hard-errors on exactly this misuse; two + # new flags behaving differently for one user mistake is the inconsistency. + if allow_host_grading and not resume: + raise typer.BadParameter( + "--allow-host-grading applies to --resume only (it decides how an executed-but-ungraded " + + "row is graded). A fresh `run` grades inside the driver the task asks for." + ) # Parse tag filters include_tags = {t.strip() for t in tags.split(",") if t.strip()} if tags else None @@ -412,8 +509,10 @@ def run_command( repeats=repeats, verbose=verbose, resume=resume, + allow_host_grading=allow_host_grading, include_skipped=include_skipped, junit_xml=junit_xml, + grade=grade, ) ) except KeyboardInterrupt: @@ -437,8 +536,10 @@ async def _run_all_tasks( repeats: int | None = None, verbose: bool = False, resume: bool = False, + allow_host_grading: bool = False, include_skipped: bool = False, junit_xml: Path | None = None, + grade: bool = True, ) -> None: """Async entry point for running all tasks (optionally in parallel). @@ -459,6 +560,7 @@ async def _run_all_tasks( experiment_path: Optional path to experiment YAML (default: experiments/default.yaml) junit_xml: Optional path to write a JUnit XML report to, after the run summary is persisted and before the failure exit-code gate. + grade: False for `coder-eval execute` — run and capture, score nothing. """ # Prepare run directory run_dir = prepare_run_directory(run_dir) @@ -484,6 +586,7 @@ async def _run_all_tasks( repeats=repeats, verbose=verbose, include_skipped=include_skipped, + grade=grade, ) from ..telemetry import flush_telemetry, track_event @@ -500,6 +603,7 @@ async def _run_all_tasks( "StreamMode": stream_mode or "none", "Resume": resume, "ExperimentProvided": experiment_path is not None, + "Grade": grade, }, ) @@ -513,7 +617,14 @@ async def _run_all_tasks( try: # Always run through experiment layer (defaults to experiments/default.yaml) summary, failed_suite_gates = await _run_with_experiment( - all_task_files, config, experiment_path, stream_mode, max_parallel, resume=resume + all_task_files, + config, + experiment_path, + stream_mode, + max_parallel, + resume=resume, + grade=grade, + allow_host_grading=allow_host_grading, ) # Aggregate task logs into run.log @@ -540,7 +651,14 @@ async def _run_all_tasks( flush_telemetry() # Exit with non-zero code if any tasks failed, errored, or any suite failed its thresholds. - if summary.tasks_failed > 0 or summary.tasks_error > 0 or failed_suite_gates > 0: + # + # An ungraded row counts too, but only under `run`: `run` was asked for a + # verdict and did not produce one (the grade crashed, or --resume could not + # grade the row), which is a failure of the command even though the row is + # neither `failed` nor `error`. Under `execute` an ungraded row is the + # expected outcome for every task, so it must not fail the command. + ungraded_but_asked_to_grade = grade and summary.tasks_not_graded > 0 + if summary.tasks_failed > 0 or summary.tasks_error > 0 or failed_suite_gates > 0 or ungraded_but_asked_to_grade: raise typer.Exit(1) @@ -593,6 +711,179 @@ def _on_task_complete(result: Any) -> None: return result +def _unreadable_row_placeholder(rt: ResolvedTask, error: Exception) -> EvaluationResult: + """A minimal ungraded row for a task whose recorded result cannot be read. + + Exists so an unreadable row stays IN ``run.json`` and in + ``tasks_not_graded`` — the counter the exit gate reads. Dropping it silently + made a resume that graded nothing exit 0. + """ + return EvaluationResult( + task_id=rt.task.task_id, + task_description=rt.task.description, + variant_id=rt.variant_id, + agent_type=str(rt.task.agent.type) if rt.task.agent and rt.task.agent.type else AgentKind.NONE.value, + started_at=datetime.now(), + final_status=FinalStatus.NOT_GRADED, + iteration_count=0, + error_message=f"Grading failed during --resume: the recorded result could not be read ({error})", + ) + + +async def _grade_resumed_tasks( + to_grade: list[ResolvedTask], *, allow_host_grading: bool = False +) -> list[tuple[ResolvedTask, TaskResult]]: + """Grade the rows ``coder-eval execute`` left NOT_GRADED, in place. + + Each task's trajectory and workspace are already on disk, so this runs the + criteria against them instead of re-running the agent — that reuse is the + whole reason to split ``execute`` from ``run``. + + The task config comes from ``rt.task`` (this run's own 5-layer resolution), + not from the recorded one: ``--resume`` re-resolves the same task files, and + a config that drifted since the execute is already surfaced by the run + fingerprint warning above. + + A task that cannot be graded is reported and folded back in with its ORIGINAL + ungraded result, so one bad row neither aborts the resume nor silently + vanishes from run.json — it stays visible as ``tasks_not_graded``, with the + reason on its ``error_message``. That covers three shapes: a helper raising, + a row too broken to read at all (skipped entirely — there is nothing to fold + back), and a re-grade that returns ``FinalStatus.ERROR``, which + ``Orchestrator.run()`` produces INSTEAD of raising and which would otherwise + make the row permanently un-regradeable. + + Returns the graded rows; the caller's exit gate fails the command whenever + any row is still ungraded, so a resume that graded nothing never exits 0. + """ + from ..orchestration.regrade import ( + RegradeError, + back_up_pre_grade_record, + default_workspace, + load_prior_result, + regrade_in_place, + restore_pre_grade_record, + ) + + graded: list[tuple[ResolvedTask, TaskResult]] = [] + for rt in to_grade: + # Inside the try: an unreadable row must skip like any other grading + # failure. Outside it, one bad task.json propagates out of the loop and + # aborts the whole resume BEFORE run_batch, so none of the `to_run` + # tasks execute either — the opposite of "one bad row never aborts". + prior: EvaluationResult | None = None + try: + prior = load_prior_result(rt.run_dir) + # The reference check lives inside regrade_in_place, so a caller + # cannot forget it. + workspace = default_workspace(rt.run_dir, prior) + # Preserve the ungraded record BEFORE the orchestrator overwrites + # task.json in this same directory. + back_up_pre_grade_record(rt.run_dir) + result = await regrade_in_place( + task=rt.task, + prior=prior, + workspace=workspace, + run_dir=rt.run_dir, + task_file=rt.task_file, + source_yaml=rt.source_yaml, + variant_id=rt.variant_id, + replicate_index=rt.replicate_index, + allow_host_grading=allow_host_grading, + ) + except (RegradeError, OSError, RuntimeError, ValueError) as e: + console.print(f"[yellow]⚠[/] Could not grade {rt.task.task_id}: {e}") + if prior is None: + # The row could not even be read, so there is no recorded result + # to fold back — but dropping it entirely removes it from + # run.json AND from `tasks_not_graded`, which is what the exit + # gate counts, so a resume whose rows were all unreadable would + # report success. Stand in a minimal ungraded row instead: it + # keeps the task visible and keeps the command non-zero. + result = _unreadable_row_placeholder(rt, e) + else: + # Stamp the reason onto the row. Without it the failure survives + # only in this console line: the folded-back result keeps the + # execute phase's empty error_message, so run.json, the reports + # and CI show an ungraded row with no explanation. + result = prior + result.error_message = f"Grading failed during --resume: {e}" + else: + if result.final_status is FinalStatus.ERROR: + # An orchestrator-level grading crash is not a verdict about the + # run. Orchestrator.run() converts internal failures into a + # populated ERROR result rather than raising, so without this the + # `except` above never sees them and the ERROR row replaces a + # perfectly re-gradeable NOT_GRADED one — and ERROR is "complete" + # for both commands, so the row could never be graded again. + # + # Fixing the in-memory result is only half of it: _finalize_result + # already wrote the ERROR task.json into this same directory + # before returning, so run.json would say NOT_GRADED while the + # row on disk says ERROR — and the on-disk one is what a later + # --resume reads. Put the pre-grade record back. + restore_pre_grade_record(rt.run_dir) + console.print( + f"[yellow]⚠[/] Grading {rt.task.task_id} errored ({result.error_message}); " + + "keeping the ungraded row so it stays re-gradeable." + ) + prior.error_message = f"Grading errored during --resume: {result.error_message}" + result = prior + graded.append( + ( + rt, + TaskResult( + task_id=rt.task.task_id, + variant_id=rt.variant_id, + result=result, + duration=result.duration_seconds or 0.0, + suite_id=rt.task.suite_id, + row_id=rt.task.row_id, + replicate_index=rt.replicate_index, + ), + ) + ) + return graded + + +async def _apply_resume( + resolved: list[ResolvedTask], *, grade: bool, allow_host_grading: bool +) -> tuple[list[ResolvedTask], list[TaskResult], list[ResolvedTask]]: + """Split a resumed run into what still needs running, and what is carried in. + + Extracted from ``_run_with_experiment``, which answers several questions at + once; this one answers "what does the resume still owe?" and is where the + grading half of the answer lives. + + Returns ``(to_run, prior_results, prior_resolved)``. ``resolved`` itself is + NOT narrowed — the suite rollups downstream need every task, run or not. + """ + from ..orchestration.batch import clear_rerun_artifacts, partition_for_resume + + part = partition_for_resume(resolved, grade=grade) + prior_results = list(part.prior_results) + prior_resolved = list(part.prior_resolved) + # A re-run task re-executes from scratch, so any leftover artifacts (only + # DIRECT_WRITE writes them live; a container killed mid-run leaves partials) + # are stale and could let a file-based criterion pass on the old output. + # to_grade is deliberately NOT cleared: its artifacts are the run's output + # and the very thing being graded. + cleared = clear_rerun_artifacts(part.to_run) + console.print( + f"[cyan]↻ Resume:[/] {len(prior_results)} task(s) already complete, " + + f"running {len(part.to_run)} remaining" + + (f", grading {len(part.to_grade)} executed-but-ungraded" if part.to_grade else "") + + (f" (cleared {cleared} stale artifact dir(s))" if cleared else "") + ) + # Grade the rows `execute` left behind, reusing the trajectory and workspace + # already on disk rather than paying for the agent twice. Folded in as + # prior_results so the summary covers them like any other. + for rt, tr in await _grade_resumed_tasks(part.to_grade, allow_host_grading=allow_host_grading): + prior_results.append(tr) + prior_resolved.append(rt) + return part.to_run, prior_results, prior_resolved + + async def _run_with_experiment( all_task_files: list[Path], config: BatchRunConfig, @@ -600,6 +891,8 @@ async def _run_with_experiment( stream_mode: str | None, max_parallel: int, resume: bool = False, + grade: bool = True, + allow_host_grading: bool = False, ) -> tuple[RunSummary, int]: """Run tasks through the experiment resolution layer. @@ -619,10 +912,8 @@ async def _run_with_experiment( RunSummary with aggregated results. """ from ..orchestration.batch import ( - clear_rerun_artifacts, compute_run_fingerprint, fingerprint_diff, - partition_for_resume, read_run_fingerprint, run_batch, write_run_fingerprint, @@ -670,6 +961,23 @@ async def _run_with_experiment( except ValueError as e: raise typer.BadParameter(str(e)) from e + # Simulation tasks are rejected under `execute`, not silently degraded. The + # dialog loop reads criteria results to decide whether to keep talking, so an + # ungraded dialog would quietly change its own stopping behavior and produce a + # trajectory that is not the one `run` would have produced. Rejecting is a + # config error (exit 2), and it names the offending tasks. + if not grade: + simulated = sorted( + rt.task.task_id for rt in resolved if rt.task.simulation is not None and rt.task.simulation.enabled + ) + if simulated: + raise typer.BadParameter( + "`coder-eval execute` does not support simulation tasks (their turn-continuation " + + "logic depends on criteria results): " + + ", ".join(simulated) + + ". Use `coder-eval run` for these." + ) + if skipped: console.print( f"[yellow]⚠[/] {len(skipped)} task file(s) skipped " @@ -705,15 +1013,8 @@ async def _run_with_experiment( prior_results: list[TaskResult] = [] prior_resolved: list[ResolvedTask] = [] if resume: - to_run, prior_results, prior_resolved = partition_for_resume(resolved) - # A re-run task re-executes from scratch, so any leftover artifacts (only - # DIRECT_WRITE writes them live; a container killed mid-run leaves partials) - # are stale and could let a file-based criterion pass on the old output. - cleared = clear_rerun_artifacts(to_run) - console.print( - f"[cyan]↻ Resume:[/] {len(prior_results)} task(s) already complete, " - + f"running {len(to_run)} remaining" - + (f" (cleared {cleared} stale artifact dir(s))" if cleared else "") + to_run, prior_results, prior_resolved = await _apply_resume( + resolved, grade=grade, allow_host_grading=allow_host_grading ) # Print execution mode @@ -745,6 +1046,12 @@ async def _run_with_experiment( # Per-suite pass-rate rollups for dataset-backed tasks (no-op when none were used). # Pass `resolved` through so suite_thresholds on each criterion can be evaluated. + # Skipped entirely under `execute`: a rollup aggregates per-criterion results, + # and there are none — running it would gate a suite on an empty aggregate and + # report a threshold failure for a run that was never measured. + if not grade: + return summary, 0 + from ..reports import write_suite_rollups rollups = write_suite_rollups(config.run_dir, task_results, resolved_tasks=resolved) diff --git a/src/coder_eval/cli/run_helpers.py b/src/coder_eval/cli/run_helpers.py index 9ef304cf..3c5b74bc 100644 --- a/src/coder_eval/cli/run_helpers.py +++ b/src/coder_eval/cli/run_helpers.py @@ -132,7 +132,23 @@ def print_execution_summary(run_dir: Path, summary: RunSummary) -> None: summary: Run execution summary """ console.print(f"\n[bold green]Run complete:[/bold green] {run_dir}") - console.print(f"[bold]Results:[/bold] {summary.tasks_succeeded}/{summary.tasks_run} succeeded") + # An ungraded run has no pass rate to report — printing "0/N succeeded" for a + # clean `coder-eval execute` reads as a total failure. Report what actually + # happened instead, and keep the graded line for whatever WAS graded. + if summary.tasks_not_graded: + console.print(f"[bold]Results:[/bold] {summary.tasks_not_graded}/{summary.tasks_run} executed, not graded") + # Point at the run-dir form, not `evaluate `: the + # two-argument shape grades a bare directory with NO trajectory, so + # command_executed / skill_triggered / trajectory-reading judges score + # differently from what `run` would have produced. The run-dir form + # restores the trajectory AND the resolved config. + console.print(f"[dim]Grade later: uv run coder-eval run --run-dir {run_dir} --resume[/dim]") + console.print("[dim] or: uv run coder-eval evaluate ///00[/dim]") + if summary.tasks_graded or not summary.tasks_not_graded: + # The `or not ...` keeps the pre-existing "0/0 succeeded" line for an + # empty run: without it a run with no tasks at all prints no Results + # line whatsoever, since both counters are falsy. + console.print(f"[bold]Results:[/bold] {summary.tasks_succeeded}/{summary.tasks_graded} succeeded") console.print(f"[dim]View report: open {run_dir / 'experiment.md'}[/dim]") console.print(f"[dim]View report: uv run coder-eval report {run_dir}[/dim]") diff --git a/src/coder_eval/cli/run_task_internal_command.py b/src/coder_eval/cli/run_task_internal_command.py index 3157e4a7..ac3c2bb2 100644 --- a/src/coder_eval/cli/run_task_internal_command.py +++ b/src/coder_eval/cli/run_task_internal_command.py @@ -34,6 +34,7 @@ CONTAINER_TASK_DIR, ConfigLineageEntry, PreservationMode, + SandboxConfig, ) from coder_eval.orchestration.task_loader import load_task @@ -51,6 +52,91 @@ def heartbeat_is_alive(current: str, last_counter: str, current_mtime: float, la return bool(current and current != last_counter) or current_mtime > last_mtime +def _arm_host_heartbeat_watchdog(output_dir: Path) -> None: + """Start the orphan-container reaper, but only inside the container. + + A module-level function rather than an inline block so the only + process-lethal code in this command sits behind one named, testable seam + instead of being a side effect of the command body. + """ + # Start the host-heartbeat watchdog: if the host process dies + # ungracefully (SIGKILL, Claude-Code Escape, crash) before it can + # `docker kill` us, the heartbeat file in output_dir goes stale and + # we self-exit -- otherwise the container would keep burning LLM + # budget orphaned. Daemon thread so it doesn't block normal shutdown. + import os as _os + import threading + import time + + # ARMED ONLY INSIDE THE CONTAINER, and not even defined outside one. The + # watchdog's whole authority is `os._exit(137)` on the process it runs in, + # and the only process that may be reaped that way is the container's own + # disposable main -- there is no container to orphan anywhere else, so + # outside one the thread can do nothing but harm. It did: a test invoked + # this command in-process (legitimately -- the command must refuse a + # malformed context.json, and proving that means calling it) and the pytest + # worker inherited the thread, which found no heartbeat and 40s later exited + # the worker mid-way through an unrelated test file. It named a different + # test on each run and on each platform, carried no traceback, and took that + # worker's coverage data with it -- so the gate reported "65.13 < 80.00", + # naming neither the test nor the cause. + # + # Gated on CODER_EVAL_IN_CONTAINER (set by docker_runner on the container's + # argv), NOT on `driver`, for the same reason the reference-permission + # window is: this command rewrites `driver: docker` -> `tempdir` before + # building the in-container Orchestrator, so a driver-based gate would + # disarm itself on exactly the path that needs it. See + # `Sandbox.enforces_permission_windows`. + if _os.environ.get("CODER_EVAL_IN_CONTAINER") == "1": + + def _watch_host_heartbeat() -> None: + heartbeat = output_dir / HEARTBEAT_FILENAME + # Grace period for the host to write the first counter value. + time.sleep(HEARTBEAT_STALE_SECONDS) + last_counter = "" + last_mtime = 0.0 + last_change = time.monotonic() + while True: + try: + current = heartbeat.read_text(encoding="utf-8") + except (FileNotFoundError, OSError): + current = "" + try: + current_mtime = heartbeat.stat().st_mtime + except (FileNotFoundError, OSError): + current_mtime = 0.0 + now = time.monotonic() + if heartbeat_is_alive(current, last_counter, current_mtime, last_mtime): + last_counter = current + last_mtime = current_mtime + last_change = now + if now - last_change > HEARTBEAT_STALE_SECONDS: + logger.error( + "Host heartbeat stale (>%ss); exiting to reap orphan container.", + HEARTBEAT_STALE_SECONDS, + ) + # os._exit skips atexit and IO flushing, so the error line + # above would routinely be lost -- making a genuine + # stale-heartbeat suicide indistinguishable from an external + # SIGKILL in the archived logs. Flush best-effort first; + # never let a flush failure stop the exit. + import sys as _sys + + for _handler in logging.getLogger().handlers: + with contextlib.suppress(Exception): + _handler.flush() + with contextlib.suppress(Exception): + _sys.stdout.flush() + with contextlib.suppress(Exception): + _sys.stderr.flush() + _os._exit(137) + time.sleep(HEARTBEAT_STALE_SECONDS / 4) + + threading.Thread(target=_watch_host_heartbeat, daemon=True).start() + else: + logger.debug("Not in a container; host-heartbeat watchdog not armed.") + + def run_task_internal_command( input_dir: Path = typer.Option( # noqa: B008 Path(CONTAINER_INPUT_DIR), @@ -82,59 +168,7 @@ def run_task_internal_command( log_level = "DEBUG" if verbose else settings.log_level setup_logging(level=log_level) - # Start the host-heartbeat watchdog: if the host process dies - # ungracefully (SIGKILL, Claude-Code Escape, crash) before it can - # `docker kill` us, the heartbeat file in output_dir goes stale and - # we self-exit -- otherwise the container would keep burning LLM - # budget orphaned. Daemon thread so it doesn't block normal shutdown. - import os as _os - import threading - import time - - def _watch_host_heartbeat() -> None: - heartbeat = output_dir / HEARTBEAT_FILENAME - # Grace period for the host to write the first counter value. - time.sleep(HEARTBEAT_STALE_SECONDS) - last_counter = "" - last_mtime = 0.0 - last_change = time.monotonic() - while True: - try: - current = heartbeat.read_text(encoding="utf-8") - except (FileNotFoundError, OSError): - current = "" - try: - current_mtime = heartbeat.stat().st_mtime - except (FileNotFoundError, OSError): - current_mtime = 0.0 - now = time.monotonic() - if heartbeat_is_alive(current, last_counter, current_mtime, last_mtime): - last_counter = current - last_mtime = current_mtime - last_change = now - if now - last_change > HEARTBEAT_STALE_SECONDS: - logger.error( - "Host heartbeat stale (>%ss); exiting to reap orphan container.", - HEARTBEAT_STALE_SECONDS, - ) - # os._exit skips atexit and IO flushing, so the error line - # above would routinely be lost -- making a genuine - # stale-heartbeat suicide indistinguishable from an external - # SIGKILL in the archived logs. Flush best-effort first; - # never let a flush failure stop the exit. - import sys as _sys - - for _handler in logging.getLogger().handlers: - with contextlib.suppress(Exception): - _handler.flush() - with contextlib.suppress(Exception): - _sys.stdout.flush() - with contextlib.suppress(Exception): - _sys.stderr.flush() - _os._exit(137) - time.sleep(HEARTBEAT_STALE_SECONDS / 4) - - threading.Thread(target=_watch_host_heartbeat, daemon=True).start() + _arm_host_heartbeat_watchdog(output_dir) task_yaml = input_dir / "task.yaml" context_json = input_dir / "context.json" @@ -146,13 +180,37 @@ def _watch_host_heartbeat() -> None: raise typer.Exit(2) context = json.loads(context_json.read_text(encoding="utf-8")) - variant_id: str = context["variant_id"] - replicate_index: int = context.get("replicate_index", 0) + # Checked, not just annotated. `json.loads` returns `Any`, so pyright accepts + # `variant_id: str = context["variant_id"]` for a value that may be anything + # at all — the annotation reads like a guarantee and enforces nothing. A + # `"replicate_index": "00"` then reached `build_task_run_dir` typed as `int`. + # This is the host→container boundary; the comment below (about `grade` + # being the one raw value) was only true because these two looked checked. + variant_id = context["variant_id"] + if not isinstance(variant_id, str): + typer.echo(f"FATAL: context.json 'variant_id' must be a string, got {variant_id!r}", err=True) + raise typer.Exit(2) + replicate_index = context.get("replicate_index", 0) + if not isinstance(replicate_index, int) or isinstance(replicate_index, bool): + typer.echo(f"FATAL: context.json 'replicate_index' must be an integer, got {replicate_index!r}", err=True) + raise typer.Exit(2) # The host resolves the driver-derived default before dispatch; the container # obeys it verbatim. This command only ever runs inside the docker driver, so # a missing key falls back to the docker default (DIRECT_WRITE) — a deliberate # default, not version back-compat. preservation_mode = PreservationMode(context.get("preservation_mode", PreservationMode.DIRECT_WRITE.value)) + # `coder-eval run` vs `coder-eval execute`, decided host-side. Defaults to + # True (grade) so a host that predates `execute` — which never writes the + # key — keeps its exact behavior. + # Coerced, not annotated, like every other value crossing this boundary — + # `grade` was once the only raw one, so a hand-edited or older-format + # `"grade": "false"` arrived as a truthy str typed as bool and silently + # graded a run that asked not to be graded. + grade_raw = context.get("grade", True) + if not isinstance(grade_raw, bool): + typer.echo(f"FATAL: context.json 'grade' must be a boolean, got {grade_raw!r}", err=True) + raise typer.Exit(2) + grade: bool = grade_raw # Docker WORKDIR alignment: the host resolves the concrete WORKDIR # (config value / "auto" -> `docker inspect` / fallback) and forwards it here. # Absent -> None -> standard run_dir/artifacts workspace. @@ -179,7 +237,18 @@ def _watch_host_heartbeat() -> None: # We're already inside the container; another nested docker would be # both wrong and impossible (no docker CLI in image). if task.sandbox.driver == "docker": - task = task.model_copy(update={"sandbox": task.sandbox.model_copy(update={"driver": "tempdir"})}) + # noqa: CE051 — the ONE legitimate rewrite. We are already inside the + # container the docker driver asked for, so the isolation the driver + # names is present, not bypassed; a nested docker would be both wrong + # and impossible (no docker CLI in the image). + # Re-validated rather than `model_copy(update=...)`, matching its sibling + # `regrade.grading_sandbox_config`: `update` skips BOTH pydantic and + # pyright, so a typo would produce a SandboxConfig violating its own + # `Literal` and only surface far downstream. Two driver-rewrite sites + # landing in one change with two different levels of type safety is how + # the weaker one becomes the pattern people copy. + rewritten = SandboxConfig.model_validate({**task.sandbox.model_dump(), "driver": "tempdir"}) # noqa: CE051 + task = task.model_copy(update={"sandbox": rewritten}) output_dir.mkdir(parents=True, exist_ok=True) @@ -197,6 +266,7 @@ def _watch_host_heartbeat() -> None: config_lineage=config_lineage, replicate_index=replicate_index, workspace_dir=workspace_dir, + grade=grade, ) # Install the stdout-NDJSON stream callback so per-tool-call events diff --git a/src/coder_eval/isolation/docker_runner.py b/src/coder_eval/isolation/docker_runner.py index 31683d75..57bac3da 100644 --- a/src/coder_eval/isolation/docker_runner.py +++ b/src/coder_eval/isolation/docker_runner.py @@ -40,7 +40,13 @@ ResourceLimits, ) from coder_eval.orchestration.evaluation import resolve_host_reference_dir -from coder_eval.path_utils import REFERENCE_COPY_IGNORE, ignore_patterns_and_symlinks, rmtree_restrictive +from coder_eval.path_utils import ( + REFERENCE_COPY_IGNORE, + TASK_JSON_FILENAME, + ignore_patterns_and_symlinks, + rmtree_restrictive, + write_text_atomic, +) from coder_eval.streaming.callbacks import safe_emit from coder_eval.streaming.wire import deserialize_event, has_prefix from coder_eval.utils import get_default_docker_image_tag @@ -540,11 +546,16 @@ def __init__( preservation_mode: PreservationMode = PreservationMode.DIRECT_WRITE, stream_callback: StreamCallback | None = None, verbose: bool = False, + grade: bool = True, ) -> None: self.rt = rt self.preservation_mode = preservation_mode self.stream_callback = stream_callback self.verbose = verbose + # Forwarded to the in-container orchestrator via context.json. It is a + # run-level decision made by the CLI, so it cannot be recovered from the + # staged task.yaml on the other side. + self.grade = grade # Set by _prepare_host_mounts: the tmp lean copy of ~/.claude that # _build_argv mounts read-write. None when there is no ~/.claude to # forward or the mount is opted out (CODER_EVAL_NO_CLAUDE_MOUNT). @@ -721,6 +732,9 @@ def _dump_task_yaml() -> str: "replicate_index": self.rt.replicate_index, "config_lineage": {k: v.model_dump(mode="json") for k, v in self.rt.config_lineage.items()}, "preservation_mode": self.preservation_mode.value, + # `coder-eval run` vs `coder-eval execute`. Not derivable from + # task.yaml on the container side (deliberately not a task field). + "grade": self.grade, "source_yaml": self.rt.source_yaml, # Docker WORKDIR alignment: concrete path the in-container # orchestrator runs at + captures out (None = standard workspace). @@ -837,7 +851,7 @@ async def _parse_result_or_raise(self, output_dir: Path, returncode: int, log_pa task.json and raise ``DockerRunError`` so the batch dispatcher records the failure as an ERROR-status result. """ - task_json = output_dir / "task.json" + task_json = output_dir / TASK_JSON_FILENAME if not await asyncio.to_thread(task_json.exists): # The container died before its orchestrator's `finally` could # write task.json (e.g. it was torn down by the cleanup above @@ -863,8 +877,57 @@ async def _parse_result_or_raise(self, output_dir: Path, returncode: int, log_pa # rather than crashing with an uncaught ValidationError/JSONDecodeError. raise await self._handle_malformed_task_json(task_json, log_path, exc) from exc self._warn_on_version_mismatch(result) + self._assert_grade_honored(result, task_json) return result + def _assert_grade_honored(self, result: EvaluationResult, task_json: Path | None = None) -> None: + """Fail loudly when `execute` came back with a graded verdict. + + ``grade`` crosses the boundary only through ``context.json``. An image + that predates ``execute`` ignores the unknown key and grades anyway, and + the image-version preflight only warns — so ``execute --driver docker`` + against a stale image would silently produce SUCCESS/FAILURE rows that + look like a normal graded run. Version skew must not change what a + command MEANS, so refuse the row rather than publish it. + + ``task_json`` is the on-disk record, quarantined before the raise. The + refusal used to be in-memory only, which left the graded ``task.json`` + sitting in the bind-mounted host run dir: a later + ``execute --resume`` read it back as a completed row (its category is + ``succeeded``, so the resume partition files it under prior results) and + plain ``aggregate`` folded it straight into ``run.json`` — publishing + exactly the row this guard declined to publish. Refusing in memory while + leaving contradictory bytes on disk is not a refusal. + """ + if self.grade: + return + # Keyed on EVIDENCE, not on the label. Exempting every execution-fact + # status let a stale image return a fully graded MAX_TURNS_EXHAUSTED row + # — criteria vector, weighted score and all — unchallenged, because the + # exemption exists for statuses a *fresh* image also produces, and a + # fresh one produces them with neither. The question is not "what status + # is this" but "did it grade". + graded_anyway = bool(result.success_criteria_results) or result.weighted_score is not None + if not graded_anyway and ( + result.final_status.is_execution_fact or result.final_status is FinalStatus.NOT_GRADED + ): + return + if task_json is not None: + # Same sidecar pattern as `_handle_malformed_task_json`. Best-effort: + # a failed move is logged, never masking the raise below. + sidecar = task_json.with_suffix(task_json.suffix + ".graded") + try: + os.replace(task_json, sidecar) + logger.warning("Quarantined the refused graded record to %s", sidecar) + except OSError as exc: + logger.warning("Could not quarantine %s: %s", task_json, exc) + raise DockerRunError( + "`coder-eval execute` asked the container not to grade, but it returned " + + f"{result.final_status.value} with {len(result.success_criteria_results)} criterion " + + "result(s). The runtime image predates `execute` and ignored the request; " + + "rebuild or pull a matching agent image." + ) + async def _handle_malformed_task_json(self, task_json: Path, log_path: Path, exc: ValueError) -> DockerRunError: """Degrade a present-but-malformed task.json; return the DockerRunError to raise. @@ -910,7 +973,9 @@ async def _record_build_failure(self, exc: DockerBuildError) -> None: await asyncio.to_thread(self.rt.run_dir.mkdir, parents=True, exist_ok=True) log_path = self.rt.run_dir / "docker.log" await asyncio.to_thread(log_path.write_text, exc.build_log or str(exc), encoding="utf-8") - await self._write_synthetic_task_json(self.rt.run_dir / "task.json", exc, status=FinalStatus.BUILD_FAILED) + await self._write_synthetic_task_json( + self.rt.run_dir / TASK_JSON_FILENAME, exc, status=FinalStatus.BUILD_FAILED + ) except OSError as io_exc: # pragma: no cover - defensive logger.warning("Failed to record build failure for %s: %s", self.rt.task.task_id, io_exc) @@ -939,9 +1004,15 @@ async def _write_synthetic_task_json( def _write() -> None: if target.exists(): return - tmp = target.with_suffix(target.suffix + ".synthetic.tmp") - tmp.write_text(result.model_dump_json(indent=2), encoding="utf-8") - os.replace(tmp, target) + # Through `write_text_atomic` like every other writer of this file. + # The hand-rolled tmp+replace here used `Path.write_text`, which + # FOLLOWS symlinks — so a pre-planted `task.json.synthetic.tmp` in a + # run directory (a shareable artifact, bind-mounted writable into the + # agent's own container) redirected this harness-privileged write to + # any path the grading user could reach. It also falsified the + # helper's "one writer, so the crash semantics cannot differ" claim, + # which is the property future readers rely on. + write_text_atomic(target, result.model_dump_json(indent=2)) try: await asyncio.to_thread(_write) diff --git a/src/coder_eval/models/enums.py b/src/coder_eval/models/enums.py index 0cba3650..f4586b6a 100644 --- a/src/coder_eval/models/enums.py +++ b/src/coder_eval/models/enums.py @@ -15,9 +15,16 @@ class FinalStatus(StrEnum): MAX_TURNS_EXHAUSTED = "MAX_TURNS_EXHAUSTED" TOKEN_BUDGET_EXCEEDED = "TOKEN_BUDGET_EXCEEDED" COST_BUDGET_EXCEEDED = "COST_BUDGET_EXCEEDED" + # `coder-eval execute` ran the agent but deliberately skipped grading, so + # there is no verdict to report. Distinct from FAILURE (which asserts the + # criteria were checked and did not pass) and from ERROR (which asserts + # something went wrong). Only SUCCESS/FAILURE collapse into it — every + # other member records an *execution* fact that still applies when the + # run is ungraded. + NOT_GRADED = "NOT_GRADED" @property - def category(self) -> Literal["succeeded", "failed", "error"]: + def category(self) -> Literal["succeeded", "failed", "error", "ungraded"]: """Classify this status into a reporting category (the SSOT for failed/succeeded/error).""" return _STATUS_CATEGORIES[self] @@ -26,12 +33,25 @@ def icon(self) -> str: """Single-character icon for reports and CLI output.""" return _STATUS_ICONS[self] + @property + def is_execution_fact(self) -> bool: + """True when this status records HOW THE RUN ENDED, not what grading decided. + + A detached grade (``evaluate `` / ``run --resume``) re-runs the + criteria over a trajectory it did not produce, so it may only move a row + between the three GRADING outcomes — ``NOT_GRADED`` -> ``SUCCESS`` / + ``FAILURE``. It must never launder a run that timed out, crashed, or blew + a budget into a pass: those statuses describe the agent phase, which the + grading pass neither repeated nor observed. + """ + return _EXECUTION_FACT_STATUSES[self] + # Every FinalStatus maps to exactly one reporting category, listed EXPLICITLY (no # catch-all default) so a newly-added status fails the assert below until it is # classified — rather than silently collapsing into "failed" (which would skew # reports AND the telemetry Category dimension). Mirrors the _STATUS_ICONS guard. -_STATUS_CATEGORIES: dict[FinalStatus, Literal["succeeded", "failed", "error"]] = { +_STATUS_CATEGORIES: dict[FinalStatus, Literal["succeeded", "failed", "error", "ungraded"]] = { FinalStatus.SUCCESS: "succeeded", FinalStatus.FAILURE: "failed", FinalStatus.ERROR: "error", @@ -43,6 +63,12 @@ def icon(self) -> str: FinalStatus.MAX_TURNS_EXHAUSTED: "failed", FinalStatus.TOKEN_BUDGET_EXCEEDED: "failed", FinalStatus.COST_BUDGET_EXCEEDED: "failed", + # A fourth category, not a fold into one of the three. Folding into + # "failed" would depress every pass rate; folding into "succeeded" would + # invent verdicts; folding into "error" would report a healthy run as + # broken. Reporting surfaces exclude it from BOTH the numerator and the + # denominator of a pass rate — an ungraded task was never measured. + FinalStatus.NOT_GRADED: "ungraded", } assert set(_STATUS_CATEGORIES) == set(FinalStatus), "Missing category for FinalStatus member" @@ -57,11 +83,31 @@ def icon(self) -> str: FinalStatus.MAX_TURNS_EXHAUSTED: "M", FinalStatus.TOKEN_BUDGET_EXCEEDED: "#", FinalStatus.COST_BUDGET_EXCEEDED: "$", + FinalStatus.NOT_GRADED: "?", } assert set(_STATUS_ICONS) == set(FinalStatus), "Missing icon for FinalStatus member" +# Explicit, no catch-all, for the same reason as the two maps above: a new status +# must be classified as "the agent phase ended this way" (True — a detached grade +# preserves it) or "grading decided this" (False — a detached grade replaces it). +# Defaulting either way silently is how an ERROR row becomes a SUCCESS. +_EXECUTION_FACT_STATUSES: dict[FinalStatus, bool] = { + FinalStatus.SUCCESS: False, + FinalStatus.FAILURE: False, + FinalStatus.NOT_GRADED: False, + FinalStatus.ERROR: True, + FinalStatus.BUILD_FAILED: True, + FinalStatus.TIMEOUT: True, + FinalStatus.MAX_TURNS_EXHAUSTED: True, + FinalStatus.TOKEN_BUDGET_EXCEEDED: True, + FinalStatus.COST_BUDGET_EXCEEDED: True, +} + +assert set(_EXECUTION_FACT_STATUSES) == set(FinalStatus), "Unclassified FinalStatus member" + + class ApiBackend(StrEnum): """API backend for LLM calls.""" diff --git a/src/coder_eval/models/experiment.py b/src/coder_eval/models/experiment.py index fe6ae6ff..b3b3671a 100644 --- a/src/coder_eval/models/experiment.py +++ b/src/coder_eval/models/experiment.py @@ -190,7 +190,11 @@ class VariantResult(BaseModel): # noqa: CE009 -- persisted result model; round- variant_id: str task_id: str - weighted_score: float + # None when nothing was graded (`coder-eval execute`), mirroring + # EvaluationResult.weighted_score. A plain float here would launder the + # ungraded None into 0.000, which renders as — and is picked as a best + # variant against — a real score of zero. + weighted_score: float | None = None final_status: FinalStatus duration_seconds: float total_tokens: int | None = None @@ -213,8 +217,9 @@ class VariantAggregate(BaseModel): # noqa: CE009 -- persisted result model; rou """Aggregated statistics for a single variant across all tasks. ``pass_rate`` uses the same denominator as ``RunSummary.pass_rate``: every task - the variant ran, errors included as misses. Otherwise an A/B whose variants - error at different rates compares two different denominators. + the variant GRADED, errors included as misses, ungraded tasks leaving BOTH + sides. Otherwise an A/B whose variants error at different rates compares two + different denominators. """ variant_id: str @@ -222,7 +227,17 @@ class VariantAggregate(BaseModel): # noqa: CE009 -- persisted result model; rou tasks_succeeded: int tasks_failed: int tasks_error: int - average_score: float + # Fourth bucket of the task_count invariant (see RunSummary.tasks_not_graded). + # Defaulted so experiment.json written before `coder-eval execute` still loads. + tasks_not_graded: int = Field( + default=0, + ge=0, + description="Tasks executed without grading (`coder-eval execute`). Excluded from pass_rate entirely.", + ) + # None when nothing in this variant was graded (`coder-eval execute`). + # A 0.0 here is indistinguishable from "measured and scored zero" — the same + # reason EvaluationResult.weighted_score is Optional. + average_score: float | None average_duration: float total_tokens: int | None = None replicate_count: int = Field( @@ -244,16 +259,30 @@ class VariantAggregate(BaseModel): # noqa: CE009 -- persisted result model; rou @model_validator(mode="after") def _check_task_count_invariant(self) -> VariantAggregate: - if self.tasks_succeeded + self.tasks_failed + self.tasks_error != self.tasks_run: - total = f"{self.tasks_succeeded} + {self.tasks_failed} + {self.tasks_error}" + buckets = self.tasks_succeeded + self.tasks_failed + self.tasks_error + self.tasks_not_graded + if buckets != self.tasks_run: + total = f"{self.tasks_succeeded} + {self.tasks_failed} + {self.tasks_error} + {self.tasks_not_graded}" raise ValueError(f"Task count invariant violated: {total} != {self.tasks_run}") return self + @computed_field # type: ignore[prop-decorator] + @property + def tasks_graded(self) -> int: + """Tasks actually measured — ``pass_rate``'s denominator. + + Serialized for the same reason as its RunSummary twin: a consumer that + cannot read the denominator re-derives the rate and drifts. + """ + return self.tasks_run - self.tasks_not_graded + @computed_field # type: ignore[prop-decorator] @property def pass_rate(self) -> float | None: - """``tasks_succeeded / tasks_run`` as a 0-1 fraction. ``None`` on an empty variant.""" - return self.tasks_succeeded / self.tasks_run if self.tasks_run else None + """``tasks_succeeded / tasks_graded`` as a 0-1 fraction. ``None`` when nothing was graded. + + Mirrors ``RunSummary.pass_rate``: ungraded tasks leave both sides. + """ + return self.tasks_succeeded / self.tasks_graded if self.tasks_graded else None class TaskExperimentSummary(BaseModel): # noqa: CE009 -- persisted result model; round-trip leniency like models/results.py diff --git a/src/coder_eval/models/results.py b/src/coder_eval/models/results.py index 8834710a..03feabcd 100644 --- a/src/coder_eval/models/results.py +++ b/src/coder_eval/models/results.py @@ -887,7 +887,20 @@ class SuiteRollup(BaseModel): rows_passed: int rows_failed: int rows_error: int - pass_rate: float = Field(ge=0.0, le=1.0, description="rows_passed / rows_total") + # The fourth bucket, matching RunSummary.tasks_not_graded and + # VariantAggregate.tasks_not_graded. Defaulted so a suite.json written before + # `execute` existed still parses. + rows_not_graded: int = Field(default=0, ge=0) + pass_rate: float | None = Field( + default=None, + ge=0.0, + le=1.0, + description=( + "rows_passed / rows_graded (ungraded rows excluded). None — never 0.0 — when nothing " + "was graded: a suite that was never measured has no pass rate, and 0.0 would publish " + "'0.0%' for it, indistinguishable from a suite where every row failed." + ), + ) average_weighted_score: float | None = Field( default=None, description="Mean weighted_score across rows that produced one." ) @@ -911,6 +924,32 @@ class SuiteRollup(BaseModel): ), ) + @computed_field # type: ignore[prop-decorator] + @property + def rows_graded(self) -> int: + """The denominator behind ``pass_rate``, serialized like its two twins. + + ``RunSummary.tasks_graded`` exists for the same reason and states it: a + consumer that cannot read the denominator has to re-derive it, which is + precisely how two surfaces end up publishing different numbers for the + same suite. + """ + return self.rows_total - self.rows_not_graded + + @model_validator(mode="after") + def _check_row_count_invariant(self) -> SuiteRollup: + """The same guard RunSummary carries, which this model was missing. + + Without it a row that lands outside all four buckets — the shape a new + FinalStatus category takes before every counter is updated — silently + drops out of the rollup instead of failing. + """ + buckets = self.rows_passed + self.rows_failed + self.rows_error + self.rows_not_graded + if buckets != self.rows_total: + total = f"{self.rows_passed} + {self.rows_failed} + {self.rows_error} + {self.rows_not_graded}" + raise ValueError(f"Suite row count invariant violated: {total} != {self.rows_total}") + return self + class SkippedTask(BaseModel): """A task YAML that was excluded from the run before reaching the orchestrator. @@ -1030,10 +1069,13 @@ def eval_result_total_cost(result: EvaluationResult) -> float | None: class RunSummary(BaseModel): """Summary of an entire evaluation run across multiple tasks. - ``pass_rate`` is ``tasks_succeeded / tasks_run``: every dispatched task is in the - denominator, errors included as misses. The previous formula excluded errors, - which paid a bonus for erroring. ``error_share`` reports how much of the rate is - errors, so a bad infrastructure night shows instead of being absorbed. + ``pass_rate`` is ``tasks_succeeded / tasks_graded``: every task that was + MEASURED is in the denominator, errors included as misses. An earlier formula + excluded errors, which paid a bonus for erroring. ``error_share`` reports how + much of the rate is errors, so a bad infrastructure night shows instead of + being absorbed. Only ungraded tasks (``coder-eval execute``) leave the + denominator — they were never measured, so a 0/0 run has no rate at all + rather than a 0% one. This is the framework's single denominator: every reporting surface reads ``pass_rate`` rather than re-deriving one. Derived metrics here are computed, @@ -1050,6 +1092,17 @@ class RunSummary(BaseModel): tasks_succeeded: int = Field(description="Number of tasks that succeeded") tasks_failed: int = Field(description="Number of tasks that failed") tasks_error: int = Field(description="Number of tasks that encountered errors") + # Part of the task_count invariant (a fourth bucket, not a sub-counter), but + # defaulted so run.json written before `coder-eval execute` existed — where + # no task can be ungraded — still deserialises. + tasks_not_graded: int = Field( + default=0, + ge=0, + description=( + "Number of tasks executed without grading (`coder-eval execute`). " + "Excluded from BOTH sides of pass_rate — an ungraded task was never measured." + ), + ) # Informational sub-counters: subsets of tasks_failed (NOT part of the # task_count invariant). Default 0 so old serialized RunSummary JSON @@ -1097,11 +1150,43 @@ class RunSummary(BaseModel): @model_validator(mode="after") def _check_task_count_invariant(self) -> RunSummary: - if self.tasks_succeeded + self.tasks_failed + self.tasks_error != self.tasks_run: - total = f"{self.tasks_succeeded} + {self.tasks_failed} + {self.tasks_error}" + buckets = self.tasks_succeeded + self.tasks_failed + self.tasks_error + self.tasks_not_graded + if buckets != self.tasks_run: + total = f"{self.tasks_succeeded} + {self.tasks_failed} + {self.tasks_error} + {self.tasks_not_graded}" raise ValueError(f"Task count invariant violated: {total} != {self.tasks_run}") return self + @computed_field # type: ignore[prop-decorator] + @property + def tasks_graded(self) -> int: + """Tasks that were actually measured — the denominator for every rate below. + + A ``computed_field`` rather than a plain property so it reaches run.json: + it is the denominator of `pass_rate` and `error_share`, and a consumer + that cannot read it has to re-derive the rate from the raw counts — which + is precisely how a consumer ends up publishing a different number for the + same run. REPORT_SCHEMA.md documents it as serialized. + """ + return self.tasks_run - self.tasks_not_graded + + @property + def _nothing_was_measured(self) -> bool: + """True when no row in this run produced a criteria verdict. + + ``tasks_graded`` is the complement of the ungraded bucket, so it still + counts ERROR rows — correct under ``run``, where an errored row was + attempted and genuinely missed. Under ``coder-eval execute`` no criterion + runs on ANY row, yet a crashed one lands in the ``error`` bucket rather + than the ``ungraded`` one, so it stayed in the denominator on its own: a + 100-task execute night with 5 crashes published ``pass_rate 0.0`` and + ``error_share 1.0`` — a measured-looking total failure for a run that was + never measured at all, and a real 0% point on the evalboard trend. + + The test is evidence again: if not one row reached a pass or a fail, the + rate has no numerator to be a fraction of. + """ + return self.tasks_not_graded > 0 and (self.tasks_succeeded + self.tasks_failed) == 0 + # Derived run metrics: computed_fields over the stored counts and # ``task_results``, so they serialize into run.json while staying impossible to # set to something the rows disagree with. Consumers should read these rather @@ -1110,18 +1195,33 @@ def _check_task_count_invariant(self) -> RunSummary: @computed_field # type: ignore[prop-decorator] @property def pass_rate(self) -> float | None: - """``tasks_succeeded / tasks_run`` as a 0-1 fraction. ``None`` on an empty run.""" - return self.tasks_succeeded / self.tasks_run if self.tasks_run else None + """``tasks_succeeded / tasks_graded`` as a 0-1 fraction. ``None`` on an empty run. + + The denominator excludes ungraded tasks (``coder-eval execute``), which were + never measured — counting them as misses would report a clean execute run as + 0% pass. Identical to ``tasks_run`` for every graded run. + + ``None`` also when no row produced a verdict at all, not just when the run + is empty — see ``_nothing_was_measured``. + """ + if self._nothing_was_measured: + return None + return self.tasks_succeeded / self.tasks_graded if self.tasks_graded else None @computed_field # type: ignore[prop-decorator] @property def error_share(self) -> float | None: - """``tasks_error / tasks_run`` as a 0-1 fraction. ``None`` on an empty run. + """``tasks_error / tasks_graded`` as a 0-1 fraction. ``None`` on an empty run. Diagnostic only, never adjusts the rate: a drop at a high error share is an - infrastructure night, the same drop at a normal share is the model. + infrastructure night, the same drop at a normal share is the model. Shares + ``pass_rate``'s denominator so the two are directly comparable — including + being ``None`` on a run where nothing was measured, or an execute night + with one crashed row reports 100% error. """ - return self.tasks_error / self.tasks_run if self.tasks_run else None + if self._nothing_was_measured: + return None + return self.tasks_error / self.tasks_graded if self.tasks_graded else None @computed_field # type: ignore[prop-decorator] @property diff --git a/src/coder_eval/models/tasks.py b/src/coder_eval/models/tasks.py index 68cfd324..52a23144 100644 --- a/src/coder_eval/models/tasks.py +++ b/src/coder_eval/models/tasks.py @@ -733,7 +733,10 @@ def check_suite_thresholds_require_dataset(self) -> Self: """ if self.dataset is None and self.suite_id is None: for c in self.success_criteria: - if getattr(c, "suite_thresholds", None): + # Direct attribute access, not getattr: suite_thresholds is + # declared on BaseSuccessCriterion, so every union member has it + # and pyright can see a rename. + if c.suite_thresholds: raise ValueError( f"success_criteria[{c.type!r}].suite_thresholds requires a dataset: block " + "(thresholds are evaluated on aggregated across-row metrics)" diff --git a/src/coder_eval/orchestration/batch.py b/src/coder_eval/orchestration/batch.py index 8576cdfc..57ca4684 100644 --- a/src/coder_eval/orchestration/batch.py +++ b/src/coder_eval/orchestration/batch.py @@ -15,7 +15,7 @@ from collections.abc import Callable, Iterator from datetime import datetime from pathlib import Path -from typing import Any +from typing import Any, NamedTuple from ..models import ( AgentKind, @@ -28,7 +28,7 @@ TaskDefinition, TaskResult, ) -from ..path_utils import format_task_log_id +from ..path_utils import TASK_JSON_FILENAME, format_task_log_id from ..pricing import unpriced_models from ..reports_experiment import eval_result_to_task_dict from ..streaming.callbacks import StreamCallback @@ -164,6 +164,7 @@ async def run_single(rt: ResolvedTask) -> TaskResult: preservation_mode=preservation_mode, stream_callback=task_callback, verbose=config.verbose, + grade=config.grade, ).run() # The in-container _finalize_result can't emit task telemetry # (connection-string env vars aren't forwarded into the @@ -185,6 +186,7 @@ async def run_single(rt: ResolvedTask) -> TaskResult: source_yaml=rt.source_yaml, config_lineage=rt.config_lineage, replicate_index=rt.replicate_index, + grade=config.grade, ) result = await orchestrator.run() tr = TaskResult( @@ -322,10 +324,38 @@ def _create_error_task_result( ) -def partition_for_resume( - resolved_tasks: list[ResolvedTask], -) -> tuple[list[ResolvedTask], list[TaskResult], list[ResolvedTask]]: - """Split resolved tasks into (to_run, prior_results, prior_resolved) for --resume. +class ResumePartition(NamedTuple): + """How ``--resume`` splits a run's tasks over what each one still needs.""" + + to_run: list[ResolvedTask] + """Never finished executing — re-run from scratch.""" + + to_grade: list[ResolvedTask] + """Executed but never scored. Needs criteria, NOT another agent run.""" + + prior_results: list[TaskResult] + """Genuinely finished — reloaded so run.json covers the whole suite.""" + + prior_resolved: list[ResolvedTask] + """The ResolvedTask for each entry of prior_results, same order.""" + + +def _owes_a_grade(result: EvaluationResult) -> bool: + """Whether a finalized row was executed but never scored. + + Evidence, not label. ``NOT_GRADED`` is the ordinary shape, but an ``execute`` + row that also tripped a run limit (TIMEOUT, a budget stop) carries an + execution-fact status whose category is ``error``/``failed`` — and no verdict + at all. Both are equally owed a grade; only the first announces it. + + A row that carries a criteria vector or a score has been graded, whatever its + status, so a genuine FAILURE/ERROR from ``run`` is untouched. + """ + return result.weighted_score is None and not result.success_criteria_results + + +def partition_for_resume(resolved_tasks: list[ResolvedTask], *, grade: bool = True) -> ResumePartition: + """Split resolved tasks over what ``--resume`` still owes each one. A task is already-complete when its task.json exists, parses, and carries a final_status. task.json is written atomically at end-of-run, so any parseable @@ -333,26 +363,51 @@ def partition_for_resume( tasks are reloaded into TaskResults (to fold into run.json) and excluded from to_run; everything else — including failed-to-parse — re-runs. + **"Finished" is relative to the resuming command, not absolute.** A + ``NOT_GRADED`` row (written by ``coder-eval execute``) has a final status, so + the naive test calls it complete. That is right for ``execute --resume``, + which owes it nothing — and wrong for ``run --resume``, which was asked to + grade: skipping it would report "already complete", grade nothing, and exit + 0. So under ``grade=True`` those rows go to ``to_grade`` instead, where the + caller runs the criteria against the trajectory and workspace already on + disk rather than paying for the agent a second time. + + The test is the row's **evidence**, not its status: a row is owed a grade + when it was executed but never scored. Keying on ``category == "ungraded"`` + alone missed every ``execute`` row that also carries an execution fact — a + TIMEOUT or budget stop aborts the run before grading, so under ``execute`` it + lands with ``weighted_score is None`` and an empty criteria vector, yet its + category is ``error``/``failed`` and resume filed it as complete. It then + stayed permanently unscored in run.json, the rollup and the evalboard, while + ``evaluate `` graded the identical bytes happily — two entry points + into one feature disagreeing about the same record. + + Note the asymmetry is only for a row that was never scored. FAILURE and ERROR + rows that DO carry a verdict stay complete under both commands — resume has + never retried failures (delete a task's task.json to force that), and this + does not change it. + Args: resolved_tasks: Fully-resolved tasks for the whole run. + grade: Whether the resuming command grades (``run``) or not (``execute``). Returns: - (to_run, prior_results, prior_resolved): - - to_run: tasks still needing execution - - prior_results: reloaded results for already-complete tasks - - prior_resolved: the ResolvedTask for each prior_result (same order) + The four-way :class:`ResumePartition`. """ to_run: list[ResolvedTask] = [] + to_grade: list[ResolvedTask] = [] prior_results: list[TaskResult] = [] prior_resolved: list[ResolvedTask] = [] for rt in resolved_tasks: tr = _load_completed_result(rt) if tr is None: to_run.append(rt) + elif grade and _owes_a_grade(tr.result): + to_grade.append(rt) else: prior_results.append(tr) prior_resolved.append(rt) - return to_run, prior_results, prior_resolved + return ResumePartition(to_run, to_grade, prior_results, prior_resolved) def clear_rerun_artifacts(to_run: list[ResolvedTask]) -> int: @@ -378,7 +433,7 @@ def clear_rerun_artifacts(to_run: list[ResolvedTask]) -> int: def _load_completed_result(rt: ResolvedTask) -> TaskResult | None: """Reconstruct a TaskResult from a finalized task.json, or None if absent/incomplete.""" - report_path = rt.run_dir / "task.json" + report_path = rt.run_dir / TASK_JSON_FILENAME try: text = report_path.read_text(encoding="utf-8") except OSError: @@ -425,7 +480,7 @@ def recover_task_results(run_dir: Path) -> list[TaskResult]: """ nested_roots = [p.parent for p in run_dir.rglob("run.json") if p.parent != run_dir] recovered: list[TaskResult] = [] - for task_json in run_dir.rglob("task.json"): + for task_json in run_dir.rglob(TASK_JSON_FILENAME): if any(root in task_json.parents for root in nested_roots): continue # belongs to a nested sub-run (its own run.json), not this one try: @@ -509,13 +564,24 @@ def read_run_fingerprint(run_dir: Path) -> dict[str, object] | None: return data if isinstance(data, dict) else None +# `grade` is excluded from the drift warning: `execute` then `run --resume` is a +# SUPPORTED flow, not a config mistake, and the warning's text ("already-finalized +# tasks keep their original-config results") is actively wrong for it — those rows +# are re-graded with the current config, which is the entire point. +_FINGERPRINT_DIFF_EXEMPT = frozenset({"grade"}) + + def fingerprint_diff(prior: dict[str, object], current: dict[str, object]) -> dict[str, tuple[object, object]]: """Keys present in BOTH stamps that disagree, as ``{key: (prior, current)}``. Only keys present in ``prior`` are compared, so adding fingerprint fields in a later version never false-flags a resume of an older run. """ - return {k: (prior[k], current[k]) for k in current if k in prior and prior[k] != current[k]} + return { + k: (prior[k], current[k]) + for k in current + if k in prior and prior[k] != current[k] and k not in _FINGERPRINT_DIFF_EXEMPT + } def _override_uip_versions_from_tasks(version_info: dict[str, Any], task_results: list[TaskResult]) -> None: @@ -665,6 +731,7 @@ def build_run_summary( tasks_succeeded=sum(1 for s in statuses if s.category == "succeeded"), tasks_failed=sum(1 for s in statuses if s.category == "failed"), tasks_error=sum(1 for s in statuses if s.category == "error"), + tasks_not_graded=sum(1 for s in statuses if s.category == "ungraded"), tasks_token_budget_exceeded=sum(1 for s in statuses if s == FinalStatus.TOKEN_BUDGET_EXCEEDED), tasks_cost_budget_exceeded=sum(1 for s in statuses if s == FinalStatus.COST_BUDGET_EXCEEDED), skipped_tasks=skipped_tasks or [], diff --git a/src/coder_eval/orchestration/config.py b/src/coder_eval/orchestration/config.py index 695b5594..80a3d812 100644 --- a/src/coder_eval/orchestration/config.py +++ b/src/coder_eval/orchestration/config.py @@ -94,6 +94,20 @@ class BatchRunConfig(BaseModel): description="CLI override for replicates per (task, variant). None = defer to experiment layers.", ) + # Grading switch: `coder-eval run` (True) vs `coder-eval execute` (False). + # It lives HERE and nowhere else on purpose — it is deliberately NOT part of + # the 5-layer task merge, so there is no `-D grade=...` path and no + # MergeField (CE014 does not apply to a scalar bool outside the merged + # roots). A task YAML must never be able to declare itself ungraded; only + # the invoking command decides. + grade: bool = Field( + default=True, + description=( + "Evaluate success criteria after execution. False = `coder-eval execute`: " + "run and capture the trajectory, score nothing, finalize as NOT_GRADED." + ), + ) + # Logging verbose: bool = Field(default=False, description="Enable verbose (DEBUG level) logging for Docker output") diff --git a/src/coder_eval/orchestration/experiment.py b/src/coder_eval/orchestration/experiment.py index 25d4ed58..8c48ef99 100644 --- a/src/coder_eval/orchestration/experiment.py +++ b/src/coder_eval/orchestration/experiment.py @@ -11,8 +11,9 @@ import importlib.resources import logging import re +from collections.abc import Sequence from pathlib import Path -from typing import Any +from typing import Any, Literal import yaml @@ -817,11 +818,65 @@ def resolve_all_tasks( def _pick_worst_status(statuses: list[FinalStatus]) -> FinalStatus: """Pick the worst final_status across replicates (error > failed > succeeded). - Unknown categories fall back to priority -1 so they sort as worst-of-all - (fail-closed: a new unrecognised status becomes the most urgent). + Every category is enumerated below and indexed directly, so a new one is a + type error here rather than a silent placement. + + "ungraded" sorts LEAST urgent (above "succeeded") — it carries no verdict, so + any replicate that does have one must win. It therefore survives only when + every replicate is ungraded. + + Mixed sets ARE reachable, contrary to an earlier note here: ``grade`` is + run-level, but ``run --resume`` grades each executed-but-ungraded row + independently and folds a row whose grade failed back in still ungraded + (``cli/run_command._grade_resumed_tasks``). So one replicate can be graded + while its sibling is not, and the ordering above is what keeps that from + absorbing an unmeasured replicate into a pass. """ - priority = {"error": 0, "failed": 1, "succeeded": 2} - return min(statuses, key=lambda s: priority.get(s.category, -1)) + # Annotated with the SAME Literal `FinalStatus.category` returns, and indexed + # directly rather than via `.get(..., -1)`. Adding the fourth `ungraded` + # bucket here was a manual step no checker could verify — an untyped + # `dict[str, int]` proves neither that every category is present nor that no + # stray key is — while the `-1` default it leaned on was already unreachable + # (the `assert set(_STATUS_CATEGORIES) == set(FinalStatus)` in models/enums.py + # makes `category` total). Worse, that default was documented as + # "fail-closed", but -1 sorts BELOW error, so a fifth category would have + # silently outranked ERROR as the worst status. + priority: dict[Literal["succeeded", "failed", "error", "ungraded"], int] = { + "error": 0, + "failed": 1, + "succeeded": 2, + "ungraded": 3, + } + return min(statuses, key=lambda s: priority[s.category]) + + +def _measured_scores(rows: Sequence[VariantResult] | Sequence[TaskResult]) -> list[float]: + """The scores of every row that was actually GRADED, errors included as 0.0. + + The filter is on ``final_status.category``, deliberately not on + ``weighted_score is not None``. Those look equivalent and are not: an + ERROR / BUILD_FAILED row also has no score, and dropping it removes it from + BOTH sides of the mean. A nightly where one image build failed would then + report a HIGHER headline score than a clean one, and A/B comparisons would + be biased toward whichever variant errored more — the exact "bonus for + erroring" the run-level rates are written to avoid. + + Only ``ungraded`` rows leave both sides: nothing measured them, so they are + not a miss. Everything else that ran and produced no score IS a miss. + """ + scores: list[float] = [] + for row in rows: + result = row if isinstance(row, VariantResult) else row.result + if result.final_status.category == "ungraded": + continue + scores.append(result.weighted_score if result.weighted_score is not None else 0.0) + return scores + + +def _mean_graded_score(vr_list: list[VariantResult]) -> float | None: + """Mean score over the measured rows; ``None`` when none were measured.""" + scores = _measured_scores(vr_list) + return sum(scores) / len(scores) if scores else None def _mean_reference_similarity(reps: list[TaskResult]) -> float | None: @@ -835,6 +890,37 @@ def _mean_reference_similarity(reps: list[TaskResult]) -> float | None: return sum(scores) / len(scores) if scores else None +def _fold_replicates(task_id: str, variant_id: str, reps: list[TaskResult]) -> VariantResult: + """Fold every replicate of one (task, variant) into a single VariantResult. + + Extracted from ``aggregate_results``, which was already the largest function + in the module before the ungraded bucket added another filter to it. + """ + # Ungraded replicates — and ONLY those — drop out entirely rather than + # contributing 0.0: `or 0.0` would average a clean `execute` run down to a + # real-looking zero, while dropping an errored one would pay it a bonus. + scores = _measured_scores(reps) + non_errored = [r for r in reps if r.result.final_status.category != "error"] + durations = [r.result.duration_seconds for r in non_errored] + iter_counts = [r.result.iteration_count for r in reps if r.result.iteration_count is not None] + asst_turns = [r.result.total_assistant_turns for r in reps if r.result.total_assistant_turns is not None] + token_vals = [r.result.total_token_usage.total_tokens for r in reps if r.result.total_token_usage is not None] + + return VariantResult( + variant_id=variant_id, + task_id=task_id, + weighted_score=sum(scores) / len(scores) if scores else None, + final_status=_pick_worst_status([r.result.final_status for r in reps]), + duration_seconds=sum(durations), + total_tokens=sum(token_vals) if token_vals else None, + iteration_count=round(sum(iter_counts) / len(iter_counts)) if iter_counts else None, + total_assistant_turns=round(sum(asst_turns) / len(asst_turns)) if asst_turns else None, + reference_similarity=_mean_reference_similarity(reps), + replicate_index=0, # aggregate — points at first replicate for link rendering + replicate_count=len(reps), + ) + + def aggregate_results( experiment_id: str, description: str, @@ -867,41 +953,32 @@ def aggregate_results( # Collect per-replicate scores keyed variant_id → task_id → [scores] for stats rendering. per_replicate_scores: dict[str, dict[str, list[float]]] = {} for (task_id, variant_id), reps in task_variant_reps.items(): - per_replicate_scores.setdefault(variant_id, {})[task_id] = [r.result.weighted_score or 0.0 for r in reps] + per_replicate_scores.setdefault(variant_id, {})[task_id] = _measured_scores(reps) task_variants: dict[str, list[VariantResult]] = {} for (task_id, variant_id), reps in task_variant_reps.items(): - scores = [r.result.weighted_score or 0.0 for r in reps] - non_errored = [r for r in reps if r.result.final_status.category != "error"] - durations = [r.result.duration_seconds for r in non_errored] - statuses = [r.result.final_status for r in reps] - iter_counts = [r.result.iteration_count for r in reps if r.result.iteration_count is not None] - asst_turns = [r.result.total_assistant_turns for r in reps if r.result.total_assistant_turns is not None] - token_vals = [r.result.total_token_usage.total_tokens for r in reps if r.result.total_token_usage is not None] - ref_similarity = _mean_reference_similarity(reps) - final_status = _pick_worst_status(statuses) - - variant_result = VariantResult( - variant_id=variant_id, - task_id=task_id, - weighted_score=sum(scores) / len(scores), - final_status=final_status, - duration_seconds=sum(durations), - total_tokens=sum(token_vals) if token_vals else None, - iteration_count=round(sum(iter_counts) / len(iter_counts)) if iter_counts else None, - total_assistant_turns=round(sum(asst_turns) / len(asst_turns)) if asst_turns else None, - reference_similarity=ref_similarity, - replicate_index=0, # aggregate — points at first replicate for link rendering - replicate_count=len(reps), - ) - task_variants.setdefault(task_id, []).append(variant_result) + task_variants.setdefault(task_id, []).append(_fold_replicates(task_id, variant_id, reps)) # Build task summaries task_summaries: list[TaskExperimentSummary] = [] for task_id, variants in task_variants.items(): - best = max(variants, key=lambda v: (v.weighted_score, v.variant_id)) - scores = [v.weighted_score for v in variants] - top_count = sum(1 for v in variants if v.weighted_score == best.weighted_score) + # Only graded variants can win or set a spread. Including ungraded ones + # at 0.0 would name an arbitrary "best" among scores that do not exist. + # + # When NOTHING was scored there is no winner, and the fallback must not + # invent one: `variants[0]` is whichever arm the input happened to list + # first, so swapping the two inputs flipped the reported winner — with + # `is_tie=False` asserting it was a real result. Sort by variant_id (so + # the field is at least deterministic) and mark it a tie among all arms, + # which is what "no arm outscored another" actually means. + scored = [(v, v.weighted_score) for v in variants if v.weighted_score is not None] + if scored: + best = max(scored, key=lambda pair: (pair[1], pair[0].variant_id))[0] + top_count = sum(1 for _, s in scored if s == best.weighted_score) + else: + best = min(variants, key=lambda v: v.variant_id) + top_count = len(variants) + scores = [s for _, s in scored] rep_counts = {v.replicate_count for v in variants} task_summaries.append( TaskExperimentSummary( @@ -909,7 +986,7 @@ def aggregate_results( variant_results=variants, best_variant=best.variant_id, is_tie=top_count > 1, - score_spread=max(scores) - min(scores), + score_spread=(max(scores) - min(scores)) if scores else 0.0, replicate_count=min(rep_counts) if rep_counts else 1, ) ) @@ -925,7 +1002,7 @@ def aggregate_results( tasks_succeeded=0, tasks_failed=0, tasks_error=0, - average_score=0.0, + average_score=None, average_duration=0.0, ) continue @@ -938,9 +1015,13 @@ def aggregate_results( tasks_succeeded=sum(1 for v in vr_list if v.final_status.category == "succeeded"), tasks_failed=sum(1 for v in vr_list if v.final_status.category == "failed"), tasks_error=sum(1 for v in vr_list if v.final_status.category == "error"), + tasks_not_graded=sum(1 for v in vr_list if v.final_status.category == "ungraded"), tasks_token_budget_exceeded=sum(1 for v in vr_list if v.final_status == FinalStatus.TOKEN_BUDGET_EXCEEDED), tasks_cost_budget_exceeded=sum(1 for v in vr_list if v.final_status == FinalStatus.COST_BUDGET_EXCEEDED), - average_score=sum(v.weighted_score for v in vr_list) / len(vr_list), + # Mean over GRADED rows only, and None when there are none: a clean + # execute run has no average score, and reporting 0.000 next to + # "Pass Rate: n/a" is a number indistinguishable from "scored zero". + average_score=_mean_graded_score(vr_list), average_duration=sum(v.duration_seconds / v.replicate_count for v in vr_list) / len(vr_list), total_tokens=total_tokens, replicate_count=vr_list[0].replicate_count if vr_list else 1, diff --git a/src/coder_eval/orchestration/regrade.py b/src/coder_eval/orchestration/regrade.py new file mode 100644 index 00000000..1b9b09cb --- /dev/null +++ b/src/coder_eval/orchestration/regrade.py @@ -0,0 +1,535 @@ +"""Grade a run that already executed — the shared core behind two callers. + +``coder-eval execute`` leaves every row ``NOT_GRADED``. Two commands can supply +the verdict afterwards, and both must do it identically: + +* ``coder-eval evaluate `` — grade one finished task explicitly. +* ``coder-eval run --resume`` — grade the ungraded rows it finds in the run dir + instead of re-executing them (see ``partition_for_resume``). + +The logic lives here rather than in ``cli/`` because the resume path is not a CLI +concern, and because two copies of "how to re-grade" would drift into two +different verdicts for the same run. Errors surface as :class:`RegradeError`, a +plain exception the CLI wraps into its own error type — ``orchestration/`` must +not depend on the CLI layer (CE004). +""" + +from __future__ import annotations + +import asyncio +import logging +from pathlib import Path + +from coder_eval.models import ( + EvaluationResult, + PreservationMode, + SandboxConfig, + TaskConfigRecord, + TaskDefinition, +) +from coder_eval.path_utils import PRE_GRADE_JSON_FILENAME, TASK_JSON_FILENAME, write_text_atomic +from coder_eval.sandbox import Sandbox + + +logger = logging.getLogger(__name__) + +ARTIFACTS_DIRNAME = "artifacts" + + +class RegradeError(Exception): + """A finished run cannot be re-graded as asked.""" + + +def load_prior_result(run_dir: Path) -> EvaluationResult: + """Read a finished run's ``task.json``.""" + path = run_dir / TASK_JSON_FILENAME + try: + raw = path.read_text(encoding="utf-8") + except OSError as e: + raise RegradeError(f"Cannot read {path}: {e}") from e + try: + return EvaluationResult.model_validate_json(raw) + except ValueError as e: + raise RegradeError(f"{path} is not a readable EvaluationResult: {e}") from e + + +def task_from_prior( + prior: EvaluationResult, + run_dir: Path, + *, + allow_recorded_commands: bool = False, + include_setup_phase: bool = True, +) -> tuple[TaskDefinition, str]: + """Rebuild the executed task from the run's own recorded config. + + Rebuilding from ``task_config.resolved`` rather than re-reading the YAML is + what makes the grade describe the run that happened: ``resolved`` is the + post-merge definition, so variant overrides, ``-D`` flags and dataset row + expansion are all already baked in. Re-loading the source YAML would silently + grade a DIFFERENT task whenever any of those were used. + + Falls back to the source YAML only when ``resolved`` will not validate (a + schema change since the run), and says so loudly — a quiet fallback would + reintroduce exactly the drift above. + + ``allow_recorded_commands`` gates the shell half. See + :func:`check_embedded_commands`. + """ + record = prior.task_config + if record is None: + raise RegradeError( + f"{run_dir / TASK_JSON_FILENAME} carries no task_config, so the executed task cannot be " + + "rebuilt. Pass the task file explicitly: coder-eval evaluate " + ) + try: + task = TaskDefinition.model_validate(record.resolved) + except ValueError as e: + return _fall_back_to_source( + record, run_dir, e, allow_recorded_commands=allow_recorded_commands, include_setup_phase=include_setup_phase + ) + check_embedded_commands( + task, run_dir, allow_recorded_commands=allow_recorded_commands, include_setup_phase=include_setup_phase + ) + return task, record.source_yaml + + +def embedded_commands(task: TaskDefinition, *, include_setup_phase: bool = True) -> list[str]: + """Every shell command a rebuilt task definition would run on this host. + + ``include_setup_phase`` covers the two capability families that exist only on + the ``--copy`` path: ``pre_run`` / ``post_run``, and the sandbox's own + provisioning. Both are SKIPPED when grading in place (``Sandbox.adopt`` runs + no installer, and re-running the hooks would overwrite the agent's + deliverables before the criteria read them), so on that path they are not a + capability the run dir has. + + Sandbox provisioning is the half this gate originally missed, and it was the + worst one. ``grading_sandbox_config`` carries the recorded ``sandbox`` block + through untouched, and the ``--copy`` branch then calls ``Sandbox.setup``, + which reaches ``uv pip install ``, ``npm install `` and ``git clone ``. A package name is arbitrary + code at install time. Because the scan walked only ``success_criteria``, a + shared run directory whose criteria were all ``file_exists`` sailed through + the gate and still ran installers of the attacker's choosing. + + ``isinstance`` narrowing, never ``getattr(c, "command", None)``: an untyped + string probe over a discriminated union is invisible to pyright, so renaming + a field silently degrades the only guard on this path to a permanent no-op — + the exact hazard ``models/tasks.py`` already documents in prose. It also + cannot reach ``agent_judge``, whose ``bash`` tooling is the widest blast + radius of the three. + """ + from coder_eval.models import ( + AgentJudgeCriterion, + LLMJudgeCriterion, + RepoSource, + RunCommandCriterion, + UiPathEvalCriterion, + ) + + commands: list[str] = [] + for c in task.success_criteria: + if isinstance(c, RunCommandCriterion): + commands.append(c.command) + elif isinstance(c, AgentJudgeCriterion): + # No command string of its own: it spawns a Claude Code SDK agent + # with tool access (Bash included) under the grader's credentials, + # which is a strictly wider capability than one shell line. + commands.append(f"") + elif isinstance(c, LLMJudgeCriterion): + # No shell, but it spends the grader's model budget and ships the + # graded artifacts (and optionally the trajectory) to a provider of + # the recorded config's choosing. That is a capability the operator + # should approve, even though nothing executes locally. + commands.append(f"") + elif isinstance(c, UiPathEvalCriterion): + # Builds and shells `uv run uipath eval …`. Every argument is + # shlex-quoted, so this is disclosure rather than injection — but it + # is still a subprocess the recorded config chose to start. + commands.append(f"uv run uipath eval {c.agent_name} {c.eval_set}") + if include_setup_phase: + commands += [c.command for c in task.pre_run] + [c.command for c in task.post_run] + sandbox = task.sandbox + if sandbox.python is not None and sandbox.python.env_packages: + commands.append(f"uv pip install {' '.join(sandbox.python.env_packages)}") + if sandbox.node is not None and sandbox.node.env_packages: + commands.append(f"npm install {' '.join(sandbox.node.env_packages)}") + for source in sandbox.template_sources or []: + if isinstance(source, RepoSource): + commands.append(f"git clone -- {source.url}") + return commands + + +def check_embedded_commands( + task: TaskDefinition, run_dir: Path, *, allow_recorded_commands: bool, include_setup_phase: bool = True +) -> None: + """Refuse — or at minimum name — the shell a rebuilt config will run here. + + ``task_config.resolved`` is data that travels inside a run directory, and a + run directory is a shareable artifact — the detached-grading flow exists so + one machine can execute and another can grade. Rebuilding the task from it + means the *run dir* decides what ``run_command`` criteria the grader runs, + with the grader's environment (API keys, cloud credentials, SSH agent). + + A warning is not a control: it is printed as the command is already being + prepared, and nobody reads a log line fast enough to stop it. So a recorded + config that carries shell is REFUSED unless the operator opted in. The common + case — ``execute`` then ``evaluate`` on your own machine — is unaffected + whenever the criteria are file/JSON checks, and the opt-in is one flag. + + Passing the task file explicitly (``evaluate ``) also + bypasses this: that config came from the operator, not from the artifact. + """ + commands = embedded_commands(task, include_setup_phase=include_setup_phase) + if not commands: + return + rendered = "; ".join(commands) + if not allow_recorded_commands: + raise RegradeError( + f"The config recorded in {run_dir / TASK_JSON_FILENAME} would run {len(commands)} shell " + + f"command(s) on this host with your environment: {rendered}\n" + + "A run directory is a shareable artifact, so its recorded config is untrusted input. " + + "Re-run with --allow-recorded-commands to accept them, or pass the task file " + + "explicitly: coder-eval evaluate " + ) + logger.warning( + "Grading %s runs %d shell command(s) taken from that run's own recorded config: %s", + run_dir, + len(commands), + rendered, + ) + + +def _fall_back_to_source( + record: TaskConfigRecord, + run_dir: Path, + e: ValueError, + *, + allow_recorded_commands: bool, + include_setup_phase: bool = True, +) -> tuple[TaskDefinition, str]: + """The loud source-YAML fallback for a resolved config that no longer validates.""" + from .task_loader import load_task + + if not record.source_file or not Path(record.source_file).is_file(): + raise RegradeError( + f"The resolved task config in {run_dir / TASK_JSON_FILENAME} no longer validates ({e}), and " + + "its source YAML is unavailable. Pass the task file explicitly." + ) from e + logger.warning( + "The recorded resolved config does not validate (%s); falling back to %s. Variant " + + "overrides, -D flags and dataset expansion from the original run are NOT reapplied, " + + "so this grade may not match what ran.", + e, + record.source_file, + ) + task, source_yaml = load_task(Path(record.source_file)) + check_embedded_commands( + task, run_dir, allow_recorded_commands=allow_recorded_commands, include_setup_phase=include_setup_phase + ) + return task, source_yaml + + +def default_workspace(run_dir: Path, prior: EvaluationResult) -> Path: + """Locate the workspace a finished run left behind. + + ``sandbox_path`` is authoritative when it still exists — it is where the run + actually worked. Otherwise fall back to the preserved artifacts tree, where + preservation nests the workspace under the task id. + + Raises rather than guessing when neither is conclusive. Guessing is worse + than failing here: grading the WRONG directory makes every path-relative + criterion fail as a locating artifact rather than as a verdict, and it + reports that as an ordinary score. + """ + if prior.sandbox_path: + recorded = Path(prior.sandbox_path) + if recorded.is_dir(): + if not _is_within(recorded, run_dir): + # An absolute path out of the run's own task.json, which is + # untrusted input for a shared run dir. Criteria execute with + # cwd there and may mutate it, so an out-of-tree location has to + # be the operator's explicit choice. + raise RegradeError( + f"The recorded sandbox_path ({recorded}) is outside the run directory " + + f"({run_dir}). Pass --workspace explicitly to grade it." + ) + return recorded + + artifacts = run_dir / ARTIFACTS_DIRNAME + if not artifacts.is_dir(): + raise RegradeError( + f"No workspace to grade: {artifacts} does not exist and the recorded sandbox_path " + + f"({prior.sandbox_path or 'unset'}) is gone. The run was probably made with " + + "--preservation-mode NONE." + ) + # The exact path, not a heuristic. `task_id` may contain "/" (dataset rows + # are "/"), so "the single child of artifacts/" resolves one + # level too high for every row task. + # + # Containment-checked like the sandbox_path branch above, and for the same + # reason: `task_id` is an unvalidated string out of the run's own task.json, + # so `"../../../../home/victim"` joins to a real directory that `is_dir()` + # happily confirms. Every run_command criterion then executes with that as + # its cwd. The two branches read the same untrusted record; only one of them + # used to check. + by_task_id = artifacts / prior.task_id + if by_task_id.is_dir(): + if not _is_within(by_task_id, artifacts): + raise RegradeError( + f"The recorded task_id ({prior.task_id!r}) resolves outside {artifacts}. " + + "Pass --workspace explicitly to grade a directory outside the run." + ) + return by_task_id + + children = [p for p in sorted(artifacts.iterdir()) if p.is_dir()] + if not children: + # A flat artifacts dir (no subdirectory) means the workspace IS artifacts/. + return artifacts + if len(children) == 1: + return children[0] + raise RegradeError( + f"Cannot tell which directory under {artifacts} is the workspace: no {prior.task_id!r} " + + f"child, and {len(children)} candidates ({', '.join(p.name for p in children)}). " + + "Pass --workspace explicitly." + ) + + +def _is_within(candidate: Path, root: Path) -> bool: + """True when ``candidate`` resolves inside ``root``.""" + try: + candidate.resolve().relative_to(root.resolve()) + except ValueError: + return False + return True + + +def verify_reference_unchanged(prior: EvaluationResult, task: TaskDefinition, task_file: Path | None) -> None: + """Refuse to grade when the reference tree changed since the run. + + ``reference_comparison`` and reference-carrying judges score against + ``task.reference.directory``. If it moved since the run, the re-grade would + silently measure the agent's old work against a new answer key. + + ``task_file`` is what ``reference.directory`` resolves against, so it is + required for any task that declares one — resolving without it raises, which + is why it is threaded through rather than passed as ``None``. + """ + if task.reference is None: + return + recorded = prior.environment_info.get("reference_digest") + if not isinstance(recorded, str): + # A run that predates the digest being persisted. Say so: silence here is + # what made this whole guard dead code for its first release. + logger.warning( + "This run recorded no reference_digest, so the answer key cannot be verified. " + + "Grading proceeds; a reference edited since the run would go undetected." + ) + return + from .evaluation import resolve_reference_dir + + try: + resolved = resolve_reference_dir(task, task_file) + except (FileNotFoundError, ValueError) as e: + raise RegradeError( + f"This run's task declares a reference directory that cannot be resolved now ({e}), " + + "so its contents cannot be verified against the executed run." + ) from e + if resolved is None or not resolved.is_dir(): + raise RegradeError( + f"The reference directory recorded for this run is gone ({resolved}). Grading now " + + "would score against a missing answer key. Restore it, or re-run the task." + ) + if _staged_digest(resolved) != recorded: + raise RegradeError( + f"The reference directory {resolved} changed since this run was executed " + + "(digest mismatch). Grading now would score the agent's work against a " + + "different answer key. Restore the reference, or re-run the task." + ) + + +def _staged_digest(source: Path) -> str: + """Digest ``source`` the way the run recorded it — through a staged copy. + + The recorded ``reference_digest`` is taken over the per-run STAGED copy + (``Orchestrator._stage_reference``), which ``stage_reference_dir`` filters + through ``REFERENCE_COPY_IGNORE`` (``.git``) and strips of symlinks. + Digesting the raw source instead compares two differently-filtered trees, so + any reference that is a git checkout — the case the ignore list exists for — + reports a permanent false mismatch and un-grades the row for good. + + Re-staging rather than re-implementing the filter keeps the two in step: a + future entry in the ignore list applies here without a second edit. + """ + import tempfile + + from coder_eval.path_utils import digest_tree + + from .evaluation import stage_reference_dir + + with tempfile.TemporaryDirectory(prefix="coder-eval-refdigest-") as tmp: + staged = stage_reference_dir(source, Path(tmp) / "reference") + return digest_tree(staged) + + +def back_up_pre_grade_record(run_dir: Path) -> None: + """Keep the ungraded ``task.json`` beside the graded one, once. + + The write-back replaces the only on-disk evidence that this run was executed + separately from grading. Copying it first keeps that auditable. Written once: + a second grade must not overwrite the ORIGINAL execute record with an + already-graded one. + """ + source, backup = run_dir / TASK_JSON_FILENAME, run_dir / PRE_GRADE_JSON_FILENAME + if backup.exists() or not source.is_file(): + return + if source.is_symlink() or backup.is_symlink(): + # Untrusted run dir: writing through a symlink would let a shared + # artifact clobber an arbitrary file the grading user can write. + logger.warning("Not preserving the pre-grade record: %s or %s is a symlink.", source, backup) + return + try: + backup.write_text(source.read_text(encoding="utf-8"), encoding="utf-8") + except OSError as e: + # Never fail a grade over the audit copy. + logger.warning("Could not preserve the pre-grade record at %s: %s", backup, e) + + +def restore_pre_grade_record(run_dir: Path) -> bool: + """Put the ungraded ``task.json`` back after a grading crash. + + ``Orchestrator._finalize_result`` writes ``task.json`` into its run dir + *before* returning, so by the time a caller sees ``FinalStatus.ERROR`` the + ERROR row is already on disk whenever the grade wrote into the run being + graded (always, on ``run --resume``). Both commands treat ERROR as complete, + so the row would be permanently un-regradeable — and a caller that only fixes + its in-memory result leaves ``run.json`` disagreeing with ``task.json``. + + Returns whether the restore happened; there is nothing to restore when the + grade wrote elsewhere and the original was never replaced. + """ + source, backup = run_dir / TASK_JSON_FILENAME, run_dir / PRE_GRADE_JSON_FILENAME + if not backup.is_file() or backup.is_symlink() or source.is_symlink(): + return False + try: + text = backup.read_text(encoding="utf-8") + if source.is_file() and source.read_text(encoding="utf-8") == text: + return False # never overwritten; nothing to undo + write_text_atomic(source, text) + except OSError as e: + logger.warning("Could not restore the ungraded record at %s: %s", source, e) + return False + logger.info("Restored the ungraded record at %s after a grading failure.", source) + return True + + +def grading_sandbox_config(task: TaskDefinition, *, allow_host_grading: bool = False) -> SandboxConfig: + """The sandbox config a grading pass runs under. + + Grading never runs a container: the docker driver dispatches through + DockerRunner, which needs an agent. So a ``driver: docker`` task can only be + graded on the host — and that is a DIFFERENT machine from the one its + criteria were written against. + + It is therefore refused rather than downgraded. A container task's criteria + address container paths (``/verifier``, ``/logs/verifier``) and container + toolchains; run on the host they score 0.0 for a trajectory ``run`` scored + 1.0, and the row is written back FAILURE. The same commands (``rm -rf + /verifier``, ``mkdir -p /logs/verifier``) also execute unsandboxed on the + grading machine. A silent rewrite additionally neutralized the ``docker`` + refusal in ``Sandbox.adopt``, which exists to catch exactly this. + + ``allow_host_grading`` is the operator's explicit acceptance of both. Rows + graded that way are stamped ``graded_on_host`` in ``environment_info`` + (:func:`stamp_host_grading`) so they are never silently comparable with rows + a container graded. + + Re-validated rather than ``model_copy(update=...)``: ``update`` skips both + pydantic validation and pyright, so a typo would produce a SandboxConfig + violating its own ``Literal`` and surface much later at an unrelated + ``if driver == "docker"`` branch. + """ + if task.sandbox.driver != "docker": + return task.sandbox.model_copy(deep=True) + if not allow_host_grading: + raise RegradeError( + f"Task {task.task_id!r} ran under `driver: docker`, and grading cannot start a container " + + "(there is no agent to run in it). Grading on the host would execute this task's " + + "criteria against a filesystem that lacks the container's paths and toolchain, " + + "scoring a FAILURE for a run that passed — and would run its shell commands " + + "unsandboxed here. Re-run with --allow-host-grading to accept that, or grade on a " + + "machine that reproduces the container." + ) + logger.warning( + "Grading %r on the host: its `driver: docker` sandbox cannot be reproduced here, so " + + "path- and toolchain-dependent criteria may score differently than they did in the run. " + + "The row is stamped graded_on_host.", + task.task_id, + ) + return SandboxConfig.model_validate({**task.sandbox.model_dump(), "driver": "tempdir"}) # noqa: CE051 + + +def stamp_host_grading(result: EvaluationResult, task: TaskDefinition) -> None: + """Record that a docker task's verdict was produced on the host. + + Written onto the result, not just logged: a console warning does not travel + with ``task.json`` into ``run.json``, the reports or the evalboard, and this + row must never be compared with a container-graded one without that caveat + attached. + """ + if task.sandbox.driver == "docker": + result.environment_info["graded_on_host"] = True + + +async def regrade_in_place( + *, + task: TaskDefinition, + prior: EvaluationResult, + workspace: Path, + run_dir: Path, + task_file: Path | None, + source_yaml: str, + variant_id: str, + replicate_index: int = 0, + allow_host_grading: bool = False, +) -> EvaluationResult: + """Run ``task``'s criteria against an already-executed ``workspace``. + + The workspace is *adopted*, never copied: it is the run's own output, and the + template-copy path filters out ``node_modules`` / ``dist`` / ``build`` / + ``.venv``, which would make a criterion reading those fail as a copying + artifact rather than as a verdict. + + ``prior`` supplies the trajectory and the run's execution facts (see + ``Orchestrator._seed_from_prior_result``), so criteria that read the agent's + tool calls score exactly as they would have during the run. + """ + from coder_eval.orchestrator import Orchestrator + + # Inside the shared entry point, not at each caller: a guard a caller has to + # remember is one a third caller will forget, and this one is the difference + # between a verdict and a verdict against the wrong answer key. + verify_reference_unchanged(prior, task, task_file) + + sandbox = Sandbox( + grading_sandbox_config(task, allow_host_grading=allow_host_grading), + task_id=task.task_id, + task_dir=task_file.parent.resolve() if task_file is not None else None, + ) + await asyncio.to_thread(sandbox.adopt, workspace) + + orchestrator = Orchestrator( + task=task, + run_dir=run_dir, + # The workspace belongs to the run being graded; never move or delete it. + preservation_mode=PreservationMode.NONE, + task_file=task_file, + sandbox=sandbox, + variant_id=variant_id, + source_yaml=source_yaml, + replicate_index=replicate_index, + prior_result=prior, + ) + result = await orchestrator.run() + stamp_host_grading(result, task) + return result diff --git a/src/coder_eval/orchestrator.py b/src/coder_eval/orchestrator.py index a2697bc0..35adc934 100644 --- a/src/coder_eval/orchestrator.py +++ b/src/coder_eval/orchestrator.py @@ -2,11 +2,12 @@ import asyncio import logging +import os import re import tempfile import time import uuid -from collections.abc import Callable +from collections.abc import Callable, Sequence from contextlib import suppress from dataclasses import dataclass from datetime import datetime @@ -66,7 +67,14 @@ from .orchestration.early_stop import EarlyStopWatcher, early_stop_active, validate_early_stop from .orchestration.evaluation import resolve_reference_dir, stage_reference_dir from .orchestration.run_limits import validate_run_limits -from .path_utils import digest_tree, format_task_log_id, rmtree_restrictive, task_log_path +from .path_utils import ( + TASK_JSON_FILENAME, + digest_tree, + format_task_log_id, + rmtree_restrictive, + task_log_path, + write_text_atomic, +) from .sandbox import Sandbox from .simulation import DialogStopReason, SimulatorResult, UserSimulator, evaluate_stop from .streaming.callbacks import CompositeStreamCallback, StreamCallback, TaskScopedCallback, safe_emit @@ -296,7 +304,6 @@ def build_task_event(result: EvaluationResult, *, driver: str, variant_id: str) "Status": result.final_status.value, "Category": result.final_status.category, "DurationMs": int((result.duration_seconds or 0.0) * 1000), - "Score": float(result.weighted_score or 0.0), "Iterations": result.iteration_count, "AgentType": result.agent_type or "", "Model": result.model_used or "", @@ -304,6 +311,13 @@ def build_task_event(result: EvaluationResult, *, driver: str, variant_id: str) "EarlyStopped": result.early_stop is not None, "EarlyStopReason": (result.early_stop.reason.value if result.early_stop is not None else ""), } + # `Score` is OMITTED, never coalesced to 0.0, when the row was not graded. + # Dashboards compute `avg(todouble(customDimensions.Score))` with no status + # filter, so a laundered zero for a `coder-eval execute` night drags every + # score tile toward zero and is indistinguishable from a genuinely bad night. + # An absent dimension drops out of the average instead. + if result.weighted_score is not None: + props["Score"] = float(result.weighted_score) return "CoderEval.Task.End", props @@ -363,6 +377,8 @@ def __init__( config_lineage: dict[str, ConfigLineageEntry] | None = None, replicate_index: int = 0, workspace_dir: Path | None = None, + grade: bool = True, + prior_result: EvaluationResult | None = None, ): """Initialize the orchestrator. @@ -385,6 +401,18 @@ def __init__( run_dir/artifacts/, and the workspace is copied out to run_dir/artifacts/ at cleanup. Resolved host-side by DockerRunner; None keeps standard behavior. Takes precedence over preservation_mode when set. + grade: Whether to evaluate success criteria after execution. False is + `coder-eval execute`: the agent runs and the full trajectory is + captured, but no criterion is checked, ``weighted_score`` stays + None, and the row finalizes as ``FinalStatus.NOT_GRADED``. It is + deliberately NOT a task-config field — a task YAML must never be + able to declare itself ungraded — so it arrives only from + ``BatchRunConfig.grade``, never from the 5-layer merge or -D. + prior_result: A completed run's ``EvaluationResult`` to re-grade + (evaluate-only mode). Its trajectory and execution facts are + carried onto the fresh result so the grade describes the run that + actually happened instead of an empty one — see + ``_seed_from_prior_result`` for the field-by-field rationale. """ self.task = task self.run_dir = run_dir @@ -403,9 +431,11 @@ def __init__( self.source_yaml = source_yaml self.config_lineage = config_lineage or {} self.replicate_index = replicate_index + self.grade = grade + self.prior_result = prior_result # Derived paths - self.report_path = self.run_dir / "task.json" + self.report_path = self.run_dir / TASK_JSON_FILENAME self.html_report_path = self.run_dir / "task.html" # Clean user<->agent transcript for simulation runs. Written alongside # task.log so a human can follow the conversation without the @@ -493,6 +523,58 @@ def _agent_name(self) -> str: return str(self.task.agent.type) return AgentKind.NONE.value + def _terminal_status(self, success: bool) -> FinalStatus: + """The status a normally-completed evaluation loop lands on. + + Extracted from ``run()`` because the chain answers one question and + ``run()`` answers several; inlining it grew ``run()`` past the + complexity bound the moment the grading switch was threaded in. + + Order matters at every step: + + * A detached grade may NOT overturn an execution fact. The prior run's + terminal status (TIMEOUT, ERROR, a budget stop) describes an agent + phase this pass neither repeated nor observed. Without the first arm a + crashed run re-graded against its half-finished workspace reports + SUCCESS — with the original ``error_message`` still attached. + * The NOT_GRADED arm sits ABOVE ``max_turns_exhausted``, and that order + is what makes ``execute`` + ``evaluate`` equal a single ``run``. + MAX_TURNS_EXHAUSTED reads like an execution fact but is not one: on the + graded path it is subordinate to the verdict — ``run`` returns SUCCESS + for a max-turns trajectory whose criteria pass, and only falls through + to MAX_TURNS_EXHAUSTED when they do not. So it is not knowable under + ``grade=False``. Consuming it here first made it terminal *and* + permanent (``is_execution_fact`` pins it in the first arm), so the same + agent output scored SUCCESS/1.0 under ``run`` and MAX_TURNS_EXHAUSTED + under ``execute`` → ``evaluate`` — and, being category ``failed``, + `run --resume` called the row complete and left it forever unscored. + Nothing is lost by deferring: the fact lives on + ``result.max_turns_exhausted``, which ``_seed_from_prior_result`` + carries, so the detached grade walks this identical chain and reaches + the arm below. + + The statuses that ARE execution facts (TIMEOUT, ERROR, the budget + stops) differ in kind: they abort the run before a verdict is + reachable, so preserving them overturns nothing. + + With ``grade=True`` and no prior result the chain is the original one. + """ + assert self.result is not None, "Result not initialized" + inherited = self.prior_result.final_status if self.prior_result is not None else None + if inherited is not None and inherited.is_execution_fact: + logger.info( + "Preserving the run's terminal status %s: grading cannot overturn an execution fact.", + inherited.value, + ) + return inherited + if success: + return FinalStatus.SUCCESS + if not self.grade: + return FinalStatus.NOT_GRADED + if self.result.max_turns_exhausted: + return FinalStatus.MAX_TURNS_EXHAUSTED + return FinalStatus.FAILURE + async def run(self) -> EvaluationResult: """Run the complete evaluation. @@ -531,6 +613,8 @@ async def run(self) -> EvaluationResult: environment_info=get_version_info(), ) + self._seed_from_prior_result() + # Calculate task log path task_log_file = task_log_path(self.run_dir) task_log_file.parent.mkdir(parents=True, exist_ok=True) # noqa: CE002 — mkdir on local FS is nanoseconds @@ -583,13 +667,7 @@ def _kill_agent_subprocess_sync() -> None: elapsed_seconds=time.time() - start_time, ) - # Update final status - if success: - self.result.final_status = FinalStatus.SUCCESS - elif self.result.max_turns_exhausted: - self.result.final_status = FinalStatus.MAX_TURNS_EXHAUSTED - else: - self.result.final_status = FinalStatus.FAILURE + self.result.final_status = self._terminal_status(success) except asyncio.CancelledError: # Re-raise cancellation to allow proper task cancellation @@ -683,6 +761,9 @@ def _kill_agent_subprocess_sync() -> None: # but BEFORE _finalize_result so task.json includes the field. # Allowlist non-success terminal statuses; SUCCESS and # MAX_TURNS_EXHAUSTED skip the tail to keep task.json compact. + # NOT_GRADED is deliberately absent: like SUCCESS and + # MAX_TURNS_EXHAUSTED it is not a diagnosis of something going + # wrong, so it keeps task.json compact. if self.result.final_status in { FinalStatus.ERROR, FinalStatus.TIMEOUT, @@ -697,6 +778,92 @@ def _kill_agent_subprocess_sync() -> None: return self.result + def _seed_from_prior_result(self) -> None: + """Carry a completed run's execution facts onto this re-grade's result. + + ``execute`` then ``evaluate`` must equal a single ``run``. Everything + below is a fact the AGENT phase established that the grading phase cannot + re-derive; leaving any of them at their defaults would publish a row that + silently disagrees with the run it grades. + + Deliberately NOT carried: ``final_status``, ``weighted_score`` and + ``success_criteria_results`` — those are exactly what this pass recomputes. + """ + prior = self.prior_result + if prior is None or self.result is None: + return + + # A task row describes the TASK, so its clock is the agent run's, not the + # grading pass's. Left alone, a 10-minute run re-graded in 2 seconds would + # report 2 seconds — and that figure feeds average_duration, the report + # tables and the evalboard, so harness-vs-harness comparisons would be + # quietly wrong. _finalize_result restores the duration after its own + # timing write; the grading pass's cost is recorded separately there. + self.result.started_at = prior.started_at + # completed_at too, so the row's three time fields stay consistent with + # each other: leaving it at grading wall-clock produces a triple where + # completed_at - started_at != duration_seconds, which misleads anyone + # deriving a duration from the timestamps. + self.result.completed_at = prior.completed_at + + # The trajectory itself. Every derived figure in _finalize_result — + # token totals, cost, command_stats, model_used, assistant turns — + # recomputes from `iterations`, so seeding it reproduces them exactly. + self.result.iterations = list(prior.iterations) + # Evaluate-only hardcodes 1; a multi-turn run must not be reported as + # single-turn just because the re-grade ran once. + self.result.iteration_count = prior.iteration_count or len(prior.iterations) + + # LOAD-BEARING for the verdict: gate selection is FIRED-ONLY. When + # early_stop is not None the checker gates on the weighted ARMED subset + # instead of strict-AND over every criterion. Dropping it would re-grade a + # truncated trajectory under the full-run gate and flip the verdict. + self.result.early_stop = prior.early_stop + + # Execution facts that outlive the agent process. + self.result.max_turns_exhausted = prior.max_turns_exhausted + self.result.error_message = prior.error_message + self.result.error_details = prior.error_details + self.result.error_log_tail = prior.error_log_tail + self.result.sdk_options = prior.sdk_options + self.result.agent_config = prior.agent_config + self.result.expected_commands = prior.expected_commands + self.result.simulation = prior.simulation + + # The hooks belong to the execute phase and are NOT re-run against an + # adopted workspace (see _skip_hooks_for_adopted), so their recorded + # outcomes would otherwise vanish from the graded row. + self.result.pre_run_results = list(prior.pre_run_results) + self.result.post_run_results = list(prior.post_run_results) + + # The artifacts pointer. An adopted sandbox is not deleted by cleanup(), + # so the path stays valid — and a SECOND grade needs it, since without it + # the caller falls back to guessing the workspace. + self.result.sandbox_path = prior.sandbox_path + + # environment_info: the prior run's capture describes the machine that + # RAN the task (installed_tools, api route, coder_eval version). Ours + # describes the machine grading it. Prior wins on conflict, and ours is + # preserved as flat `graded_by_*` scalars rather than being interleaved — + # a report that shows the grader's tool versions as the run's is worse + # than one that shows neither. + # Flattened to scalars rather than nested wholesale: environment_info is + # a flat map everywhere it is consumed (the HTML report `_esc`apes each + # value into a table cell; the evalboard types it as + # Record>), so a + # whole nested env capture renders as a Python dict repr. Only the three + # facts that identify the grading HOST are kept, and only when they + # differ from the run's. + grader = self.result.environment_info + provenance = { + f"graded_by_{key}": grader[key] + for key in ("coder_eval", "git_commit", "cli_version") + if key in grader and grader.get(key) != prior.environment_info.get(key) + } + # A second grade must not lose the first grader's stamp — merging prior + # over ours would otherwise clobber it and collapse the chain silently. + self.result.environment_info = {**grader, **prior.environment_info, **provenance} + async def _run_evaluation_with_failure_evidence( self, *, @@ -785,6 +952,11 @@ async def _evaluate_post_failure_criteria(self) -> None: """ if self.result is None: return + if not self.grade: + # Grading site 4 of 4. Under `execute` no criterion is checked on any + # path, diagnostics included — recording a not_evaluated vector here + # would imply criteria we were supposed to run and couldn't. + return if self.success_checker is None or self.sandbox is None: self._record_post_failure_not_evaluated("the sandbox or success checker was unavailable") return @@ -873,13 +1045,27 @@ def _finalize_result(self, start_time: float) -> None: self.result.completed_at = datetime.now() self.result.duration_seconds = time.time() - start_time + # Re-grade: the row keeps the agent run's duration (see + # _seed_from_prior_result). The grading pass's own cost is preserved + # alongside rather than discarded, so a slow judge is still visible. + if self.prior_result is not None: + self.result.environment_info["grading_duration_seconds"] = round(self.result.duration_seconds, 3) + self.result.duration_seconds = self.prior_result.duration_seconds + # Weighted score. This call site is wrapped because _finalize_result runs # inside run()'s finally — an unguarded raise here would skip persistence and # lose task.json. The other calculate_weighted_score calls (the simulation # path) run inside run()'s try, whose broad `except Exception` already converts # a raise into a populated ERROR result, so they intentionally stay unwrapped. try: - self.result.calculate_weighted_score(self.task.success_criteria) + if self.grade: + self.result.calculate_weighted_score(self.task.success_criteria) + else: + # Explicit None, NOT the 0.0 calculate_weighted_score writes for an + # empty results list — that value is indistinguishable from a task + # that was graded and scored zero, and every downstream `score or + # 0.0` would launder it into a real-looking failure. + self.result.weighted_score = None except ValueError as e: logger.error("Weighted-score computation failed; marking row ERROR: %s", e, exc_info=True) self.result.weighted_score = None @@ -948,10 +1134,13 @@ def _finalize_result(self, start_time: float) -> None: # Terminal per-task summary line. Emitted before report writes so a # write failure cannot swallow the one-line outcome. logger.info( - "Task finished: status=%s duration=%.1fs score=%.3f iterations=%d", + "Task finished: status=%s duration=%.1fs score=%s iterations=%d", self.result.final_status.value, self.result.duration_seconds or 0.0, - self.result.weighted_score or 0.0, + # "n/a", not 0.000: this line is read when diagnosing a run, and a + # zero here would say the criteria scored nothing rather than that + # nothing was scored. + "n/a" if self.result.weighted_score is None else f"{self.result.weighted_score:.3f}", self.result.iteration_count, ) @@ -981,10 +1170,8 @@ def _finalize_result(self, start_time: float) -> None: # docker-driver host-heartbeat watchdog firing) would otherwise leave # a truncated task.json that the host parses as malformed-JSON rather # than as "no result", conflating two distinct failure modes. - import os as _os - - report_tmp = self.report_path.with_suffix(self.report_path.suffix + ".tmp") - report_tmp.write_text( # noqa: CE002 — small JSON write at end of run + write_text_atomic( # noqa: CE002 — small JSON write at end of run + self.report_path, self.result.model_dump_json( indent=2, # Strip inline transcripts: they live in sibling YAML files @@ -993,9 +1180,7 @@ def _finalize_result(self, start_time: float) -> None: # in the row record without losing any data. exclude=TASK_JSON_TRANSCRIPT_EXCLUDE, ), - encoding="utf-8", ) - _os.replace(report_tmp, self.report_path) # Also emit an HTML trace/report alongside task.json. HTML failure must # never mask the underlying run outcome — write_task_html logs and @@ -1134,6 +1319,14 @@ def _join_litellm_actual_cost(self) -> None: """ if not (isinstance(self.route, LiteLLMRoute) and settings.litellm_cost_log and self.result is not None): return + if self.prior_result is not None: + # Re-grading someone else's trajectory. The join keys on THIS + # Orchestrator's per-attempt nonce, which the original turns were + # never tagged with, so it would match nothing and overwrite the + # already-corrected per-turn costs with a warning about a missing + # bill. The prior run's cost is the real one; leave it alone. + logger.debug("Re-grade of a prior trajectory: keeping its recorded cost, skipping the LiteLLM join.") + return try: applied = apply_actual_cost( self.result, @@ -1217,6 +1410,12 @@ async def _stage_reference(self) -> None: destination = staging / "reference" self._reference_dir = await asyncio.to_thread(stage_reference_dir, source, destination) self._reference_digest = await asyncio.to_thread(digest_tree, self._reference_dir) + # Persist it: a DETACHED grade happens in a different process with no + # access to this instance, and refuses to score old work against a new + # answer key by comparing the tree it stages against this recorded hash + # (orchestration/regrade.py::verify_reference_unchanged). + if self.result is not None: + self.result.environment_info["reference_digest"] = self._reference_digest self._validate_reference_consumers() def _validate_reference_consumers(self) -> None: @@ -1284,7 +1483,18 @@ async def _setup(self) -> None: # armed evaluate-only re-grade builds an inert (never-fed) watcher — # harmless, and keeps a single creation point. if early_stop_active(self.task): - self._early_stop_watcher = EarlyStopWatcher.for_task(self.task) + if self.grade: + self._early_stop_watcher = EarlyStopWatcher.for_task(self.task) + else: + # Early stop cuts the run once coder-eval's own criteria decide the + # outcome. Under `execute` there is no outcome to decide and the + # trajectory is the deliverable (an external harness grades it), so + # an armed criterion must not truncate it. Same effect as the + # run_limits.stop_early kill switch, decided one layer up. + logger.info( + "Grading disabled (execute mode): early-stop is armed but stays disabled; " + + "the full trajectory is the deliverable." + ) # Stage the reference BEFORE either branch returns: judge criteria with # include_reference=true (and any $REFERENCE_DIR/... file entry) expect it @@ -1297,6 +1507,14 @@ async def _setup(self) -> None: self.sandbox.reference_dir = self._reference_dir self.result.sandbox_path = str(self.sandbox.sandbox_dir) + # PATH parity with the run being graded. _sync_sandbox_command_path_ + # with_agent recorded the agent's effective PATH; no agent runs here, + # so restore it explicitly or `run_command` criteria resolve binaries + # against ambient PATH and can disagree with the original verdict. + restored_path = self.result.environment_info.get("command_base_path") + if isinstance(restored_path, str) and restored_path: + self.sandbox.set_command_base_path(self._sanitize_restored_path(restored_path)) + self._resolve_routes() self._record_route_environment_info() return @@ -1468,6 +1686,13 @@ def _sync_sandbox_command_path_with_agent(self) -> None: path = sdk_env.get("PATH") if isinstance(path, str) and path: self.sandbox.set_command_base_path(path) + # Persist it so a LATER detached grade (`coder-eval evaluate` over a + # finished run dir) can restore the same PATH. Without this the + # "evaluate-only mode" gap named above is permanent: the re-grade + # would resolve `run_command` criteria against ambient PATH and could + # reach a different verdict than the run it claims to be grading. + if self.result is not None: + self.result.environment_info["command_base_path"] = path def _eval_route_overrides(self) -> EvalRouteOverrides: """The ``(backend, model)`` pair from ``task.checker_context.api_route``, if any. @@ -1550,6 +1775,25 @@ def _reject_litellm_agent_judge_if_unsupported(self) -> None: ) raise ValueError(msg) + def _record_grader_route_provenance(self) -> None: + """Record the GRADING host's route without touching the run's. + + Two keys only — the agent route and the judge route — because those are + the two that decide what a re-grade's LLM criteria actually talked to. + Omitted entirely when they match the run, so an ordinary same-machine + re-grade adds nothing to the record. + """ + assert self.result is not None + assert self.route is not None + env = self.result.environment_info + grader_routes = {"graded_by_api_routing": ROUTE_NAMES[type(self.route)]} + if self.eval_route is not None: + grader_routes["graded_by_eval_routing"] = ROUTE_NAMES[type(self.eval_route)] + for key, value in grader_routes.items(): + run_key = key.removeprefix("graded_by_") + if env.get(run_key) != value: + env[key] = value + def _record_route_environment_info(self) -> None: """Persist resolved route + judge transport into ``result.environment_info``. @@ -1559,6 +1803,17 @@ def _record_route_environment_info(self) -> None: """ assert self.result is not None assert self.route is not None + if self.prior_result is not None: + # A detached grade resolves routes for ITS OWN host, which may be a + # different backend from the one that ran the task. Writing them into + # the run's keys contradicts the "prior wins" contract in + # _seed_from_prior_result and leaves a self-contradictory record — + # `api_routing: anthropic_direct` beside the run's stale `aws_region` + # and `bedrock_model`. Keep the run's routing; record the grader's + # alongside it, under the same `graded_by_` provenance prefix the + # seeding uses, and only when it actually differs. + self._record_grader_route_provenance() + return self.result.environment_info["api_routing"] = ROUTE_NAMES[type(self.route)] # The judge side (llm_judge / agent_judge) may run on a different, # constant backend — pinned to Claude when the agent is on LiteLLM — so @@ -1875,6 +2130,124 @@ def _accumulate_judge_usage( # forward so it isn't dropped from the latest results list. r.token_usage = prior + def _sanitize_restored_path(self, recorded: str) -> str: + """Filter a PATH restored from a run's own ``task.json`` before prepending it. + + The restored value is PREPENDED ahead of the host PATH, and it arrives + from a file inside the directory being graded — a run dir is a shareable + artifact (that is the whole point of the detached-grading flow), and under + ``driver: docker`` it is bind-mounted writable into the container the agent + runs in. Prepending it verbatim lets a run dir decide which binary + ``pytest`` resolves to on the grader's host. + + Four filters, all cheap and all about what PATH parity actually needs: + + * **Absolute only.** A relative entry resolves against the grader's + *current working directory*, which has nothing to do with the run — so + ``evilbin`` in a recorded PATH becomes ``$PWD/evilbin`` at the front of + every criterion subprocess's PATH. It also cannot be the toolchain + location it claims to be, since the run resolved it somewhere else. + * Drop anything that is not an existing directory (a dead entry buys no + parity). + * Drop any entry inside the **workspace** being graded — that tree is + agent-writable, so a shim dropped there would shadow a real tool. + * Drop any entry inside the **run directory** as a whole. The workspace + is only part of it; ``artifacts/``, a sibling replicate's tree and the + run root itself all travel in the same shared artifact and are all + equally attacker-chosen. + + What remains is the run's genuine toolchain locations. + """ + workspace = self.sandbox.sandbox_dir.resolve() if self.sandbox and self.sandbox.sandbox_dir else None + run_root = self.run_dir.resolve() + blocked = [p for p in (workspace, run_root) if p is not None] + kept: list[str] = [] + for entry in recorded.split(os.pathsep): + if not entry: + continue + candidate = Path(entry) + if not candidate.is_absolute(): + logger.warning( + "Dropping recorded PATH entry %r: it is relative, so it would resolve against " + + "the grader's working directory rather than the run's toolchain.", + entry, + ) + continue + if not candidate.is_dir(): + logger.debug("Dropping recorded PATH entry %s: not a directory here.", entry) + continue + resolved = candidate.resolve() + if any(resolved == root or root in resolved.parents for root in blocked): + logger.warning( + "Dropping recorded PATH entry %s: it lies inside the run being graded, " + + "so a binary there could shadow a real tool on the grader's host.", + entry, + ) + continue + kept.append(str(resolved)) + return os.pathsep.join(kept) + + def _select_gate(self) -> bool: + """Apply the verdict gate to the criteria results already on ``self.result``. + + Gate selection is FIRED-ONLY: the weighted armed gate applies iff the + watcher actually cut the run (``early_stop is not None``) — on a truncated + trajectory the unarmed criteria never had the chance to be satisfied, so + they stay advisory. A run that completed naturally (armed or not, watcher + never fired or disarmed fail-open) has a full trajectory and gates + strict-AND over every gating criterion, exactly like an unarmed run — + arming a criterion (e.g. adding a ``decide_within`` fail-fast timeout) + must never change the verdict of a run it didn't cut. + + BOTH SINGLE-SHOT grading paths must call this — the live one and the + evaluate-only one, which are its two call sites. A detached grade + (``evaluate `` / ``run --resume``) reaches the verdict through + the evaluate-only branch, where ``early_stop`` arrives via + ``_seed_from_prior_result`` rather than from a live watcher; selecting + the gate there in a second, hand-written place is exactly how the + seeded field came to be carried but never read — re-grading an + early-stopped run under the full-run strict-AND gate flips its verdict. + + The simulation dialog path does NOT route through here, and saying "both + grading paths" without that qualifier read as though it did. It is + benign only because ``result.early_stop`` is never assigned on the + dialog path, so an armed simulation task silently gates strict-AND on a + possibly-truncated trajectory. Wiring the dialog path through this seam + means also setting ``early_stop`` there; until then the limit is stated + rather than implied. + """ + assert self.result is not None + if self.result.early_stop is not None: + # One gate for every early-stopped run, no per-reason branches: a + # decision-budget stop is just a fail-stop whose deciding criterion + # timed out (the watcher only fires once the weighted ceiling + # proves the armed gate cannot pass). The ceiling is an upper bound + # on the authoritative armed score only because the watcher reduces + # the SAME trajectory the checker scores — it records UNRESOLVED + # tool ends exactly like the agent's EventCollector does (see + # EarlyStopWatcher._on_event_impl) — so the weighted armed gate is + # correct whether the watcher fired on a pass, a fail, or a timeout. + gate_threshold = ( + self.task.run_limits.stop_early_gate_threshold + if self.task.run_limits is not None + else DEFAULT_STOP_EARLY_GATE_THRESHOLD + ) + armed_count = sum(1 for c in self.task.success_criteria if c.is_stop_armed) + logger.info( + "Early-stopped run (%s): gating on %d armed criteria (%d advisory, not gated).", + self.result.early_stop.reason.value, + armed_count, + len(self.task.success_criteria) - armed_count, + ) + return self.result.armed_criteria_passed(self.task.success_criteria, gate_threshold) + + if self._early_stop_watcher is not None: + if self._early_stop_watcher.disarmed: + logger.info("early-stop watcher disarmed fail-open (verdict error): gating on the full set.") + else: + logger.info("early-stop armed but never fired (run completed naturally): gating on the full set.") + return self.result.all_criteria_passed(self.task.success_criteria) + async def _evaluation_loop(self) -> bool: """Run the main evaluation loop. @@ -1888,6 +2261,14 @@ async def _evaluation_loop(self) -> bool: assert self.task.agent is not None if self.agent is None: + # Grading site 1 of 4. Evaluate-only with grading off would neither + # run an agent nor check anything — a no-op that still writes a + # task.json. Refuse instead of producing an empty row. + if not self.grade: + raise ValueError( + "grade=False is meaningless on the evaluate-only path (no agent attached): " + + "the run would neither execute nor grade." + ) # No agent attached: evaluate-only re-grade of a completed sandbox. # (No-op tasks have a NoOpAgent here, so they take the normal path # below.) Check the criteria directly against the sandbox. @@ -1901,7 +2282,12 @@ async def _evaluation_loop(self) -> bool: "Criteria %s require agent execution; results may be incomplete with no agent", unsupported, ) - self.result.iteration_count = 1 + # A bare `evaluate ` has no trajectory, so one nominal + # iteration stands for the single grading pass. A re-grade seeded + # from a prior result already carries the real count (and the turns + # the trajectory-reading criteria need) — do not flatten it to 1. + if self.prior_result is None: + self.result.iteration_count = 1 # Load reference in evaluate-only mode too: judge criteria with # include_reference=true expect this populated even when no agent # runs. The agent-driven branch below has the same call. @@ -1911,7 +2297,7 @@ async def _evaluation_loop(self) -> bool: turn_records=self.result.iterations, ) self.result.success_criteria_results = criteria_results - return self.result.all_criteria_passed(self.task.success_criteria) + return self._select_gate() # Working directory context prepended to every prompt (including feedback). # The agent resumes its session between iterations via session_id. @@ -1955,6 +2341,38 @@ async def _evaluation_loop(self) -> bool: logger.debug(f"Agent response received ({len(turn_record.agent_output)} chars)") + # Facts about the RUN, recorded before the grading switch. `execute` + # withholds the verdict, never the facts: `_seed_from_prior_result` + # cannot restore a fact the execute phase never captured, so a later + # `evaluate` would inherit the wrong terminal status. + # + # Recording the fact is NOT the same as finalizing on it. `max_turns` + # exhaustion decides the status only when the criteria fail (see + # `_terminal_status`), so under `grade=False` this flag is carried into + # task.json and consumed by the detached grade, not turned into a + # terminal status here. + if turn_record.max_turns_exhausted: + self.result.max_turns_exhausted = True + logger.warning( + "Agent exhausted max_turns (%s).", + self.task.run_limits.max_turns if self.task.run_limits else None, + ) + # Soft cumulative-turn check (logs once; never aborts). + self._check_expected_turns(iteration=iteration) + + # Grading site 2 of 4. `execute` stops here: the trajectory is captured + # and persisted exactly as on a graded run, but nothing is scored. + # Returning False keeps FinalStatus off SUCCESS; run()'s status chain + # turns it into NOT_GRADED. The reference-integrity check is skipped too + # — it exists to protect a grade that is not happening. + if not self.grade: + logger.info("Grading disabled (execute mode): skipping success criteria.") + # The budget gate is a run limit, not a verdict. Its only reason to + # sit after the criteria on the graded path is partial-credit + # visibility, and there is no partial credit here. + self._check_run_limits(iteration=iteration) + return False + # Check success criteria (reference_dir feeds reference_comparison + judges) logger.debug("Checking success criteria") await self._verify_reference_integrity() @@ -1971,64 +2389,22 @@ async def _evaluation_loop(self) -> bool: pairs = list(zip(criteria_results, self.task.success_criteria, strict=True)) passed_count = sum(1 for r, c in pairs if r.score >= c.pass_threshold) total_count = len(pairs) - # Gate selection is FIRED-ONLY: the weighted armed gate applies iff the - # watcher actually cut the run (early_stop is not None) — on a truncated - # trajectory the unarmed criteria never had the chance to be satisfied, - # so they stay advisory. A run that completed naturally (armed or not, - # watcher never fired or disarmed fail-open) has a full trajectory and - # gates strict-AND over every gating criterion, exactly like an unarmed - # run — arming a criterion (e.g. adding a decide_within fail-fast - # timeout) must never change the verdict of a run it didn't cut. - if self.result.early_stop is not None: - # One gate for every early-stopped run, no per-reason branches: a - # decision-budget stop is just a fail-stop whose deciding criterion - # timed out (the watcher only fires once the weighted ceiling - # proves the armed gate cannot pass). The ceiling is an upper bound - # on the authoritative armed score only because the watcher reduces - # the SAME trajectory the checker scores — it records UNRESOLVED - # tool ends exactly like the agent's EventCollector does (see - # EarlyStopWatcher._on_event_impl) — so the weighted armed gate is - # correct whether the watcher fired on a pass, a fail, or a timeout. - gate_threshold = ( - self.task.run_limits.stop_early_gate_threshold - if self.task.run_limits is not None - else DEFAULT_STOP_EARLY_GATE_THRESHOLD - ) - all_passed = self.result.armed_criteria_passed(self.task.success_criteria, gate_threshold) - armed_count = sum(1 for c in self.task.success_criteria if c.is_stop_armed) - logger.info( - "Early-stopped run (%s): gating on %d armed criteria (%d advisory, not gated).", - self.result.early_stop.reason.value, - armed_count, - total_count - armed_count, - ) - else: - if self._early_stop_watcher is not None: - if self._early_stop_watcher.disarmed: - logger.info("early-stop watcher disarmed fail-open (verdict error): gating on the full set.") - else: - logger.info("early-stop armed but never fired (run completed naturally): gating on the full set.") - all_passed = self.result.all_criteria_passed(self.task.success_criteria) + all_passed = self._select_gate() # Reuse the model method for weighted score (single source of truth) self.result.calculate_weighted_score(self.task.success_criteria) - current_score = self.result.weighted_score or 0.0 + # calculate_weighted_score just ran, so a score exists; the fallback is + # for the type, not for an unmeasured row (this branch only runs when + # grading did). + current_score = self.result.weighted_score or 0.0 # noqa: CE049 — graded here by construction logger.info(f"Success criteria: {passed_count}/{total_count} passed, weighted score: {current_score:.3f}") self._emit_criteria_event(criteria_results) - if turn_record.max_turns_exhausted: - self.result.max_turns_exhausted = True - logger.warning( - "Agent exhausted max_turns (%s) without passing criteria.", - self.task.run_limits.max_turns if self.task.run_limits else None, - ) - - # Soft cumulative-turn check (logs once; never aborts). - self._check_expected_turns(iteration=iteration) - # Budget gate runs AFTER criteria so partial-credit visibility is preserved. + # (max_turns capture and the soft turn check are recorded above, before + # the grading switch — they are facts about the run, not verdicts.) self._check_run_limits(iteration=iteration) return all_passed @@ -2073,6 +2449,27 @@ async def _run_dialog_criteria_check( """ assert self.result is not None assert self.success_checker is not None + # Grading site 3 of 4. Unreachable today — `execute` rejects simulation + # tasks at the CLI, because the dialog's turn-continuation logic reads + # criteria results to decide whether to keep talking, so an ungraded + # dialog would silently change its own stopping behavior. + # + # RAISES rather than returning an empty list, matching the evaluate-only + # path's refusal. The empty-list version described itself as a + # "defensive no-op so the gate holds", and it was neither: both callers + # go straight on to `all_criteria_passed`, whose first act is a + # length pre-check that raises on a mismatch — and an empty criteria + # list is forbidden by `TaskDefinition.validate_success_criteria`, so + # the mismatch was guaranteed. If the simulation restriction ever lifts, + # that "no-op" turns every ungraded dialog into FinalStatus.ERROR. A + # loud refusal here is honest about the fact that this path has no + # ungraded semantics yet. + if not self.grade: + raise ValueError( + "Grading is disabled but the simulation dialog path requires criteria results to " + + "decide turn continuation. `execute` refuses simulation tasks at the CLI; reaching " + + "here means that refusal was bypassed." + ) await self._verify_reference_integrity() criteria_results = await self.success_checker.check_all_async( self.task.success_criteria, @@ -2507,7 +2904,8 @@ def _emit_criteria_event(self, criteria_results: list[CriterionResult]) -> None: pairs = list(zip(criteria_results, self.task.success_criteria, strict=True)) passed_count = sum(1 for r, c in pairs if r.score >= c.pass_threshold) total_count = len(pairs) - current_score = self.result.weighted_score or 0.0 + # Reached only from the grading path, where a score has been computed. + current_score = self.result.weighted_score or 0.0 # noqa: CE049 — graded here by construction criteria_details = [ f"{criterion.type}: {'PASS' if result.score >= criterion.pass_threshold else 'FAIL'}" + f" ({result.score:.2f})" @@ -2670,8 +3068,10 @@ async def _run_pre_run_commands(self) -> None: outer ``except Exception`` handler and lands the run as ``FinalStatus.ERROR``. Post-run commands and cleanup still execute via the ``finally`` block. + + Skipped entirely on an adopted sandbox — see ``_skip_hooks_for_adopted``. """ - if self.result is None: + if self.result is None or self._skip_hooks_for_adopted(self.task.pre_run, "pre_run"): return await self._run_command_list(self.task.pre_run, self.result.pre_run_results, "pre_run") @@ -2681,11 +3081,44 @@ async def _run_post_run_commands(self) -> None: See ``_run_command_list``. Post-run commands are informational only — ``fail_on_error`` is not part of ``PostRunCommand``, so failures are warning-logged and never affect the evaluation verdict. + + Skipped entirely on an adopted sandbox — see ``_skip_hooks_for_adopted``. """ - if self.result is None: + if self.result is None or self._skip_hooks_for_adopted(self.task.post_run, "post_run"): return await self._run_command_list(self.task.post_run, self.result.post_run_results, "post_run") + def _skip_hooks_for_adopted(self, commands: Sequence[PreRunCommand | PostRunCommand], phase: str) -> bool: + """True when ``phase``'s commands must not run against an adopted sandbox. + + ``adopt()`` guarantees it materializes nothing into the workspace, but + that guarantee is only as strong as its weakest caller: ``run()`` invokes + the pre/post-run hooks unconditionally, and those commands run with + ``cwd = sandbox_dir``. Several in-tree tasks stage fixtures there + (``cp -a /app/[!.]* "$PWD/"``), so re-running them during a detached + grade would overwrite the agent's deliverables *before* the criteria read + them — silently changing the verdict and destroying preserved artifacts. + + The hooks belong to the EXECUTE phase; the prior run already ran them, + and their recorded results are carried over by ``_seed_from_prior_result``. + + ``commands`` is passed in rather than looked up from ``phase``. The + lookup was a stringly-typed branch whose only consumer was a log line, so + a typo (``"prerun"``) would silently report the post_run count with + nothing — not pyright, not ruff — able to see it, in the same module + CE050 was written to protect from exactly that. + """ + if self.sandbox is None or not self.sandbox.was_adopted: + return False + if commands: + logger.info( + "Skipping %d %s command(s): the sandbox was adopted for grading, and re-running them " + + "would mutate the workspace under evaluation.", + len(commands), + phase, + ) + return True + async def _cleanup(self) -> None: """Clean up all resources.""" # Stop agent @@ -2749,8 +3182,15 @@ async def _cleanup(self) -> None: await asyncio.to_thread(self.sandbox.grant_read_access) logger.info(f"Sandbox preserved (in-place): {self.sandbox.sandbox_dir}") elif self.preservation_mode == PreservationMode.NONE and self.result: - # Sandbox will be deleted by cleanup() below; clear stale path. - self.result.sandbox_path = None + if self.sandbox.was_adopted: + # An adopted sandbox belongs to the caller and survives + # cleanup(), so the path is not stale — and clearing it + # would strip the graded row of its artifacts pointer + # (which a second grade needs to find the workspace). + self.result.sandbox_path = str(self.sandbox.sandbox_dir) + else: + # Sandbox will be deleted by cleanup() below; clear stale path. + self.result.sandbox_path = None elif self.result: # Defensive: a future PreservationMode member with no arm here # would otherwise silently fall through. Treat as no-preserve. diff --git a/src/coder_eval/path_utils.py b/src/coder_eval/path_utils.py index 44e47d91..1215b3fd 100644 --- a/src/coder_eval/path_utils.py +++ b/src/coder_eval/path_utils.py @@ -5,6 +5,7 @@ import logging import os import platform +import secrets import shutil from collections.abc import Callable from datetime import datetime @@ -15,6 +16,17 @@ TASK_LOG_FILENAME = "task.log" +# The per-task result record, and the pre-grade snapshot a detached grade keeps +# beside it. Module-level because ~12 sites name them — including three that +# `rglob` for the first — and two half-copies of the same string in different +# packages is how a rename becomes a silent no-op on the sites it missed. +TASK_JSON_FILENAME = "task.json" +PRE_GRADE_JSON_FILENAME = "task.execute.json" + +# The virtualenv directory `setup` creates and `adopt` discovers. Named because +# whether it is on PATH decides which binaries a criterion resolves. +VENV_DIRNAME = ".venv" + # Ignore list for every copy of a reference solution tree. A module-level # constant, not an inline literal at each call site: the host-side docker mount # (`DockerRunner._prepare_reference_mount`) and the per-run staged copy @@ -25,6 +37,62 @@ REFERENCE_COPY_IGNORE = [".git"] +def write_text_atomic(path: Path, text: str) -> None: + """Write ``text`` to ``path`` via a temp file + ``os.replace``. + + A plain ``write_text`` truncates first, so a SIGKILL or a full disk mid-write + leaves a half-file. For ``task.json`` that is worse than no file: a truncated + record parses as *malformed*, which the recovery paths treat as "not + complete" — so a later ``--resume`` re-executes the task and pays for the + agent again, and the row vanishes from ``run.json``. One writer, so the + orchestrator and the detached grade's write-back cannot have different crash + semantics for the same file. + + The temp file is opened ``O_CREAT | O_EXCL | O_NOFOLLOW`` under a name that + is UNIQUE per call. Without ``O_NOFOLLOW`` a pre-planted ``task.json.tmp`` + *symlink* in a shared run directory makes this an arbitrary-file-overwrite + primitive — and one that bypasses the destination symlink refusal in + ``evaluate``'s write-back, since the guard checks the destination while the + truncation happens through the temp name. + + The name must be unique, not fixed, and that is a correctness requirement + rather than tidiness. ``os.replace`` is the only step that can be interrupted + without trace, and this function exists precisely because the process may be + SIGKILLed (the docker host-heartbeat watchdog does exactly that) — so a + crash between ``open`` and ``replace`` WILL sometimes leave the temp file + behind. Under a fixed name, ``O_EXCL`` then turned that leftover into a + permanent refusal to write the record at all: the row reported ERROR, and + ``--resume`` saw no ``task.json``, re-ran the task into the same run dir, and + hit the same stale file — an unbounded loop that re-pays for the agent every + time. A unique name keeps ``O_EXCL``'s guarantee while making a leftover + inert. It can litter a dead ``.tmp`` beside the record after a hard kill; + that is strictly better than wedging finalization, and the litter is + recognisable by its embedded pid. + + Mode is ``0o644`` — the same mode a plain ``write_text`` produced, and the + widest one that is never group- or world-*writable* whatever the umask. + Creating it 0600 broke the docker driver on Linux: + the in-container orchestrator writes ``task.json`` as root straight into the + bind-mounted host run dir, and the host then reads it back as the invoking + uid — an unguarded read that raises ``PermissionError`` for every task. A + result record is not a secret, and the symlink hazard is closed by + ``O_NOFOLLOW`` and the unpredictable name rather than by the mode. + """ + # pid + random: unique across concurrent writers AND across a crashed + # predecessor, so O_EXCL can never collide with our own leftovers. + tmp = path.with_name(f"{path.name}.{os.getpid()}.{secrets.token_hex(4)}.tmp") + flags = os.O_CREAT | os.O_EXCL | os.O_WRONLY | getattr(os, "O_NOFOLLOW", 0) + fd = os.open(tmp, flags, 0o644) + try: + with os.fdopen(fd, "w", encoding="utf-8") as fh: + fh.write(text) + os.replace(tmp, path) + except BaseException: + with contextlib.suppress(OSError): + os.unlink(tmp) + raise + + def digest_tree(root: Path) -> str: """Content hash of every file under ``root``, stable across runs. diff --git a/src/coder_eval/reports.py b/src/coder_eval/reports.py index 88345636..d04488d0 100644 --- a/src/coder_eval/reports.py +++ b/src/coder_eval/reports.py @@ -20,7 +20,7 @@ row_cost_incomplete, sum_costs, ) -from .path_utils import build_task_run_dir +from .path_utils import TASK_JSON_FILENAME, build_task_run_dir if TYPE_CHECKING: @@ -237,8 +237,24 @@ def early_stop_gate_note(reason: str) -> str: def _pass_rate_lines(summary: RunSummary) -> list[str]: - """The pass rate over every dispatched task, plus the error share when non-zero.""" - lines = [f"- **Pass Rate**: {_fmt_rate(summary.pass_rate)} ({summary.tasks_succeeded}/{summary.tasks_run})"] + """The pass rate over every GRADED task, plus the error share when non-zero. + + An ungraded run (``coder-eval execute``) has no pass rate at all, so it says + so rather than rendering ``0.0% (0/N)`` — which reads as a total failure. + """ + # Only an ungraded run gets the explanatory line. An ordinary EMPTY run keeps + # its original "n/a (0/0)" rendering — the two are different facts. + # + # `pass_rate is None` rather than `not tasks_graded`: an execute night with a + # crashed row has tasks_graded > 0 (an ERROR row is category `error`, not + # `ungraded`, so it stays in the denominator) while still having measured + # nothing — and this line then rendered `0.0% (0/5)` plus `Error Share: + # 100.0%`, exactly the total-failure reading the guard exists to prevent. + # Deferring to the model keeps one rule for run.md, run.json and the + # evalboard instead of three. + if summary.tasks_not_graded and summary.pass_rate is None: + return [f"- **Pass Rate**: n/a — {summary.tasks_not_graded} task(s) executed without grading"] + lines = [f"- **Pass Rate**: {_fmt_rate(summary.pass_rate)} ({summary.tasks_succeeded}/{summary.tasks_graded})"] if summary.tasks_error: lines.append( f"- **Error Share**: {_fmt_rate(summary.error_share)} of tasks never produced a " @@ -389,6 +405,7 @@ def _summary_section_lines(summary: RunSummary) -> list[str]: f"- **Succeeded**: {summary.tasks_succeeded}", failed_line, f"- **Errors**: {summary.tasks_error}", + *([f"- **Not Graded**: {summary.tasks_not_graded}"] if summary.tasks_not_graded else []), *_pass_rate_lines(summary), ] @@ -725,7 +742,7 @@ def _aggregate_command_statistics(run_dir: Path) -> CommandStatistics | None: all_turns: list[TurnRecord] = [] # Find all task.json files recursively to handle both flat and nested (experiment) layouts - for report_path in run_dir.rglob("task.json"): + for report_path in run_dir.rglob(TASK_JSON_FILENAME): if "artifacts" in report_path.parts or ".git" in report_path.parts: continue try: @@ -859,6 +876,10 @@ def _compute_suite_rollup( rows_passed = sum(1 for r in rows if r.result.final_status.category == "succeeded") rows_failed = sum(1 for r in rows if r.result.final_status.category == "failed") rows_error = sum(1 for r in rows if r.result.final_status.category == "error") + # An ungraded row was never measured, so it leaves BOTH sides of the rate — + # the same rule RunSummary.pass_rate and VariantAggregate.pass_rate follow. + rows_not_graded = sum(1 for r in rows if r.result.final_status.category == "ungraded") + rows_graded = rows_total - rows_not_graded scored = [r.result.weighted_score for r in rows if r.result.weighted_score is not None] average_weighted_score = sum(scored) / len(scored) if scored else None @@ -900,8 +921,10 @@ def _compute_suite_rollup( per_rows = [ row.result.success_criteria_results[i] for row in rows if i < len(row.result.success_criteria_results) ] - suite_thresholds = getattr(criterion, "suite_thresholds", None) - description = getattr(criterion, "description", None) + # Both are declared on BaseSuccessCriterion, so they are typed + # attributes on every union member — no string probe needed. + suite_thresholds = criterion.suite_thresholds + description = criterion.description try: checker_cls = CriterionRegistry.get_checker(ctype) except KeyError: @@ -929,7 +952,12 @@ def _compute_suite_rollup( # Sample up to K failed/errored rows for error analysis failed_samples: list[FailedRowSummary] = [] for row in rows: - if row.result.final_status.category == "succeeded": + # "succeeded" is not the only non-failure. An UNGRADED row was never + # measured, so it has no failure reasons to report and listing it here + # (in a field documented as failed/errored rows) contradicts the same + # function's own rule two blocks up, where it leaves both sides of the + # pass rate. + if row.result.final_status.category in ("succeeded", "ungraded"): continue if len(failed_samples) >= _FAILED_SAMPLE_LIMIT: break @@ -945,7 +973,8 @@ def _compute_suite_rollup( if len(reasons) >= _FAILURE_REASONS_PER_ROW: break task_json_path = ( - build_task_run_dir(run_dir, variant_id, row.task_id, replicate_index=row.replicate_index) / "task.json" + build_task_run_dir(run_dir, variant_id, row.task_id, replicate_index=row.replicate_index) + / TASK_JSON_FILENAME ) try: # Persist as POSIX — this value lands in suite.json and in @@ -973,7 +1002,10 @@ def _compute_suite_rollup( rows_passed=rows_passed, rows_failed=rows_failed, rows_error=rows_error, - pass_rate=rows_passed / rows_total if rows_total else 0.0, + rows_not_graded=rows_not_graded, + # None, never 0.0: a suite where nothing was graded has no pass rate, + # and 0.0 renders as "0.0%" beside a full set of rows. + pass_rate=rows_passed / rows_graded if rows_graded else None, average_weighted_score=average_weighted_score, criterion_stats=criterion_stats, failed_samples=failed_samples, @@ -989,10 +1021,15 @@ def _render_suite_markdown(rollup: SuiteRollup) -> str: "", f"**Variant**: `{rollup.variant_id}`", ( - f"**Rows**: {rollup.rows_total} total — " - f"{rollup.rows_passed} passed, {rollup.rows_failed} failed, {rollup.rows_error} errored" + f"**Rows**: {rollup.rows_total} total — {rollup.rows_passed} passed, " + + f"{rollup.rows_failed} failed, {rollup.rows_error} errored" + + (f", {rollup.rows_not_graded} not graded" if rollup.rows_not_graded else "") + ), + ( + f"**Pass rate**: {rollup.pass_rate * 100:.1f}% ({rollup.rows_passed}/{rollup.rows_graded})" + if rollup.pass_rate is not None + else "**Pass rate**: n/a (no rows were graded)" ), - f"**Pass rate**: {rollup.pass_rate * 100:.1f}%", ] if rollup.average_weighted_score is not None: lines.append(f"**Average weighted score**: {rollup.average_weighted_score:.3f}") @@ -1187,12 +1224,12 @@ def write_suite_rollups( (suite_dir / "suite.md").write_text(_render_suite_markdown(rollup), encoding="utf-8") rollups.append(rollup) logger.info( - "Wrote suite rollup: variant=%s suite=%s pass_rate=%.1f%% (%d/%d) gate=%s", + "Wrote suite rollup: variant=%s suite=%s pass_rate=%s (%d/%d) gate=%s", variant_id, suite_id, - rollup.pass_rate * 100, + f"{rollup.pass_rate * 100:.1f}%" if rollup.pass_rate is not None else "n/a", rollup.rows_passed, - rollup.rows_total, + rollup.rows_graded, "PASS" if rollup.passed else "FAIL", ) return rollups diff --git a/src/coder_eval/reports_experiment.py b/src/coder_eval/reports_experiment.py index e141174b..dc3b5eb6 100644 --- a/src/coder_eval/reports_experiment.py +++ b/src/coder_eval/reports_experiment.py @@ -26,6 +26,8 @@ describe_prompt_config, fmt_mean_sd, fmt_p, + format_score, + is_env_table_key, load_variant_eval_results, paired_comparison, stddev, @@ -248,7 +250,8 @@ def generate_task_report(summary: TaskExperimentSummary) -> str: tokens_str = f"{v.total_tokens:,}" if v.total_tokens is not None else "N/A" avg_dur = v.duration_seconds / v.replicate_count lines.append( - f"| {v.variant_id} | {v.weighted_score:.3f} | {v.final_status}" + f" | {avg_dur:.1f}s | {tokens_str} |" + f"| {v.variant_id} | {format_score(v.weighted_score)} | {v.final_status}" + + f" | {avg_dur:.1f}s | {tokens_str} |" ) return "\n".join(lines) @@ -343,7 +346,19 @@ def _aggregate_count_rows(result: ExperimentResult, show_p_values: bool) -> list row += " | —" lines.append(row + " |") - # Every task the variant ran is in the denominator, errors included. + # Row: Not Graded — conditional, like the budget sub-rows above. Without + # it Tasks Run / Succeeded / Failed / Errors stop summing to tasks_run on + # an ungraded run, with nothing in the table to say where the rest went. + if any(result.variant_aggregates[vid].tasks_not_graded > 0 for vid in result.variant_ids): + row = "| Not Graded" + for vid in result.variant_ids: + row += f" | {result.variant_aggregates[vid].tasks_not_graded}" + if show_p_values: + row += " | —" + lines.append(row + " |") + + # Every task the variant GRADED is in the denominator, errors included; + # ungraded rows leave both sides. row = "| Pass Rate" for vid in result.variant_ids: rate = result.variant_aggregates[vid].pass_rate @@ -483,7 +498,7 @@ def _win_loss_lines(result: ExperimentResult) -> list[str]: vr = scores_by_variant.get(vid) if vr: status_icon = vr.final_status.icon - cells.append(f"{vr.weighted_score:.3f} ({status_icon})") + cells.append(f"{format_score(vr.weighted_score)} ({status_icon})") else: cells.append("N/A") best_str = f"{'TIE' if ts.is_tie else ts.best_variant}" @@ -626,8 +641,11 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir: f"- **Succeeded**: {agg.tasks_succeeded}", failed_line, f"- **Errors**: {agg.tasks_error}", - f"- **Pass Rate**: {pass_rate_str} ({agg.tasks_succeeded}/{agg.tasks_run})", - f"- **Average Score**: {agg.average_score:.3f}", + *([f"- **Not Graded**: {agg.tasks_not_graded}"] if agg.tasks_not_graded else []), + # Denominator is the GRADED count, matching VariantAggregate.pass_rate — + # an ungraded task was never measured and belongs on neither side. + f"- **Pass Rate**: {pass_rate_str} ({agg.tasks_succeeded}/{agg.tasks_graded})", + f"- **Average Score**: {format_score(agg.average_score)}", f"- **Average Duration**: {agg.average_duration:.1f}s", f"- **Total Tokens**: {tokens_str}", ] @@ -636,7 +654,7 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir: variant_results = [ vr for ts in result.task_summaries for vr in ts.variant_results if vr.variant_id == variant_id ] - scores = [vr.weighted_score for vr in variant_results] + scores = [vr.weighted_score for vr in variant_results if vr.weighted_score is not None] durations = [vr.duration_seconds / vr.replicate_count for vr in variant_results] if scores and len(scores) >= 2: @@ -670,7 +688,8 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir: for vr in ts.variant_results: if vr.variant_id == variant_id: avg_duration = vr.duration_seconds / vr.replicate_count - row = f"| {ts.task_id} | {vr.weighted_score:.3f} | {vr.final_status} | {avg_duration:.1f}s |" + score_text = format_score(vr.weighted_score) + row = f"| {ts.task_id} | {score_text} | {vr.final_status} | {avg_duration:.1f}s |" if has_reps: row += f" {vr.replicate_count} |" if has_similarity: @@ -717,7 +736,7 @@ def generate_variant_report(variant_id: str, result: ExperimentResult, run_dir: # Environment (from first result with data) for er in eval_results: if er.environment_info: - env = {k: v for k, v in er.environment_info.items() if k != "installed_tools"} + env = {k: v for k, v in er.environment_info.items() if is_env_table_key(k)} if env: lines.extend(["", "## Environment", ""]) for key, value in env.items(): diff --git a/src/coder_eval/reports_html.py b/src/coder_eval/reports_html.py index def6591f..7f8fcade 100644 --- a/src/coder_eval/reports_html.py +++ b/src/coder_eval/reports_html.py @@ -20,6 +20,7 @@ from coder_eval.models import FinalStatus, eval_result_total_cost, sum_costs from .reports import early_stop_gate_note +from .reports_stats import format_score, is_env_table_key if TYPE_CHECKING: @@ -283,7 +284,10 @@ def _status_badge(status: Any) -> str: status_str = getattr(status, "value", None) or str(status) try: fs = status if isinstance(status, FinalStatus) else FinalStatus(str(status)) - cls = {"succeeded": "success", "failed": "failure", "error": "error"}[fs.category] + # "ungraded" -> neutral: the row carries no verdict, so it must render as + # neither green nor red. Same class an unrecognised status falls back to, + # reached deliberately here rather than by accident. + cls = {"succeeded": "success", "failed": "failure", "error": "error", "ungraded": "neutral"}[fs.category] except (ValueError, KeyError): cls = "neutral" # unknown / non-FinalStatus input return f'{_esc(status_str)}' @@ -1070,8 +1074,8 @@ def _render_simulation(result: EvaluationResult) -> str: def _render_environment(result: EvaluationResult) -> str: - """Render Environment section (excluding installed_tools, which has its own).""" - env = {k: v for k, v in (result.environment_info or {}).items() if k != "installed_tools"} + """Render Environment section (excluding the keys with their own treatment).""" + env = {k: v for k, v in (result.environment_info or {}).items() if is_env_table_key(k)} if not env: return "" rows = "".join(f"{_esc(k)}{_esc(v)}" for k, v in env.items()) @@ -1121,7 +1125,7 @@ def _variant_stddev_lines(variant_id: str, result: ExperimentResult | None) -> s from .reports_stats import stddev vrs = [vr for ts in result.task_summaries for vr in ts.variant_results if vr.variant_id == variant_id] - scores = [vr.weighted_score for vr in vrs] + scores = [vr.weighted_score for vr in vrs if vr.weighted_score is not None] durations = [vr.duration_seconds for vr in vrs] extras: list[str] = [] if len(scores) >= 2: @@ -1355,6 +1359,17 @@ def _row(label: str, values: list[str], p: str | None) -> str: ) rows.append(_row("Failed", [str(result.variant_aggregates[vid].tasks_failed) for vid in result.variant_ids], None)) rows.append(_row("Errors", [str(result.variant_aggregates[vid].tasks_error) for vid in result.variant_ids], None)) + # The fourth bucket, conditionally like its siblings elsewhere. Without it + # Tasks Run / Succeeded / Failed / Errors no longer sum to tasks_run on an + # ungraded run, with nothing on the page to say where the rest went. + if any(result.variant_aggregates[vid].tasks_not_graded > 0 for vid in result.variant_ids): + rows.append( + _row( + "Not Graded", + [str(result.variant_aggregates[vid].tasks_not_graded) for vid in result.variant_ids], + None, + ) + ) def _pass_rate(vid: str) -> str: rate = result.variant_aggregates[vid].pass_rate @@ -1448,7 +1463,7 @@ def _experiment_per_task_comparison(result: ExperimentResult) -> str: if vr is None: cells.append("N/A") else: - cells.append(f"{vr.weighted_score:.3f} ({_esc(vr.final_status.icon)})") + cells.append(f"{format_score(vr.weighted_score)} ({_esc(vr.final_status.icon)})") best = "TIE" if ts.is_tie else ts.best_variant cells.append(f"{_esc(best)}") cells.append(f"{ts.score_spread:.3f}") @@ -1578,6 +1593,15 @@ def generate_variant_html( ) stddev_lines = _variant_stddev_lines(variant_id, result) rich_sections = _variant_rich_sections(variant_id, result, run_dir) + # Only rendered when non-zero, so an ordinary graded run's tile is + # unchanged — but a `coder-eval execute` run says where its tasks went + # instead of showing Succeeded/Failed/Errors all at zero. + ungraded_stat = ( + '
Not Graded
' + + f'
{agg.tasks_not_graded}
' + if agg.tasks_not_graded > 0 + else "" + ) budget_stats = "" if agg.tasks_token_budget_exceeded > 0: budget_stats += ( @@ -1606,6 +1630,7 @@ def generate_variant_html(
Succeeded
{agg.tasks_succeeded}
Failed
{agg.tasks_failed}
Errors
{agg.tasks_error}
+ {ungraded_stat} {budget_stats} {stddev_lines} @@ -1647,7 +1672,7 @@ def generate_experiment_html( {_esc(vid)} {_score_pill(agg.average_score)} - {agg.tasks_succeeded}/{agg.tasks_run} + {agg.tasks_succeeded}/{agg.tasks_graded} """ ) diff --git a/src/coder_eval/reports_junit.py b/src/coder_eval/reports_junit.py index 133f9187..069dd53d 100644 --- a/src/coder_eval/reports_junit.py +++ b/src/coder_eval/reports_junit.py @@ -29,6 +29,7 @@ from .evaluation.judge_context import truncate from .models import FinalStatus, RunSummary, SuiteRollup +from .path_utils import TASK_JSON_FILENAME logger = logging.getLogger(__name__) @@ -54,7 +55,7 @@ def _xml_safe(text: str) -> str: return _ILLEGAL_XML.sub("", text) -def _category_of(status: str) -> Literal["succeeded", "failed", "error"]: +def _category_of(status: str) -> Literal["succeeded", "failed", "error", "ungraded"]: """Map a serialized status string to a reporting category via the SSOT. Goes through ``FinalStatus(value).category`` (an explicit allowlist, CE018); @@ -169,14 +170,14 @@ def _load_task_json(run_dir: Path, row: dict[str, Any], variant: str) -> dict[st replicate_index = row.get("replicate_index") task_dir = run_dir / variant / task_id if isinstance(replicate_index, int) and not isinstance(replicate_index, bool): - candidate = task_dir / f"{replicate_index:02d}" / "task.json" + candidate = task_dir / f"{replicate_index:02d}" / TASK_JSON_FILENAME else: - matches = sorted(task_dir.glob("*/task.json")) + matches = sorted(task_dir.glob(f"*/{TASK_JSON_FILENAME}")) # With no replicate index, picking one of several would misattribute # another replicate's failure detail to this row — degrade instead. if len(matches) > 1: return None - candidate = matches[0] if matches else task_dir / "task.json" + candidate = matches[0] if matches else task_dir / TASK_JSON_FILENAME try: # Belt-and-braces containment check (catches symlink escapes too). @@ -303,6 +304,14 @@ def _task_case(row: dict[str, Any], run_dir: Path) -> ET.Element: if category == "succeeded": return case + if category == "ungraded": + # `coder-eval execute`: the task ran but was deliberately not scored. + # is JUnit's only "no verdict" element — reporting it as a + # would turn a healthy ungraded run red in CI, and reporting + # it as a pass would invent a verdict. _set_counts already counts these. + ET.SubElement(case, "skipped", {"message": "not graded (coder-eval execute)"}) + return case + message = status if status in _KNOWN_STATUSES else f"unknown status: {status}" tag = "failure" if category == "failed" else "error" child = ET.SubElement(case, tag, {"message": _xml_safe(message)}) diff --git a/src/coder_eval/reports_stats.py b/src/coder_eval/reports_stats.py index 25b8c1c4..ba9f9de3 100644 --- a/src/coder_eval/reports_stats.py +++ b/src/coder_eval/reports_stats.py @@ -16,6 +16,8 @@ from coder_eval.models import EvaluationResult, ExperimentResult, ExperimentVariant, TaskExperimentSummary +from .path_utils import TASK_JSON_FILENAME + logger = logging.getLogger(__name__) @@ -310,6 +312,30 @@ class VariantSeries(NamedTuple): asst_turns: list[float] +# environment_info keys the Environment table must NOT render as ordinary rows. +# `installed_tools` has its own dedicated section; the rest are harness +# bookkeeping the reader did not ask for — `command_base_path` is a full PATH +# string on every row, and the graded_by_* provenance keys only appear on a +# re-graded row where they would read as facts about the run itself. +ENV_TABLE_EXCLUDE = frozenset({"installed_tools", "command_base_path", "reference_digest"}) + + +def is_env_table_key(key: str) -> bool: + """Whether ``key`` belongs in a rendered Environment table.""" + return key not in ENV_TABLE_EXCLUDE and not key.startswith("graded_by_") + + +# What an ungraded row shows where a score would go. Deliberately not "0.000": +# an ungraded task was never measured, and a zero is indistinguishable from a +# task that was measured and scored nothing. +UNGRADED_SCORE_TEXT = "n/a" + + +def format_score(score: float | None) -> str: + """Render a weighted score for a report table, or ``n/a`` when ungraded.""" + return UNGRADED_SCORE_TEXT if score is None else f"{score:.3f}" + + def collect_variant_series(result: ExperimentResult) -> dict[str, VariantSeries]: """Per-variant (scores, durations, tokens, assistant-turns) series, keyed by variant id. @@ -324,7 +350,22 @@ def collect_variant_series(result: ExperimentResult) -> dict[str, VariantSeries] s = series.get(vr.variant_id) if s is None: # a task result for a variant not in variant_ids continue - s.scores.append(vr.weighted_score) + # Only the SCORE is dropped when there is none — never the row. + # Duration, tokens and assistant turns are facts about the run that + # grading has nothing to do with, and `execute`'s stated contract is + # that only the verdict is withheld. Skipping the row whole made an + # all-ungraded experiment render `Avg Duration | N/A | N/A` with the + # Tokens and Assistant Turns rows absent entirely. + # + # The series are consumed independently (each statistic reads one + # list), so they need not be index-aligned with each other; + # `paired_comparison` pairs across VARIANTS by task id, not by index + # into these lists. An earlier note here claimed an experiment is + # either entirely graded or entirely ungraded because `grade` is + # run-level — `run --resume` grades rows independently and folds a + # failed one back ungraded, so mixed experiments are real. + if vr.weighted_score is not None: + s.scores.append(vr.weighted_score) s.durations.append(vr.duration_seconds / vr.replicate_count) if vr.total_tokens is not None: s.tokens.append(float(vr.total_tokens)) @@ -488,7 +529,7 @@ def load_variant_eval_results( if not task_dir.is_dir(): continue for rep_subdir in sorted(task_dir.glob("[0-9][0-9]")): - task_json = rep_subdir / "task.json" + task_json = rep_subdir / TASK_JSON_FILENAME if task_json.exists(): try: results.append(EvaluationResult.model_validate_json(task_json.read_text(encoding="utf-8"))) diff --git a/src/coder_eval/sandbox.py b/src/coder_eval/sandbox.py index 55a24b5a..a19f6bc8 100644 --- a/src/coder_eval/sandbox.py +++ b/src/coder_eval/sandbox.py @@ -23,6 +23,7 @@ StarterFilesSource, TemplateDirSource, ) +from .path_utils import VENV_DIRNAME from .resources import get_ignore_patterns, should_ignore_path @@ -163,6 +164,12 @@ def __init__( self.sandbox_dir: Path | None = None self.venv_dir: Path | None = None self._cleanup_on_exit = True + # True once adopt() takes over an existing workspace. Read by the + # Orchestrator to suppress every step that would MUTATE the tree it was + # asked to grade (pre_run/post_run above all — several in-tree tasks + # copy fixtures over the workspace there) and to keep sandbox_path in + # the result, since an adopted directory outlives cleanup(). + self.was_adopted = False self.installed_tool_versions: dict[str, str] = {} self._command_base_path: str | None = None # Cached canonical `node_modules/@uipath`; pins UiPath CLI plugin discovery @@ -266,6 +273,97 @@ def setup(self, target_dir: Path | None = None) -> Path: ) raise ValueError(f"Unsupported sandbox driver: {self.config.driver}") + def adopt(self, workspace: Path) -> Path: + """Use ``workspace`` **as** the sandbox, materializing nothing into it. + + The grade-in-place counterpart to :meth:`setup`. ``setup`` builds a + workspace: it copies template sources in, generates ``record_cli`` shims, + creates a venv, installs packages. ``adopt`` takes a workspace that + already exists — an ``execute`` run's artifacts, or a verifier's ``/app`` + — and only derives the *environment* the criteria need to run against it + (mock-dir ``+x``, venv discovery, the plugin-tools pin). + + Why not ``setup(target_dir=workspace)``: that already adopts a + caller-supplied directory and sets ``_cleanup_on_exit=False``, but it + then runs ``_setup_template()``, which would write over the very files + it was asked to grade. + + In-place is more CORRECT here, not merely faster: + + * ``_setup_template`` filters what it copies through + ``_should_ignore_template_file`` — ``node_modules``, ``dist``, + ``build``, ``venv``, ``.git`` and friends are dropped. A criterion like + ``test -f dist/bundle.js`` therefore fails as a *copying artifact* + rather than as a verdict on the agent's work. + * ``run_command`` criteria execute with ``cwd = sandbox_dir``, so on the + copy path they see the copy's paths, not the ones the agent worked at. + * Copying a real workspace costs minutes. + + "Materializing nothing" means it writes no FILES. It does still chmod + ``+x`` over the task's declared mock-PATH directories inside the tree — + a mode change the criteria need in order to resolve the same shimmed + binaries the agent did. + + The caller keeps ownership: ``_cleanup_on_exit`` stays False, so + ``cleanup()`` never deletes an adopted directory. Criteria CAN still + mutate it (a ``run_command`` that writes), which is why the copy path + remains the default for a bare user-supplied work dir. + + Args: + workspace: An existing directory to grade in place. + + Returns: + Path to the sandbox directory (``workspace``). + + Raises: + RuntimeError: If the driver is ``docker`` (a container workspace is + not reachable from the host), or ``workspace`` is not a directory. + """ + if self.config.driver == "docker": + raise RuntimeError( + "Sandbox.adopt() is host-side only; a driver='docker' workspace lives inside " + + "the container. Grade it from within the container, or copy it out first." + ) + if not workspace.is_dir(): + raise RuntimeError(f"Cannot adopt {workspace}: not an existing directory") + + self.sandbox_dir = workspace.resolve() + # Never flipped True: an adopted directory belongs to the caller. + self._cleanup_on_exit = False + self.was_adopted = True + + # Only NON-materializing steps below. Deliberately skipped, and why: + # _setup_template would overwrite the workspace being graded + # _generate_cli_recorders writes shims into it + # _setup_virtualenv / + # _install_*_packages the execute phase already provisioned these; + # re-running mutates the graded tree + # _maybe_remediate_home_plugins_pollution + # destructive on $HOME, and it is remediation + # rather than derivation — the execute phase + # already ran it if it was enabled + self._prepare_mock_path_dirs() + + # Discover an existing venv instead of creating one, so `run_command` + # criteria get the same VIRTUAL_ENV/PATH the agent had. Absent venv -> + # None, exactly as for a task with no python config. + # + # Gated on `config.python` for the same reason `setup` is: venv_dir + # prepends the venv's bin/ to PATH and exports VIRTUAL_ENV for every + # criterion subprocess, so discovering one a task never asked for grades + # it under a PATH it never ran under — the exact divergence the + # command_base_path round trip exists to close. It would also let an + # agent shadow binaries by writing `.venv/bin/` into its own workspace. + if self.config.python: + candidate = self.sandbox_dir / VENV_DIRNAME + if candidate.is_dir(): + self.venv_dir = candidate + + self._check_parent_node_modules_contamination() + self._refresh_plugin_tools_dir() + + return self.sandbox_dir + def _setup_tempdir(self, target_dir: Path | None = None) -> Path: """Set up a sandbox directory. @@ -353,7 +451,13 @@ def _apply_repo_source(self, source: RepoSource) -> None: assert self.sandbox_dir is not None, "Sandbox directory not initialized" repo_dir = self.sandbox_dir / "repo" - cmd = ["git", "clone", source.url, str(repo_dir)] + # `--` before the URL: it is argv position 2, so without the separator a + # value beginning with `-` is parsed by git as an OPTION rather than a + # repository (`--upload-pack=…` runs a command of the caller's choosing). + # That URL is task-authored, and since `evaluate ` rebuilds the + # task from a shareable run directory it is no longer necessarily the + # operator's own string. + cmd = ["git", "clone", "--", source.url, str(repo_dir)] try: subprocess.run(cmd, check=True, capture_output=True, text=True, encoding="utf-8", timeout=60) @@ -716,7 +820,7 @@ def _setup_virtualenv(self) -> None: if not self.sandbox_dir: raise RuntimeError("Sandbox directory not initialized") - self.venv_dir = self.sandbox_dir / ".venv" + self.venv_dir = self.sandbox_dir / VENV_DIRNAME # Use uv to create virtual environment (faster than venv) try: @@ -1192,6 +1296,46 @@ def run_command(self, command: str, timeout: float | int | None = None) -> tuple # needs filesystem access beyond the sandbox root (e.g., reading installed packages, # system headers). Path traversal protection is handled at the agent permission level. + def _within_sandbox(self, candidate: Path) -> bool: + """Whether a resolved criterion path stays inside the sandbox. + + The read-side twin of :meth:`_resolve_within_sandbox`, which every OTHER + task-authored path already goes through. Criterion paths were the one + consumer that skipped it, and ``Path('/tmp/sandbox') / '/etc/passwd'`` is + ``/etc/passwd`` — pathlib discards the prefix on an absolute right + operand — so ``file_contains`` / ``file_check`` / ``file_matches_regex`` + were a pass-fail oracle over any file the grading user could read, and + ``json_check`` could surface parsed values in ``details``. + + That was defensible while a task YAML was operator-supplied. It stopped + being so when ``evaluate `` began rebuilding the criteria list + from a shareable run directory. + + Returns False rather than raising: an out-of-sandbox path is + indistinguishable to the criterion from a file that is not there, which + is the same answer the template and mock-dir paths give, and raising + here would book a config error as an agent crash (CE039). + + Silent by design — :meth:`resolve_files` reports the escape ONCE per + criterion, naming the pattern the task author actually wrote. Logging + here instead named a resolved absolute path (uninformative: it is the + author's own string joined onto a tempdir) once per rejected glob + match, so a wide pattern produced a burst of near-identical warnings. + """ + assert self.sandbox_dir is not None + root = self.sandbox_dir.resolve() + try: + resolved = candidate.resolve() + except OSError: + return False + return resolved == root or root in resolved.parents + + def _warn_escaped(self, path: str) -> None: + """Report that a criterion's declared path left the sandbox.""" + logger.warning( + "Criterion path %r resolves outside the sandbox (%s); treating it as no match.", path, self.sandbox_dir + ) + def resolve_files(self, path: str) -> list[Path]: """Resolve a criterion ``path`` to the sandbox files it addresses. @@ -1226,7 +1370,10 @@ def resolve_files(self, path: str) -> list[Path]: # Literal first: an existing path is never reinterpreted as a pattern. candidate = self.sandbox_dir / path if candidate.exists(): - return [candidate] + if self._within_sandbox(candidate): + return [candidate] + self._warn_escaped(path) + return [] if not _is_glob(path): return [] @@ -1235,14 +1382,20 @@ def resolve_files(self, path: str) -> list[Path]: pinned = {segment for segment in path.split("/") if segment and not _is_glob(segment)} matches: list[Path] = [] + escaped = False for match in self.sandbox_dir.glob(path): if not match.is_file(): continue + if not self._within_sandbox(match): + escaped = True + continue discovered = [part for part in match.relative_to(self.sandbox_dir).parts if part not in pinned] if discovered and should_ignore_path(Path(*discovered), patterns): continue matches.append(match) + if escaped: + self._warn_escaped(path) return sorted(matches) def resolved_path_label(self, path: str) -> str | None: diff --git a/tests/lint/rules/ce018_no_final_status_name_denylist.py b/tests/lint/rules/ce018_no_final_status_name_denylist.py index 2e9c12a1..1a3b4f96 100644 --- a/tests/lint/rules/ce018_no_final_status_name_denylist.py +++ b/tests/lint/rules/ce018_no_final_status_name_denylist.py @@ -35,6 +35,7 @@ "MAX_TURNS_EXHAUSTED", "TOKEN_BUDGET_EXCEEDED", "COST_BUDGET_EXCEEDED", + "NOT_GRADED", } ) diff --git a/tests/lint/rules/ce047_env_info_key_round_trip.py b/tests/lint/rules/ce047_env_info_key_round_trip.py new file mode 100644 index 00000000..6638077a --- /dev/null +++ b/tests/lint/rules/ce047_env_info_key_round_trip.py @@ -0,0 +1,126 @@ +"""CE047: every ``environment_info`` key that is READ must also be WRITTEN. + +``EvaluationResult.environment_info`` is a ``dict[str, Any]`` bag, so nothing — +not pydantic, not pyright — connects the site that writes a key to the site that +reads it back. A reader whose writer was never added (or was later removed) is +silently inert: ``.get("k")`` returns ``None``, the guard takes its early return, +and the feature reports success while doing nothing. + +The motivating case: ``verify_reference_unchanged`` read +``environment_info.get("reference_digest")`` to refuse a re-grade whose answer key +had changed. Nothing anywhere wrote that key — a whole-tree grep found exactly one +occurrence, the read itself. The anti-cheat guard shipped, was documented in +CLAUDE.md and the user guide as protection, and never fired once. Every automated +gate in the repo was green. + +This is deliberately a one-way check. An unread key is ordinary (recorded for a +human or a downstream consumer); an unwritten key is always a bug. + +Use ``# noqa: CE047`` for a key genuinely supplied from outside this repo. +""" + +import ast +import re +from pathlib import Path + +from tests.lint.rules.base import BaseRule + + +_SRC_ROOT = Path("src/coder_eval") + +# Keys written by a consumer outside src/ (the docker container's own capture, +# a plugin) or copied wholesale from another dict. Each needs a reason. +_EXTERNALLY_WRITTEN: dict[str, str] = {} + + +def _written_keys() -> set[str]: + """Every string literal assigned into an ``environment_info`` subscript. + + Text-scanned rather than AST-walked across the tree so a write inside any + module counts regardless of how the dict was reached (``self.result.``, + ``result.``, a local alias). The rule only needs to know a literal is + written SOMEWHERE — attributing it precisely would add false positives + without catching anything more. + """ + written: set[str] = set() + pattern = re.compile(r"""environment_info\[\s*["']([\w.-]+)["']\s*\]\s*=""") + # Also count keys named in a dict literal that becomes environment_info, and + # the f-string-built provenance keys (`f"graded_by_{key}"`), which no literal + # scan can resolve — those are covered by the prefix allowance below. + for path in sorted(_SRC_ROOT.rglob("*.py")): + try: + text = path.read_text(encoding="utf-8") + except OSError: # pragma: no cover - unreadable file in src is not our problem + continue + written.update(pattern.findall(text)) + return written + + +class EnvInfoKeyRoundTrip(BaseRule): + id = "CE047" + + # `(^|sep)`, not a bare leading separator: a repo-relative path + # ("src/coder_eval/x.py") is how every caller in tests/ addresses a file, + # and requiring the separator silently put those out of scope — which + # would make a rule-behaviour test pass while proving nothing. + _SRC_PATH = re.compile(r"(?:^|[/\\])src[/\\]coder_eval[/\\]") + _written: set[str] | None = None + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = bool(self._SRC_PATH.search(filepath)) + if self._in_scope and EnvInfoKeyRoundTrip._written is None: + EnvInfoKeyRoundTrip._written = _written_keys() + + def visit_Call(self, node: ast.Call) -> None: + self._check_get(node) + self.generic_visit(node) + + def visit_Subscript(self, node: ast.Subscript) -> None: + self._check_subscript(node) + self.generic_visit(node) + + def _check_get(self, node: ast.Call) -> None: + """``<...>.environment_info.get("key")``.""" + func = node.func + if not isinstance(func, ast.Attribute) or func.attr != "get": + return + if not _is_env_info(func.value) or not node.args: + return + self._require_writer(node, node.args[0]) + + def _check_subscript(self, node: ast.Subscript) -> None: + """``<...>.environment_info["key"]`` in a READ position. + + A write is an ``ast.Store`` context, which is exactly what makes it a + writer — only loads are checked. + """ + if not isinstance(node.ctx, ast.Load) or not _is_env_info(node.value): + return + self._require_writer(node, node.slice) + + def _require_writer(self, node: ast.AST, key_node: ast.AST) -> None: + if not self._in_scope: + return + if not isinstance(key_node, ast.Constant) or not isinstance(key_node.value, str): + return # a computed key; nothing to resolve statically + key = key_node.value + if key in _EXTERNALLY_WRITTEN or key.startswith("graded_by_"): + # graded_by_* keys are built with an f-string from a name list, so no + # literal write exists to find. + return + if key in (EnvInfoKeyRoundTrip._written or set()): + return + self.violation( + node, + f"environment_info key {key!r} is read here but never written anywhere in src/coder_eval. " + + "A reader with no writer is silently inert — it returns None, the guard takes its early " + + "return, and the feature reports success while doing nothing (see CE047's docstring for " + + "the anti-cheat guard that shipped this way). Add the write, or list the key in " + + "_EXTERNALLY_WRITTEN with the out-of-tree producer that supplies it.", + ) + + +def _is_env_info(node: ast.AST) -> bool: + """Whether ``node`` is an ``…​.environment_info`` attribute access.""" + return isinstance(node, ast.Attribute) and node.attr == "environment_info" diff --git a/tests/lint/rules/ce048_no_in_process_typer_command_call.py b/tests/lint/rules/ce048_no_in_process_typer_command_call.py new file mode 100644 index 00000000..37c10a2f --- /dev/null +++ b/tests/lint/rules/ce048_no_in_process_typer_command_call.py @@ -0,0 +1,106 @@ +"""CE048: never call a Typer command function in process. + +Typer builds a command's parser from its signature, so every parameter's default +is an ``OptionInfo`` / ``ArgumentInfo`` sentinel, not the value it stands for. +Click substitutes the real defaults when it *invokes* the command; a direct +Python call does not — every unspecified argument arrives as a truthy sentinel +object. + +The failure is silent, which is what makes it worth a rule. ``evaluate``'s +``in_place: bool | None = typer.Option(None, "--in-place/--copy")`` reads as "no +preference" and selects copy-vs-in-place from the target shape; called +in-process, ``in_place`` was an ``OptionInfo``, which is truthy, so the tests +silently graded in place and the default they meant to cover was never +exercised. Nothing failed — the wrong branch simply ran. + +The fix is the one already applied to ``run`` / ``execute`` / ``evaluate``: keep +the Typer signature as a thin wrapper and put the body in a plain function with +real Python defaults (``run_pipeline``, ``run_evaluation``). Call THAT. + +Use ``# noqa: CE048`` only where the sentinel behavior is itself under test. +""" + +import ast +import re +from pathlib import Path + +from tests.lint.rules.base import BaseRule + + +_CLI_ROOT = Path("src/coder_eval/cli") + + +def _typer_command_names() -> set[str]: + """Functions whose signature is a Typer parser — i.e. whose parameters carry + ``typer.Option`` / ``typer.Argument`` defaults. + + Detected by the defaults rather than by the ``app.command(...)`` registration + site, because registration happens in ``cli/__init__.py`` by reference and a + command that is merely *about* to be registered has the same hazard. + """ + names: set[str] = set() + for path in sorted(_CLI_ROOT.rglob("*.py")): + try: + tree = ast.parse(path.read_text(encoding="utf-8")) + except (OSError, SyntaxError): # pragma: no cover - unparseable file in cli/ fails elsewhere + continue + for node in ast.walk(tree): + if not isinstance(node, ast.FunctionDef | ast.AsyncFunctionDef): + continue + if any(_is_typer_param(default) for default in node.args.defaults): + names.add(node.name) + return names + + +def _is_typer_param(node: ast.expr) -> bool: + """``typer.Option(...)`` / ``typer.Argument(...)`` as a parameter default.""" + if not isinstance(node, ast.Call): + return False + func = node.func + return isinstance(func, ast.Attribute) and func.attr in {"Option", "Argument"} + + +class NoInProcessTyperCommandCall(BaseRule): + id = "CE048" + + # The registration site itself hands these to Typer by reference; and the + # module that defines a command may call its own sibling. + _EXEMPT_FILES = re.compile(r"[/\\]src[/\\]coder_eval[/\\]cli[/\\]__init__\.py$") + _commands: set[str] | None = None + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = not self._EXEMPT_FILES.search(filepath) + self._imported_from_cli: set[str] = set() + if NoInProcessTyperCommandCall._commands is None: + NoInProcessTyperCommandCall._commands = _typer_command_names() + + def visit_ImportFrom(self, node: ast.ImportFrom) -> None: + # Only names imported FROM a cli module count. A command name is not + # unique in the tree — `run_command` is both a Typer command and + # `Sandbox.run_command` — so matching on the bare name alone would flag + # every criterion that shells out. + if (node.module and "coder_eval.cli" in node.module) or (node.level and node.module == "cli"): + self._imported_from_cli.update(alias.asname or alias.name for alias in node.names) + self.generic_visit(node) + + def visit_Call(self, node: ast.Call) -> None: + self._check(node) + self.generic_visit(node) + + def _check(self, node: ast.Call) -> None: + if not self._in_scope or not isinstance(node.func, ast.Name): + return + name = node.func.id + if name not in self._imported_from_cli: + return + if name not in (NoInProcessTyperCommandCall._commands or set()): + return + self.violation( + node, + f"'{name}' is a Typer command: its parameter defaults are OptionInfo sentinels, not values, " + + "so calling it in process hands every unspecified argument a truthy placeholder and " + + "silently runs the wrong branch. Call the plain-function body instead (the " + + "run_pipeline / run_evaluation split exists for this), or drive it through " + + "typer.testing.CliRunner.", + ) diff --git a/tests/lint/rules/ce049_no_score_or_zero.py b/tests/lint/rules/ce049_no_score_or_zero.py new file mode 100644 index 00000000..49153617 --- /dev/null +++ b/tests/lint/rules/ce049_no_score_or_zero.py @@ -0,0 +1,70 @@ +"""CE049: never coalesce a possibly-unmeasured score to a numeric literal. + +``weighted_score is None`` means *nothing measured this row*, and it is a +different fact from ``weighted_score == 0.0``, which means *this row was measured +and scored nothing*. ``score or 0.0`` erases that difference — and it does it +silently, producing a real-looking number that every downstream consumer treats +as a genuine miss. + +The motivating bug: ``build_task_event`` published ``Score = float( +result.weighted_score or 0.0)`` on every ``CoderEval.Task.End``. Four shipped +App Insights tiles compute ``avg(todouble(customDimensions.Score))`` with no +status filter, so one ``coder-eval execute`` night dragged every score tile +toward zero, indistinguishable from a genuinely bad night. The hazard was +already documented in prose in ``orchestrator.py`` ("every downstream +`score or 0.0` would launder it into a real-looking failure") — this rule makes +it mechanical. + +Fires on `` or `` where the left operand's trailing name +looks like a score or a rate. The fix is to omit the value, keep it ``None``, or +branch explicitly on ``is None``. + +``# noqa: CE049`` for a genuinely aggregate-internal use where a missing value +really is a miss — e.g. summing a variant's scores where an errored row must +count as 0.0 (see ``orchestration/experiment._measured_scores``, which makes that +decision explicitly and states why). +""" + +import ast +import re + +from tests.lint.rules.base import BaseRule + + +# Trailing-segment match, so `result.weighted_score`, `v.average_score` and +# `summary.pass_rate` all fire while `sample_rate_limit` does not. +_SCORE_NAME = re.compile(r"^(weighted_score|score|average_score|pass_rate|[a-z_]*_rate)$") + + +def _scoreish(node: ast.expr) -> str | None: + """The name of a score/rate-looking operand, or None.""" + if isinstance(node, ast.Attribute): + name = node.attr + elif isinstance(node, ast.Name): + name = node.id + else: + return None + return name if _SCORE_NAME.match(name) else None + + +def _numeric_literal(node: ast.expr) -> bool: + return isinstance(node, ast.Constant) and isinstance(node.value, int | float) and not isinstance(node.value, bool) + + +class NoScoreOrZero(BaseRule): + id = "CE049" + + def visit_BoolOp(self, node: ast.BoolOp) -> None: + if isinstance(node.op, ast.Or) and len(node.values) == 2: + name = _scoreish(node.values[0]) + if name is not None and _numeric_literal(node.values[1]): + self.violation( + node, + f"{name!r} is coalesced to a numeric literal. `None` means the row was never " + + "measured; a literal means it was measured and scored that value — and the " + + "coalesce publishes the second while meaning the first, which is how an " + + "ungraded run reached four avg(Score) dashboards as a real zero. Omit the " + + "value, keep None, or branch on `is None`. Use `# noqa: CE049` where a " + + "missing value genuinely IS a miss, with a comment saying so.", + ) + self.generic_visit(node) diff --git a/tests/lint/rules/ce050_no_union_getattr_probe.py b/tests/lint/rules/ce050_no_union_getattr_probe.py new file mode 100644 index 00000000..166d6be2 --- /dev/null +++ b/tests/lint/rules/ce050_no_union_getattr_probe.py @@ -0,0 +1,132 @@ +"""CE050: no untyped ``getattr`` probe for a field of a discriminated union. + +``getattr(criterion, "command", None)`` reads as "the members that have a +command". It is not: it is a string the type checker cannot see. Rename +``RunCommandCriterion.command`` and pyright reports nothing, ruff reports +nothing, and the probe silently returns ``None`` forever — the guard it powers +becomes a permanent no-op with every gate green. + +``models/tasks.py`` already states the rule in prose, verbatim: "isinstance +narrowing, NOT getattr(c, 'files'/'command'): with an untyped string probe, +renaming ... turns this load-time guard into a silent no-op that pyright cannot +see." This promotes that convention to a gate. + +The motivating bug: ``orchestration/regrade.warn_on_embedded_commands`` — the +only disclosure of what shell a rebuilt, untrusted run config would execute on +the grader's host — probed with ``getattr(c, "command", None)``. Besides being +rename-fragile it structurally could not name ``agent_judge``, the criterion +that spawns a tool-using agent and therefore has the widest blast radius of all. + +Fires on ``getattr(, "", ...)`` in ``src/coder_eval/`` where the +literal is a field name declared by a member of one of the tracked discriminated +unions AND ```` is named like a criterion / template source / route. The +field list is derived from the models at collection time, so it tracks renames +instead of going stale. + +The receiver-name filter is deliberate, and it is the rule's known limit. Field +names like ``command``, ``tool`` and ``prompt`` are far too common to flag on +their own — the agents legitimately probe raw SDK event objects for exactly those +— so a name-only rule would fire a dozen times on code that has nothing to do +with these unions and would be turned off within a week. Scoping to the +receiver's name catches the real shape (``for c in task.success_criteria: ... +getattr(c, "command", None)``) and leaves an unusual receiver name uncovered. + +The fix is ``isinstance`` narrowing. ``# noqa: CE050`` for a probe that really is +duck-typed across unrelated objects. +""" + +import ast +import re + +from tests.lint.rules.base import BaseRule + + +def _union_field_names() -> set[str]: + """Field names declared by any member of the tracked unions. + + Derived from the models rather than hardcoded: a hardcoded list is the same + class of staleness the rule exists to prevent. Names shared with ordinary + object attributes (``type``, ``description``, ``weight`` …) are excluded — + they are not what a rename would break, and flagging them would only teach + people to noqa the rule. + """ + import typing + + from coder_eval.models import ApiRoute, SuccessCriterion, TemplateSource + + common = {"type", "description", "weight", "pass_threshold", "model", "path"} + + def _members(annotation: object) -> list[object]: + """Flatten ``Annotated[Union[...], Field(discriminator=...)]`` to its members. + + The unions are all discriminated, so a bare ``__args__`` yields + ``(Union[...], FieldInfo)`` and the model classes are one level deeper — + which silently produced an EMPTY field set, i.e. a rule that could never + fire. Recurse instead. + """ + args = typing.get_args(annotation) + if not args: + return [annotation] + out: list[object] = [] + for arg in args: + out.extend(_members(arg) if typing.get_args(arg) else [arg]) + return out + + names: set[str] = set() + for union in (SuccessCriterion, TemplateSource, ApiRoute): + for member in _members(union): + names |= set(getattr(member, "model_fields", {})) + assert names, "CE050 derived no union field names — the rule could never fire" + return names - common + + +# Receivers that name a member of one of the tracked unions. See the module +# docstring for why the rule is scoped this way rather than on the field alone. +_RECEIVER = re.compile(r"^(c|cr|crit|criterion|source|template_source|route|api_route)$|(_criterion|_source|_route)$") + + +def _union_receiver(node: ast.expr) -> bool: + """Whether the probed object is named like a union member.""" + if isinstance(node, ast.Name): + return bool(_RECEIVER.search(node.id)) + if isinstance(node, ast.Attribute): + return bool(_RECEIVER.search(node.attr)) + return False + + +class NoUnionGetattrProbe(BaseRule): + id = "CE050" + + # `(^|sep)` so a repo-relative path is in scope too; see CE047. + _SRC_PATH = re.compile(r"(?:^|[/\\])src[/\\]coder_eval[/\\]") + _fields: set[str] | None = None + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = bool(self._SRC_PATH.search(filepath)) + if self._in_scope and NoUnionGetattrProbe._fields is None: + NoUnionGetattrProbe._fields = _union_field_names() + + def visit_Call(self, node: ast.Call) -> None: + self._check(node) + self.generic_visit(node) + + def _check(self, node: ast.Call) -> None: + if not self._in_scope: + return + if not isinstance(node.func, ast.Name) or node.func.id != "getattr" or len(node.args) < 2: + return + key = node.args[1] + if not isinstance(key, ast.Constant) or not isinstance(key.value, str): + return + if key.value not in (NoUnionGetattrProbe._fields or set()): + return + if not _union_receiver(node.args[0]): + return + self.violation( + node, + f"getattr(..., {key.value!r}) probes a discriminated-union field with an untyped " + + "string. pyright cannot see it, so a rename turns this into a silent no-op that " + + "returns None forever — and it cannot reach members that express the same " + + "capability under a different field. Narrow with isinstance instead.", + ) diff --git a/tests/lint/rules/ce051_no_driver_override.py b/tests/lint/rules/ce051_no_driver_override.py new file mode 100644 index 00000000..f6943139 --- /dev/null +++ b/tests/lint/rules/ce051_no_driver_override.py @@ -0,0 +1,94 @@ +"""CE051: a sandbox driver may not be rewritten silently. + +The driver IS the isolation boundary. Rewriting ``docker`` to ``tempdir`` behind +the caller's back does not degrade gracefully — it moves execution from a +container onto the operator's own machine, where the task's criteria address +paths and toolchains that do not exist. They score 0.0 and the row is written +back FAILURE for a trajectory that passed, and the same commands (``rm -rf +/verifier``, ``mkdir -p /logs/verifier``) run unsandboxed on the grading host. + +The motivating bug: ``regrade.grading_sandbox_config`` rewrote the driver +unconditionally on BOTH new grading entry points, which also neutralized the +``driver: docker`` refusal in ``Sandbox.adopt`` — a guard added in the same +change specifically to catch this. + +A driver downgrade must be an explicit, logged, operator-visible decision. Fires +on any construction that carries an existing sandbox config forward while +replacing ``driver``: + + * ``SandboxConfig.model_validate({**cfg.model_dump(), "driver": ...})`` + * ``cfg.model_copy(update={"driver": ...})`` + * ``setattr(cfg, "driver", ...)`` / ``cfg.driver = ...`` + +Exempt: ``models/sandbox.py`` (the model's own construction), and any site +carrying ``# noqa: CE051`` with a reason — today the two legitimate ones are the +in-container rewrite in ``run_task_internal_command`` and the opt-in host-grading +branch, which refuses by default and stamps ``graded_on_host`` on the row. +""" + +import ast +import re + +from tests.lint.rules.base import BaseRule + + +_MESSAGE = ( + "This rewrites `driver` on an existing sandbox config. The driver is the isolation " + "boundary: silently moving a docker task onto the host makes its criteria address paths " + "and a toolchain that are not there, so they score 0.0 and the row is written back FAILURE " + "for a run that passed — and its shell runs unsandboxed on this machine. Refuse, or make it " + "an explicit opt-in that stamps the row, and add `# noqa: CE051` naming the reason." +) + + +def _has_driver_key(node: ast.expr) -> bool: + """True when ``node`` is a dict/dict-display whose keys include ``"driver"``.""" + if not isinstance(node, ast.Dict): + return False + return any(isinstance(k, ast.Constant) and k.value == "driver" for k in node.keys if k is not None) + + +class NoDriverOverride(BaseRule): + id = "CE051" + + # The model's own module legitimately constructs and defaults the field. + _EXEMPT_PATH = re.compile(r"[/\\]models[/\\]sandbox\.py$") + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = bool( + re.search(r"(?:^|[/\\])src[/\\]coder_eval[/\\]", filepath) + ) and not self._EXEMPT_PATH.search(filepath) + + def visit_Call(self, node: ast.Call) -> None: + if self._in_scope: + self._check_call(node) + self.generic_visit(node) + + def visit_Assign(self, node: ast.Assign) -> None: + if self._in_scope: + for target in node.targets: + if isinstance(target, ast.Attribute) and target.attr == "driver": + self.violation(node, _MESSAGE) + self.generic_visit(node) + + def _check_call(self, node: ast.Call) -> None: + # cfg.model_copy(update={"driver": ...}) + if isinstance(node.func, ast.Attribute) and node.func.attr == "model_copy": + for kw in node.keywords: + if kw.arg == "update" and _has_driver_key(kw.value): + self.violation(node, _MESSAGE) + return + # SandboxConfig.model_validate({**cfg.model_dump(), "driver": ...}) + if isinstance(node.func, ast.Attribute) and node.func.attr == "model_validate" and node.args: + arg = node.args[0] + # Only a SPREAD dict — a literal built from scratch is an ordinary + # construction, not a rewrite of somebody else's config. + if _has_driver_key(arg) and isinstance(arg, ast.Dict) and any(k is None for k in arg.keys): + self.violation(node, _MESSAGE) + return + # setattr(cfg, "driver", ...) + if isinstance(node.func, ast.Name) and node.func.id == "setattr" and len(node.args) >= 2: + key = node.args[1] + if isinstance(key, ast.Constant) and key.value == "driver": + self.violation(node, _MESSAGE) diff --git a/tests/lint/rules/ce052_process_lethal_must_be_container_gated.py b/tests/lint/rules/ce052_process_lethal_must_be_container_gated.py new file mode 100644 index 00000000..a05e670c --- /dev/null +++ b/tests/lint/rules/ce052_process_lethal_must_be_container_gated.py @@ -0,0 +1,91 @@ +"""CE052: a process-lethal call must be gated on actually being in the container. + +``os._exit`` bypasses ``atexit``, buffered IO, ``finally`` blocks and every +exception handler: the process is simply gone. That is the correct primitive for +exactly one thing in this codebase — reaping the container's own disposable main +process when the host that started it has died — and it is safe there only +because that process is *ours to destroy*. In any other process it is not a +degraded outcome, it is an unattributable one. + +The motivating bug: ``run_task_internal_command`` armed its host-heartbeat +watchdog — a daemon thread whose whole authority is ``os._exit(137)`` — as an +unconditional side effect of the command body. A test invoked that command +in-process (legitimately: the command must refuse a malformed ``context.json``, +and asserting that means calling it), and the pytest worker inherited the +thread. Forty seconds later — 20s grace plus the 20s stale window — it found no +heartbeat and exited the worker, mid-way through whatever unrelated test file +that worker had since moved on to. + +Every property of that failure is the one this rule exists to prevent: + + * it named the wrong test — a different one on each run, on each platform, + with no traceback, because there is no exception to raise; + * it was invisible at low load — with 14 local workers the file finished and + the run ended before the timer fired, so it reproduced only on CI's 2; + * and it took the coverage gate with it. A dead worker returns no coverage + data, so a single killed process reported as "total of 65.13 is less than + fail-under=80.00" — a failure naming neither the test nor the cause. + +Fires on ``os._exit(...)`` anywhere in ``src/coder_eval/`` that is not lexically +inside a branch testing ``CODER_EVAL_IN_CONTAINER``. That env var is the repo's +established in-container predicate (``Sandbox.enforces_permission_windows``, +``orchestration/evaluation.resolve_reference_dir``) and is deliberately NOT +``sandbox.driver`` — ``run_task_internal_command`` rewrites the driver to +``tempdir`` before building the in-container Orchestrator, so a driver-based gate +disables itself on precisely the path that needs it. + +The check is lexical (an enclosing ``if``/``elif`` whose test mentions the var), +not a data-flow proof. That is enough to force the guard to be written down at +the site, which is the property that was missing. +""" + +import ast +import re + +from tests.lint.rules.base import BaseRule + + +_GATE = "CODER_EVAL_IN_CONTAINER" + +_MESSAGE = ( + "`os._exit` here is not gated on CODER_EVAL_IN_CONTAINER. It kills the process outright — " + "no atexit, no finally, no traceback — which is the right primitive ONLY for the container's " + "own main process. Anywhere else it destroys a host process that merely called this code: an " + "unconditionally-armed watchdog once exited a pytest worker 40s after the test that armed it, " + "reporting as a random crash in an unrelated file and as a bogus coverage failure. Gate it on " + '`os.environ.get("CODER_EVAL_IN_CONTAINER") == "1"`, or add `# noqa: CE052` with a reason.' +) + + +def _is_os_exit(node: ast.Call) -> bool: + """``os._exit(...)`` under any module alias (it is imported as ``_os`` here).""" + return isinstance(node.func, ast.Attribute) and node.func.attr == "_exit" + + +class ProcessLethalMustBeContainerGated(BaseRule): + id = "CE052" + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + # `(^|sep)` so a repo-relative path is in scope too; see CE047. + self._in_scope = bool(re.search(r"(?:^|[/\\])src[/\\]coder_eval[/\\]", filepath)) + # Tests of enclosing `if`/`elif` statements, innermost last. + self._guards: list[ast.expr] = [] + + def visit_If(self, node: ast.If) -> None: + # Only the body is guarded — the `else` arm is the ungated branch, which + # is exactly where an inverted guard would put the lethal call. + self._guards.append(node.test) + for child in node.body: + self.visit(child) + self._guards.pop() + for child in node.orelse: + self.visit(child) + + def visit_Call(self, node: ast.Call) -> None: + if self._in_scope and _is_os_exit(node) and not self._container_gated(): + self.violation(node, _MESSAGE) + self.generic_visit(node) + + def _container_gated(self) -> bool: + return any(_GATE in ast.dump(test) for test in self._guards) diff --git a/tests/lint/rules/ce053_run_record_filename_literal.py b/tests/lint/rules/ce053_run_record_filename_literal.py new file mode 100644 index 00000000..41d30aa5 --- /dev/null +++ b/tests/lint/rules/ce053_run_record_filename_literal.py @@ -0,0 +1,70 @@ +"""CE053: no bare run-record filename literal outside ``path_utils``. + +``path_utils`` defines ``TASK_JSON_FILENAME`` / ``PRE_GRADE_JSON_FILENAME`` and +its comment states why: "~12 sites name them — including three that ``rglob`` for +the first — and two half-copies of the same string in different packages is how a +rename becomes a silent no-op on the sites it missed." + +The constant shipped with that rationale and the twelve pre-existing literals +were not converted, so it created exactly the second source of truth it argues +against and delivered zero rename safety: the new modules used the constant, and +``orchestrator.py``, ``batch.py``, ``docker_runner.py``, ``reports.py``, +``reports_junit.py``, ``reports_stats.py`` and ``report_command.py`` kept the +string — the three ``rglob("task.json")`` calls the comment specifically cites +among them. + +A rationale that only a human remembers is not a rule. Fires on any string +constant in ``src/coder_eval/`` (outside ``path_utils.py``) that equals one of +those filenames, or embeds it as a trailing path segment (``"*/task.json"``). +Import the constant instead; ``# noqa: CE053`` for a genuinely unrelated string. +""" + +import ast +import re + +from tests.lint.rules.base import BaseRule + + +def _run_record_filenames() -> set[str]: + """The filenames from ``path_utils``, read from the module rather than retyped.""" + from coder_eval.path_utils import PRE_GRADE_JSON_FILENAME, TASK_JSON_FILENAME + + return {TASK_JSON_FILENAME, PRE_GRADE_JSON_FILENAME} + + +class NoRunRecordFilenameLiteral(BaseRule): + id = "CE053" + + # `(^|sep)` so a repo-relative path is in scope too; see CE047. + _SRC_PATH = re.compile(r"(?:^|[/\\])src[/\\]coder_eval[/\\]") + # The module that DEFINES them, and the container-path module that mirrors + # the in-container layout as its own vocabulary. + _EXEMPT = re.compile(r"[/\\]path_utils\.py$") + _names: set[str] | None = None + + def __init__(self, filepath: str) -> None: + super().__init__(filepath) + self._in_scope = bool(self._SRC_PATH.search(filepath)) and not self._EXEMPT.search(filepath) + if self._in_scope and NoRunRecordFilenameLiteral._names is None: + NoRunRecordFilenameLiteral._names = _run_record_filenames() + + def visit_Constant(self, node: ast.Constant) -> None: + if self._in_scope and isinstance(node.value, str): + self._check(node, node.value) + self.generic_visit(node) + + def _check(self, node: ast.Constant, value: str) -> None: + for name in NoRunRecordFilenameLiteral._names or set(): + # Exact, or a glob/path whose LAST segment is the filename. Not a + # bare `in`: that would fire on prose in a docstring or an error + # message, where naming the file is the point. + if value == name or (("/" in value) and value.rsplit("/", 1)[-1] == name): + self.violation( + node, + f"{value!r} names a run record by literal. path_utils exports a constant for it " + "precisely so a rename cannot leave half the tree behind — and the constant " + "shipped while twelve sites kept the string, which is the second source of " + "truth it was added to prevent. Import TASK_JSON_FILENAME / " + "PRE_GRADE_JSON_FILENAME, or add `# noqa: CE053` if the string is unrelated.", + ) + return diff --git a/tests/lint/runner.py b/tests/lint/runner.py index 092e97a6..bcd51561 100644 --- a/tests/lint/runner.py +++ b/tests/lint/runner.py @@ -26,6 +26,13 @@ from tests.lint.rules.ce039_config_error_escalates import ConfigErrorEscalates from tests.lint.rules.ce043_no_command_output_truncation import NoCommandOutputTruncation from tests.lint.rules.ce046_env_info_spreads_super import EnvInfoSpreadsSuper +from tests.lint.rules.ce047_env_info_key_round_trip import EnvInfoKeyRoundTrip +from tests.lint.rules.ce048_no_in_process_typer_command_call import NoInProcessTyperCommandCall +from tests.lint.rules.ce049_no_score_or_zero import NoScoreOrZero +from tests.lint.rules.ce050_no_union_getattr_probe import NoUnionGetattrProbe +from tests.lint.rules.ce051_no_driver_override import NoDriverOverride +from tests.lint.rules.ce052_process_lethal_must_be_container_gated import ProcessLethalMustBeContainerGated +from tests.lint.rules.ce053_run_record_filename_literal import NoRunRecordFilenameLiteral from tests.lint.rules.no_agent_timing_access import NoAgentTimingAccess from tests.lint.rules.no_blocking_io_in_async import NoBlockingIoInAsync from tests.lint.rules.no_cli_imports_in_core import NoCliImportsInCore @@ -75,6 +82,13 @@ ConfigErrorEscalates, NoCommandOutputTruncation, EnvInfoSpreadsSuper, + EnvInfoKeyRoundTrip, + NoInProcessTyperCommandCall, + NoScoreOrZero, + NoUnionGetattrProbe, + NoDriverOverride, + ProcessLethalMustBeContainerGated, + NoRunRecordFilenameLiteral, ] # Anti-shadow invariant (mirrors AgentRegistry / register_pricing): every CE rule diff --git a/tests/test_cleanup_preservation_guard.py b/tests/test_cleanup_preservation_guard.py index e02c2f8e..b2519cd8 100644 --- a/tests/test_cleanup_preservation_guard.py +++ b/tests/test_cleanup_preservation_guard.py @@ -201,6 +201,10 @@ async def test_none_without_workspace_dir_discards_path(tmp_path) -> None: orchestrator.workspace_dir = None orchestrator.result.sandbox_path = "/stale/path" # must be cleared mock_sandbox = MagicMock() + # A real Sandbox that was not adopted. Explicit because a bare MagicMock + # attribute is truthy, which would take the adopted arm (that one KEEPS the + # path, since an adopted directory survives cleanup) and hide the discard. + mock_sandbox.was_adopted = False orchestrator.sandbox = mock_sandbox await orchestrator._cleanup() diff --git a/tests/test_cli_telemetry.py b/tests/test_cli_telemetry.py index 1c757524..06f49cad 100644 --- a/tests/test_cli_telemetry.py +++ b/tests/test_cli_telemetry.py @@ -79,7 +79,7 @@ def test_help_never_crashes_when_telemetry_enabled_and_config_unwritable(tmp_pat async def test_run_emits_run_start_and_flushes(tmp_path): - summary = Mock(tasks_failed=0, tasks_error=0) + summary = Mock(tasks_failed=0, tasks_error=0, tasks_not_graded=0) with ( patch("coder_eval.cli.run_command.prepare_run_directory", return_value=tmp_path), @@ -115,7 +115,7 @@ async def test_run_emits_run_start_and_flushes(tmp_path): async def test_run_start_uses_default_fallbacks_for_none_inputs(tmp_path): # agent_type=None / stream_mode=None must surface as the "default"/"none" # fallback property values, not as null. - summary = Mock(tasks_failed=0, tasks_error=0) + summary = Mock(tasks_failed=0, tasks_error=0, tasks_not_graded=0) with ( patch("coder_eval.cli.run_command.prepare_run_directory", return_value=tmp_path), diff --git a/tests/test_custom_lint.py b/tests/test_custom_lint.py index 64508fe7..4ebb8bc4 100644 --- a/tests/test_custom_lint.py +++ b/tests/test_custom_lint.py @@ -23,13 +23,21 @@ SRC = Path(__file__).parent.parent / "src" +# Rules whose defect class lives in the TEST tree, not in src/. CE048's whole +# subject is an in-process call to a Typer command, and the only place that +# happens is a test — scanning src/ alone would leave the rule permanently green +# while the bug it exists for sat five lines away. +_ALSO_SCAN_TESTS = {"CE048"} + + @pytest.mark.lint @pytest.mark.parametrize("rule_class", ALL_RULES, ids=[r.id for r in ALL_RULES]) def test_no_violations(rule_class: type) -> None: import sys mod_doc = (getattr(sys.modules.get(rule_class.__module__), "__doc__", "") or "").splitlines()[0].strip() - violations = check_paths([SRC], rules=[rule_class]) + paths = [SRC, Path(__file__).parent] if rule_class.id in _ALSO_SCAN_TESTS else [SRC] + violations = check_paths(paths, rules=[rule_class]) assert not violations, ( f"\n{len(violations)} violation(s) for {rule_class.id} ({mod_doc}):\n\n" + "\n".join(f" {v}" for v in violations) @@ -3600,3 +3608,294 @@ def test_yaml_walk_flags_a_literal_tail_behind_a_variable(self, tmp_path: Path): encoding="utf-8", ) assert not self._offending_paths_in(task) + + +class TestCE047EnvInfoKeyRoundTrip: + """CE047 fires when an environment_info key is read with no writer anywhere. + + The rule shipped with only the whole-tree "finds nothing" scan, which cannot + tell a rule that is CORRECT from one that can never fire — the exact failure + its own docstring is about. The path form matters too: the rule scopes itself + with a leading-separator regex, so a repo-relative path must still be in + scope or a house-style test would pass vacuously. + """ + + @staticmethod + def _run(src: str, filepath: str = "src/coder_eval/orchestrator.py"): + import ast + + from tests.lint.rules.ce047_env_info_key_round_trip import EnvInfoKeyRoundTrip + + return EnvInfoKeyRoundTrip(filepath).check(ast.parse(src)) + + def test_flags_a_get_with_no_writer(self): + assert self._run('x = self.result.environment_info.get("no_such_key_anywhere")') + + def test_flags_a_subscript_read_with_no_writer(self): + assert self._run('x = self.result.environment_info["no_such_key_anywhere"]') + + def test_allows_a_key_that_is_written_in_src(self): + # reference_digest gained a writer; that is the whole point of the rule. + assert not self._run('x = self.result.environment_info.get("reference_digest")') + + def test_allows_the_graded_by_prefix(self): + assert not self._run('x = self.result.environment_info.get("graded_by_git_commit")') + + def test_ignores_a_computed_key(self): + assert not self._run("x = self.result.environment_info.get(key)") + + def test_is_out_of_scope_outside_src(self): + assert not self._run('x = r.environment_info.get("no_such_key_anywhere")', filepath="tests/test_thing.py") + + def test_scope_is_the_same_for_relative_and_absolute_paths(self): + """A repo-relative path must be in scope, or every house-style test lies.""" + src = 'x = self.result.environment_info.get("no_such_key_anywhere")' + relative = self._run(src, filepath="src/coder_eval/orchestrator.py") + absolute = self._run(src, filepath="/home/u/repo/src/coder_eval/orchestrator.py") + assert bool(relative) == bool(absolute) is True + + +class TestCE048NoInProcessTyperCommandCall: + """CE048 fires on an in-process call to a Typer command function.""" + + @staticmethod + def _run(src: str, filepath: str = "tests/test_thing.py"): + import ast + + from tests.lint.rules.ce048_no_in_process_typer_command_call import NoInProcessTyperCommandCall + + return NoInProcessTyperCommandCall(filepath).check(ast.parse(src)) + + def test_flags_calling_a_command_function_directly(self): + assert self._run("from coder_eval.cli.evaluate_command import evaluate_command\nevaluate_command(x)") + + def test_allows_the_plain_python_entry_point(self): + assert not self._run("from coder_eval.cli.evaluate_command import run_evaluation\nrun_evaluation(x=1)") + + +class TestCE049NoScoreOrZero: + """CE049 flags coalescing an unmeasured score into a real-looking number.""" + + @staticmethod + def _run(src: str): + import ast + + from tests.lint.rules.ce049_no_score_or_zero import NoScoreOrZero + + return NoScoreOrZero("src/coder_eval/orchestrator.py").check(ast.parse(src)) + + def test_flags_weighted_score_or_zero(self): + assert self._run("x = float(result.weighted_score or 0.0)") + + def test_flags_a_bare_score_name_and_an_int_literal(self): + assert self._run("x = score or 0") + + def test_flags_a_rate(self): + assert self._run("x = summary.pass_rate or 0.0") + + def test_allows_an_explicit_none_branch(self): + assert not self._run("x = 0.0 if result.weighted_score is None else result.weighted_score") + + def test_allows_a_non_numeric_fallback(self): + assert not self._run('x = result.weighted_score or "n/a"') + + def test_ignores_an_unrelated_name(self): + assert not self._run("x = retry_count or 0") + + +class TestCE050NoUnionGetattrProbe: + """CE050 flags an untyped getattr probe for a discriminated-union field.""" + + @staticmethod + def _run(src: str, filepath: str = "src/coder_eval/orchestration/regrade.py"): + import ast + + from tests.lint.rules.ce050_no_union_getattr_probe import NoUnionGetattrProbe + + return NoUnionGetattrProbe(filepath).check(ast.parse(src)) + + def test_flags_the_command_probe(self): + assert self._run('cmd = getattr(c, "command", None)') + + def test_allows_isinstance_narrowing(self): + assert not self._run("cmd = c.command if isinstance(c, RunCommandCriterion) else None") + + def test_ignores_a_name_no_union_member_declares(self): + assert not self._run('x = getattr(obj, "definitely_not_a_criterion_field", None)') + + def test_ignores_a_computed_key(self): + assert not self._run("x = getattr(obj, name, None)") + + def test_is_out_of_scope_outside_src(self): + assert not self._run('cmd = getattr(c, "command", None)', filepath="tests/test_thing.py") + + +class TestCE051NoDriverOverride: + """CE051 flags a silent sandbox-driver rewrite.""" + + @staticmethod + def _run(src: str, filepath: str = "src/coder_eval/orchestration/regrade.py"): + import ast + + from tests.lint.rules.ce051_no_driver_override import NoDriverOverride + + return NoDriverOverride(filepath).check(ast.parse(src)) + + def test_flags_a_spread_model_validate(self): + assert self._run('SandboxConfig.model_validate({**task.sandbox.model_dump(), "driver": "tempdir"})') + + def test_flags_model_copy_update(self): + assert self._run('cfg.model_copy(update={"driver": "tempdir"})') + + def test_flags_setattr(self): + assert self._run('setattr(cfg, "driver", "tempdir")') + + def test_flags_attribute_assignment(self): + assert self._run('cfg.driver = "tempdir"') + + def test_allows_a_config_built_from_scratch(self): + assert not self._run('SandboxConfig.model_validate({"driver": "tempdir"})') + + def test_allows_carrying_a_config_forward_unchanged(self): + assert not self._run("task.sandbox.model_copy(deep=True)") + + def test_is_out_of_scope_in_the_model_module(self): + assert not self._run( + 'cfg.model_copy(update={"driver": "tempdir"})', + filepath="src/coder_eval/models/sandbox.py", + ) + + +class TestCE052ProcessLethalMustBeContainerGated: + """CE052 flags an `os._exit` that is not gated on being in the container.""" + + @staticmethod + def _run(src: str, filepath: str = "src/coder_eval/cli/run_task_internal_command.py"): + import ast + + from tests.lint.rules.ce052_process_lethal_must_be_container_gated import ( + ProcessLethalMustBeContainerGated, + ) + + return ProcessLethalMustBeContainerGated(filepath).check(ast.parse(src)) + + def test_flags_an_ungated_exit(self): + assert self._run("os._exit(137)") + + def test_flags_it_under_an_unrelated_guard(self): + assert self._run("if stale:\n os._exit(137)") + + def test_flags_it_in_the_else_arm_of_the_container_guard(self): + """An inverted guard is the shape a well-meaning refactor produces.""" + src = 'if os.environ.get("CODER_EVAL_IN_CONTAINER") == "1":\n pass\nelse:\n os._exit(137)' + assert self._run(src) + + def test_allows_a_gated_exit(self): + src = 'if os.environ.get("CODER_EVAL_IN_CONTAINER") == "1":\n os._exit(137)' + assert not self._run(src) + + def test_allows_it_nested_deeper_inside_the_gate(self): + """The real site defines a function and a loop inside the guard.""" + src = ( + 'if _os.environ.get("CODER_EVAL_IN_CONTAINER") == "1":\n' + "\n" + " def _watch() -> None:\n" + " while True:\n" + " if stale:\n" + " _os._exit(137)\n" + ) + assert not self._run(src) + + def test_is_out_of_scope_outside_the_package(self): + assert not self._run("os._exit(137)", filepath="scripts/reap.py") + + def test_the_real_module_is_clean(self): + """The rule must actually pass on the site it was written for — a rule + that only ever fires on synthetic input proves nothing about the tree.""" + from pathlib import Path + + path = Path("src/coder_eval/cli/run_task_internal_command.py") + source = path.read_text(encoding="utf-8") + assert not self._run(source, filepath=str(path)) + assert "_os._exit(137)" in source, "the guarded call must still exist" + + +class TestRuffExternalCoversEveryRule: + """Every CE rule's documented `# noqa` must be accepted by ruff. + + `[tool.ruff.lint] external` is what stops ruff reporting RUF102 "Invalid + rule code" for a suppression it does not own. It was hand-maintained and had + fallen ~14 ids behind — including CE047 and CE048, whose own docstrings + advertise `# noqa: CE047` / `# noqa: CE048` as the supported escape hatch. So + the first person to use the documented exemption got a red `make check` + instead, for doing exactly what the rule told them to. + """ + + @staticmethod + def _external() -> set[str]: + import tomllib + from pathlib import Path + + data = tomllib.loads(Path("pyproject.toml").read_text(encoding="utf-8")) + return set(data["tool"]["ruff"]["lint"]["external"]) + + def test_every_registered_rule_is_listed(self): + from tests.lint.runner import ALL_RULES + + missing = sorted({r.id for r in ALL_RULES} - self._external()) + assert not missing, f"add to [tool.ruff.lint] external in pyproject.toml: {missing}" + + def test_every_listed_id_is_well_formed(self): + """Cheap guard against a typo silently widening the allowlist.""" + bad = sorted(i for i in self._external() if not re.fullmatch(r"CE\d{3}", i)) + assert not bad, f"not a CE rule id: {bad}" + + +class TestCE053NoRunRecordFilenameLiteral: + """CE053 flags a `task.json` literal outside path_utils.""" + + @staticmethod + def _run(src: str, filepath: str = "src/coder_eval/reports.py"): + import ast + + from tests.lint.rules.ce053_run_record_filename_literal import NoRunRecordFilenameLiteral + + return NoRunRecordFilenameLiteral(filepath).check(ast.parse(src)) + + def test_flags_a_bare_literal(self): + assert self._run('path = run_dir / "task.json"') + + def test_flags_an_rglob(self): + """The three rglob sites are the ones the constant's own comment cites.""" + assert self._run('for p in run_dir.rglob("task.json"): pass') + + def test_flags_a_trailing_path_segment(self): + assert self._run('matches = sorted(d.glob("*/task.json"))') + + def test_flags_the_pre_grade_record_too(self): + assert self._run('backup = run_dir / "task.execute.json"') + + def test_allows_the_constant(self): + assert not self._run("path = run_dir / TASK_JSON_FILENAME") + + def test_allows_prose_that_merely_mentions_the_file(self): + """Naming the file in an error message is the point of the message.""" + assert not self._run('raise ValueError("no task.json in that directory")') + + def test_is_out_of_scope_in_path_utils(self): + assert not self._run('TASK_JSON_FILENAME = "task.json"', filepath="src/coder_eval/path_utils.py") + + def test_the_tree_is_clean(self): + """The migration must actually have happened — a rule whose only + evidence is synthetic proves nothing about the repo.""" + import ast + from pathlib import Path + + from tests.lint.rules.ce053_run_record_filename_literal import NoRunRecordFilenameLiteral + + offenders = [] + for path in Path("src/coder_eval").rglob("*.py"): + tree = ast.parse(path.read_text(encoding="utf-8")) + if NoRunRecordFilenameLiteral(str(path)).check(tree): + offenders.append(str(path)) + assert not offenders, offenders diff --git a/tests/test_detached_grading_boundaries.py b/tests/test_detached_grading_boundaries.py new file mode 100644 index 00000000..468ae91e --- /dev/null +++ b/tests/test_detached_grading_boundaries.py @@ -0,0 +1,709 @@ +"""The guards that decide WHERE and WHETHER a detached grade runs. + +Three boundaries, each shipped without a behavioural test: + +* the docker ``grade`` boundary — the only thing standing between + ``execute --driver docker`` against a stale image and a run that silently + publishes real verdicts; +* ``grading_sandbox_config`` — which decides whether a container task's criteria + may run on the grading host at all; +* the crash-recovery arm — the one a REAL grading failure takes, which is not the + one the existing test exercises (see ``TestGradingCrashLeavesTheRowRegradeable``). +""" + +from __future__ import annotations + +import json +import logging +import os +from datetime import datetime +from pathlib import Path +from unittest.mock import MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from coder_eval.cli import app +from coder_eval.models import ( + AgentKind, + EvaluationResult, + FileExistsCriterion, + FinalStatus, + ResolvedTask, + SandboxConfig, + TaskDefinition, + parse_agent_config, +) +from coder_eval.orchestration.regrade import ( + RegradeError, + grading_sandbox_config, + restore_pre_grade_record, + stamp_host_grading, +) +from coder_eval.path_utils import PRE_GRADE_JSON_FILENAME, TASK_JSON_FILENAME + + +runner = CliRunner() + + +def _task(driver: str = "tempdir") -> TaskDefinition: + return TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + sandbox=SandboxConfig(driver=driver), # type: ignore[arg-type] + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + + +def _result(status: FinalStatus = FinalStatus.NOT_GRADED) -> EvaluationResult: + return EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2026, 1, 1), + final_status=status, + iteration_count=0, + ) + + +# -------------------------------------------------------------------------- +# grading_sandbox_config: a docker task may not be graded on the host by default +# -------------------------------------------------------------------------- + + +class TestGradingSandboxConfig: + def test_a_docker_task_is_refused_by_default(self) -> None: + """Grading cannot start a container, and grading on the host runs the + task's criteria against a filesystem without the container's paths or + toolchain — scoring FAILURE for a trajectory `run` scored 1.0, and + executing its shell unsandboxed here.""" + with pytest.raises(RegradeError) as exc: + grading_sandbox_config(_task("docker")) + assert "--allow-host-grading" in str(exc.value) + + def test_the_opt_in_downgrades_to_tempdir(self) -> None: + cfg = grading_sandbox_config(_task("docker"), allow_host_grading=True) + assert cfg.driver == "tempdir" + + def test_a_non_docker_task_is_carried_through_unchanged(self) -> None: + """No rewrite at all on the ordinary path — the config the run used IS + the config the grade uses.""" + task = _task("tempdir") + cfg = grading_sandbox_config(task) + assert cfg.driver == "tempdir" + assert cfg == task.sandbox + + def test_a_host_graded_docker_row_is_stamped(self) -> None: + """A console warning does not travel with task.json into run.json, the + reports or the evalboard. The row must carry the caveat itself, or it is + silently comparable with a container-graded one.""" + result = _result() + stamp_host_grading(result, _task("docker")) + assert result.environment_info["graded_on_host"] is True + + def test_a_normal_row_carries_no_stamp(self) -> None: + result = _result() + stamp_host_grading(result, _task("tempdir")) + assert "graded_on_host" not in result.environment_info + + +# -------------------------------------------------------------------------- +# The docker `grade` boundary +# -------------------------------------------------------------------------- + + +def _docker_runner(*, grade: bool, tmp_path: Path): + from coder_eval.isolation.docker_runner import DockerRunner + + rt = ResolvedTask( + task=_task("docker"), + task_file=tmp_path / "t.yaml", + run_dir=tmp_path / "run", + variant_id="default", + original_task_id="t", + ) + return DockerRunner(rt, grade=grade) + + +class TestDockerGradeBoundary: + """`grade` crosses the container boundary only through context.json, so an + image that predates `execute` ignores the key and grades anyway.""" + + def test_a_graded_verdict_from_an_execute_run_is_refused(self, tmp_path: Path) -> None: + from coder_eval.isolation.docker_runner import DockerRunError + + runner_ = _docker_runner(grade=False, tmp_path=tmp_path) + with pytest.raises(DockerRunError, match="predates `execute`"): + runner_._assert_grade_honored(_result(FinalStatus.SUCCESS)) + + def test_an_ungraded_row_is_accepted(self, tmp_path: Path) -> None: + _docker_runner(grade=False, tmp_path=tmp_path)._assert_grade_honored(_result()) + + def test_an_execution_fact_is_exempt(self, tmp_path: Path) -> None: + """TIMEOUT / ERROR describe the agent phase, not grading. `execute` + reports them exactly as `run` does, so they are not evidence the image + graded anything.""" + for status in (FinalStatus.TIMEOUT, FinalStatus.ERROR, FinalStatus.BUILD_FAILED): + _docker_runner(grade=False, tmp_path=tmp_path)._assert_grade_honored(_result(status)) + + def test_a_graded_run_short_circuits(self, tmp_path: Path) -> None: + _docker_runner(grade=True, tmp_path=tmp_path)._assert_grade_honored(_result(FinalStatus.SUCCESS)) + + +class TestInContainerGradeCoercion: + """The container side of the same boundary.""" + + @staticmethod + def _run_with_context(tmp_path: Path, grade: object): + input_dir = tmp_path / "input" + input_dir.mkdir() + # Only the keys read BEFORE the grade coercion need real values; the + # command must refuse before it ever builds an Orchestrator. + context = {"variant_id": "default", "source_yaml": "task_id: t\n", "grade": grade} + (input_dir / "context.json").write_text(json.dumps(context), encoding="utf-8") + (input_dir / "task.yaml").write_text("task_id: t\n", encoding="utf-8") + return runner.invoke( + app, + ["_run-task-internal", "--input", str(input_dir), "--output", str(tmp_path / "out")], + ) + + def test_invoking_the_command_here_arms_no_process_lethal_watchdog(self, tmp_path: Path) -> None: + """The command's heartbeat watchdog reaps an ORPHANED CONTAINER by calling + `os._exit(137)` on itself. This suite invokes the command in-process, so an + unconditionally-armed thread exits the pytest WORKER instead — 40s later + (20s grace + 20s stale), inside whatever unrelated test that worker has + moved on to. It shipped that way: it killed a different test on each run + and on each platform, with no traceback, and the dead worker's lost + coverage data then failed the gate as `65.13 < 80.00`. + + Asserted on the live thread list rather than by patching `threading`, so + the guard is proven at the only place that matters — whether a thread now + exists in this process. + """ + import threading + + before = {t.name for t in threading.enumerate()} + self._run_with_context(tmp_path, True) + leaked = [t for t in threading.enumerate() if t.name not in before and t.daemon] + + assert not leaked, f"the command armed a process-lethal daemon thread outside a container: {leaked}" + + def test_a_non_boolean_grade_is_a_hard_error(self, tmp_path: Path) -> None: + """A hand-edited or older-format `"grade": "false"` is a truthy str typed + as bool, which would silently grade a run that asked not to be.""" + result = self._run_with_context(tmp_path, "false") + assert result.exit_code == 2 + assert "must be a boolean" in result.output + + # The in-container default is asserted BEHAVIOURALLY by + # `TestGradePlumbedIntoTheContainerOrchestrator::test_an_absent_key_still_grades`. + # It used to be a `assert 'context.get("grade", True)' in source` grep, which + # is the same static check that already failed here once: it passes happily + # while the line it describes is never executed. + + +# -------------------------------------------------------------------------- +# The crash-recovery arm +# -------------------------------------------------------------------------- + + +class TestGradingCrashLeavesTheRowRegradeable: + """``Orchestrator.run()`` converts an internal failure into a populated + ``FinalStatus.ERROR`` result rather than raising, so a REAL grading crash + takes the ``else:`` arm — not the ``except`` arm the pre-existing test + exercises by patching ``regrade_in_place`` to raise. + + Both arms must leave ``task.json`` ungraded. ERROR is "complete" for both + commands, so an ERROR row written over a NOT_GRADED one can never be graded + again without hand-restoring ``task.execute.json``. + """ + + @staticmethod + def _run_dir_with_backup(tmp_path: Path) -> Path: + run_dir = tmp_path / "00" + run_dir.mkdir(parents=True) + ungraded = _result().model_dump_json(indent=2) + (run_dir / PRE_GRADE_JSON_FILENAME).write_text(ungraded, encoding="utf-8") + # What _finalize_result already wrote before the caller saw the ERROR. + (run_dir / TASK_JSON_FILENAME).write_text( + _result(FinalStatus.ERROR).model_dump_json(indent=2), encoding="utf-8" + ) + return run_dir + + def test_restore_puts_the_ungraded_record_back(self, tmp_path: Path) -> None: + run_dir = self._run_dir_with_backup(tmp_path) + assert restore_pre_grade_record(run_dir) is True + on_disk = EvaluationResult.model_validate_json((run_dir / TASK_JSON_FILENAME).read_text(encoding="utf-8")) + assert on_disk.final_status is FinalStatus.NOT_GRADED + + def test_restore_is_a_no_op_when_nothing_was_overwritten(self, tmp_path: Path) -> None: + """The grade wrote into a fresh run dir, so the original is untouched.""" + run_dir = tmp_path / "00" + run_dir.mkdir(parents=True) + text = _result().model_dump_json(indent=2) + (run_dir / PRE_GRADE_JSON_FILENAME).write_text(text, encoding="utf-8") + (run_dir / TASK_JSON_FILENAME).write_text(text, encoding="utf-8") + assert restore_pre_grade_record(run_dir) is False + + def test_restore_refuses_to_write_through_a_symlink(self, tmp_path: Path) -> None: + run_dir = self._run_dir_with_backup(tmp_path) + victim = tmp_path / "victim.json" + victim.write_text("keep me", encoding="utf-8") + (run_dir / TASK_JSON_FILENAME).unlink() + (run_dir / TASK_JSON_FILENAME).symlink_to(victim) + + assert restore_pre_grade_record(run_dir) is False + assert victim.read_text(encoding="utf-8") == "keep me" + + async def test_the_resume_error_arm_restores_and_folds_back_ungraded(self, tmp_path: Path) -> None: + """The arm a real grading crash takes: `regrade_in_place` RETURNS an + ERROR result instead of raising.""" + from coder_eval.cli.run_command import _grade_resumed_tasks + + # task.json starts UNGRADED, as `execute` left it. The ERROR lands on + # disk during the grade, exactly as _finalize_result writes it before + # returning — which is the whole reason the in-memory fix is not enough. + run_dir = tmp_path / "00" + run_dir.mkdir(parents=True) + (run_dir / TASK_JSON_FILENAME).write_text(_result().model_dump_json(indent=2), encoding="utf-8") + rt = ResolvedTask( + task=_task(), + task_file=tmp_path / "t.yaml", + run_dir=run_dir, + variant_id="v", + original_task_id="t", + ) + + async def _crash(**_kwargs) -> EvaluationResult: + errored = _result(FinalStatus.ERROR) + (run_dir / TASK_JSON_FILENAME).write_text(errored.model_dump_json(indent=2), encoding="utf-8") + return errored + + with ( + patch("coder_eval.orchestration.regrade.default_workspace", return_value=tmp_path), + patch("coder_eval.orchestration.regrade.regrade_in_place", new=_crash), + ): + graded = await _grade_resumed_tasks([rt]) + + assert len(graded) == 1 + folded = graded[0][1].result + assert folded.final_status is FinalStatus.NOT_GRADED + assert "Grading errored during --resume" in (folded.error_message or "") + # And the on-disk row too — fixing only the in-memory result leaves + # run.json disagreeing with task.json, and task.json is what a later + # --resume reads. + on_disk = EvaluationResult.model_validate_json((run_dir / TASK_JSON_FILENAME).read_text(encoding="utf-8")) + assert on_disk.final_status is FinalStatus.NOT_GRADED + + async def test_an_unreadable_row_still_appears_as_ungraded(self, tmp_path: Path) -> None: + """Dropping it removed it from run.json AND from `tasks_not_graded`, + which is the counter the exit gate reads — so a resume whose rows were + all unreadable reported success.""" + from coder_eval.cli.run_command import _grade_resumed_tasks + + run_dir = tmp_path / "00" + run_dir.mkdir(parents=True) + (run_dir / TASK_JSON_FILENAME).write_text("{not json", encoding="utf-8") + rt = ResolvedTask( + task=_task(), + task_file=tmp_path / "t.yaml", + run_dir=run_dir, + variant_id="v", + original_task_id="t", + ) + + graded = await _grade_resumed_tasks([rt]) + + assert len(graded) == 1 + assert graded[0][1].result.final_status is FinalStatus.NOT_GRADED + assert "could not be read" in (graded[0][1].result.error_message or "") + + +# -------------------------------------------------------------------------- +# write_text_atomic hardening +# -------------------------------------------------------------------------- + + +class TestAtomicWriteIsNotAnOverwritePrimitive: + """``evaluate``'s write-back guards its DESTINATION against a symlink, but the + truncation happens through the temp name — so a pre-planted temp symlink + bypassed the guard entirely. The name is now unpredictable AND ``O_NOFOLLOW``, + so both halves are closed.""" + + def test_a_symlink_at_the_temp_name_is_never_followed(self, tmp_path: Path) -> None: + """Pinned by pinning the random half, since a real attacker cannot guess + it — the point is that O_NOFOLLOW still refuses even if they could.""" + from coder_eval import path_utils + + victim = tmp_path / "victim" + victim.write_text("keep me", encoding="utf-8") + target = tmp_path / "task.json" + planted = tmp_path / f"task.json.{os.getpid()}.deadbeef.tmp" + planted.symlink_to(victim) + + with ( + patch.object(path_utils.secrets, "token_hex", return_value="deadbeef"), + pytest.raises(OSError), + ): + path_utils.write_text_atomic(target, "attacker content") + assert victim.read_text(encoding="utf-8") == "keep me" + + def test_a_stale_temp_file_does_not_wedge_the_write(self, tmp_path: Path) -> None: + """The regression that mattered most. Under a FIXED temp name, `O_EXCL` + turned a leftover from a SIGKILL into a permanent refusal to persist the + record — so the row reported ERROR, `--resume` saw no task.json, re-ran + the task into the same run dir, and hit the same file again, re-paying + for the agent on every pass.""" + from coder_eval.path_utils import write_text_atomic + + target = tmp_path / "task.json" + (tmp_path / "task.json.tmp").write_text("leftover from a hard kill", encoding="utf-8") + + write_text_atomic(target, "hello") + + assert target.read_text(encoding="utf-8") == "hello" + + def test_the_record_is_readable_by_other_uids(self, tmp_path: Path) -> None: + """Under `driver: docker` the in-container orchestrator writes this file + as root straight into the bind-mounted host run dir, and the host reads it + back as the invoking uid through an UNGUARDED `read_text`. Creating it + 0600 made that raise PermissionError for every docker-driver task on + Linux — invisible on macOS, where Docker Desktop remaps ownership.""" + from coder_eval.path_utils import write_text_atomic + + target = tmp_path / "task.json" + write_text_atomic(target, "hello") + + if os.name == "nt": + pytest.skip("Windows has no POSIX mode bits; stat reports 0o666 for any writable file") + + mode = target.stat().st_mode & 0o777 + assert mode & 0o044, f"task.json is not group/other-readable: {mode:#o}" + # Readable is the requirement; WRITABLE is not. A create mode of 0o666 + # produced 0644 under the usual umask and a world-writable record under + # umask 0 — the widest mode that keeps the fix is 0o644. + assert not mode & 0o022, f"task.json is group/other-writable: {mode:#o}" + + def test_an_ordinary_write_still_works(self, tmp_path: Path) -> None: + from coder_eval.path_utils import write_text_atomic + + target = tmp_path / "task.json" + write_text_atomic(target, "hello") + assert target.read_text(encoding="utf-8") == "hello" + assert not list(tmp_path.glob("*.tmp")) + + def test_a_failed_write_leaves_no_temp_file(self, tmp_path: Path) -> None: + from coder_eval.path_utils import write_text_atomic + + target = tmp_path / "task.json" + with patch("os.replace", side_effect=OSError("boom")), pytest.raises(OSError): + write_text_atomic(target, "hello") + assert not list(tmp_path.glob("*.tmp")) + + +# -------------------------------------------------------------------------- +# default_workspace traversal +# -------------------------------------------------------------------------- + + +def test_a_task_id_that_escapes_the_artifacts_tree_is_refused(tmp_path: Path) -> None: + """`task_id` is an unvalidated string out of the run's own task.json, and + `artifacts / "../../.."` joins to a real directory that `is_dir()` confirms. + Every run_command criterion would then execute with that as its cwd. The + sibling `sandbox_path` branch was containment-checked; this one was not.""" + from coder_eval.orchestration.regrade import default_workspace + + (tmp_path / "run" / "artifacts").mkdir(parents=True) + (tmp_path / "outside").mkdir() + + prior = _result() + prior.task_id = "../../outside" + + with pytest.raises(RegradeError, match="resolves outside"): + default_workspace(tmp_path / "run", prior) + + +# -------------------------------------------------------------------------- +# The reference digest +# -------------------------------------------------------------------------- + + +def test_the_reference_digest_is_computed_over_a_staged_copy(tmp_path: Path) -> None: + """The recorded digest is taken over the per-run STAGED copy, which strips + `.git`. Digesting the raw source instead compares two differently-filtered + trees, so any reference that is a git checkout — the case the ignore list + exists for — reports a permanent false mismatch and un-grades the row. + """ + from coder_eval.orchestration.evaluation import stage_reference_dir + from coder_eval.orchestration.regrade import _staged_digest + from coder_eval.path_utils import digest_tree + + source = tmp_path / "reference" + (source / ".git").mkdir(parents=True) + (source / ".git" / "HEAD").write_text("ref: refs/heads/main\n", encoding="utf-8") + (source / "solution.py").write_text("print('hi')\n", encoding="utf-8") + + staged = stage_reference_dir(source, tmp_path / "staged") + recorded = digest_tree(staged) # exactly what Orchestrator._stage_reference records + + assert _staged_digest(source) == recorded + # And the naive comparison this replaced would have failed. + assert digest_tree(source) != recorded + + +def test_a_reference_edited_since_the_run_is_still_caught(tmp_path: Path) -> None: + """The guard must not become permissive: only the FILTER changed, not what + counts as a change.""" + from coder_eval.orchestration.regrade import _staged_digest + + source = tmp_path / "reference" + source.mkdir() + (source / "solution.py").write_text("print('hi')\n", encoding="utf-8") + before = _staged_digest(source) + + (source / "solution.py").write_text("print('tampered')\n", encoding="utf-8") + assert _staged_digest(source) != before + + +def test_a_reference_that_vanished_is_refused_not_skipped(tmp_path: Path) -> None: + """A missing answer key must raise, not return silently — grading against + nothing produces an ordinary-looking score.""" + from coder_eval.orchestration.regrade import verify_reference_unchanged + + task = _task() + task.reference = MagicMock() + prior = _result() + prior.environment_info["reference_digest"] = "deadbeef" + + with ( + patch("coder_eval.orchestration.evaluation.resolve_reference_dir", return_value=tmp_path / "gone"), + pytest.raises(RegradeError, match="is gone"), + ): + verify_reference_unchanged(prior, task, tmp_path / "t.yaml") + + +# -------------------------------------------------------------------------- +# The recorded-config gate must cover PROVISIONING, not just criteria +# -------------------------------------------------------------------------- + + +class TestRecordedProvisioningIsGatedToo: + """`grading_sandbox_config` carries the recorded `sandbox` block through + untouched, and the `--copy` branch then calls `Sandbox.setup` — which reaches + `uv pip install `, `npm install ` and + `git clone `. A package name is arbitrary code at install time. + + The gate walked only `success_criteria` + hooks, so a shared run directory + whose criteria were all `file_exists` passed it and still ran installers. + """ + + @staticmethod + def _task_with(**sandbox_kwargs) -> TaskDefinition: + return TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + sandbox=SandboxConfig(**sandbox_kwargs), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + + def test_python_packages_are_named(self) -> None: + from coder_eval.models import PythonEnvConfig + from coder_eval.orchestration.regrade import embedded_commands + + task = self._task_with(python=PythonEnvConfig(env_packages=["attacker-pkg"])) + assert any("attacker-pkg" in c for c in embedded_commands(task)) + + def test_node_packages_are_named(self) -> None: + from coder_eval.models import NodeEnvConfig + from coder_eval.orchestration.regrade import embedded_commands + + task = self._task_with(node=NodeEnvConfig(env_packages=["evil-npm"])) + assert any("evil-npm" in c for c in embedded_commands(task)) + + def test_a_repo_source_url_is_named(self) -> None: + from coder_eval.models import RepoSource + from coder_eval.orchestration.regrade import embedded_commands + + task = self._task_with(template_sources=[RepoSource(url="https://evil.example/repo.git")]) + assert any("evil.example" in c for c in embedded_commands(task)) + + def test_provisioning_alone_triggers_the_refusal(self, tmp_path: Path) -> None: + """The exact repro: every criterion is a file_exists, so the old scan + found nothing and the installers ran with no opt-in.""" + from coder_eval.models import PythonEnvConfig + from coder_eval.orchestration.regrade import check_embedded_commands + + task = self._task_with(python=PythonEnvConfig(env_packages=["attacker-pkg"])) + with pytest.raises(RegradeError, match="--allow-recorded-commands"): + check_embedded_commands(task, tmp_path, allow_recorded_commands=False) + + def test_the_in_place_path_is_exempt(self, tmp_path: Path) -> None: + """`adopt()` installs nothing, so in place these are not capabilities the + run dir has — and refusing there would break the headline flow.""" + from coder_eval.models import PythonEnvConfig + from coder_eval.orchestration.regrade import check_embedded_commands + + task = self._task_with(python=PythonEnvConfig(env_packages=["attacker-pkg"])) + check_embedded_commands(task, tmp_path, allow_recorded_commands=False, include_setup_phase=False) + + def test_an_llm_judge_is_named(self) -> None: + """No shell, but it spends the grader's model budget and ships the graded + artifacts to a provider the recorded config chose.""" + from coder_eval.models import LLMJudgeCriterion + from coder_eval.orchestration.regrade import embedded_commands + + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[LLMJudgeCriterion(description="x", prompt="grade it")], + ) + assert any("llm_judge" in c for c in embedded_commands(task)) + + +class TestGradePlumbedIntoTheContainerOrchestrator: + """The `grade` value reaching the in-container Orchestrator was asserted only + by grepping the module's own source text, which passes while the line it + describes is never executed — deleting `grade=grade` left every test green.""" + + @staticmethod + def _invoke(tmp_path: Path, context: dict) -> object: + captured: dict[str, object] = {} + + class _FakeOrchestrator: + def __init__(self, **kwargs): + captured.update(kwargs) + + async def run(self): + return _result() + + input_dir = tmp_path / "input" + input_dir.mkdir() + task_yaml = ( + "task_id: t\ndescription: d\nagent:\n type: none\n" + "success_criteria:\n - type: file_exists\n path: p.txt\n description: x\n" + ) + (input_dir / "task.yaml").write_text(task_yaml, encoding="utf-8") + (input_dir / "context.json").write_text( + json.dumps({"variant_id": "default", "source_yaml": task_yaml, **context}), encoding="utf-8" + ) + with patch("coder_eval.orchestrator.Orchestrator", _FakeOrchestrator): + runner.invoke( + app, + ["_run-task-internal", "--input", str(input_dir), "--output", str(tmp_path / "out")], + ) + return captured.get("grade") + + def test_execute_forwards_grade_false(self, tmp_path: Path) -> None: + assert self._invoke(tmp_path, {"grade": False}) is False + + def test_an_absent_key_still_grades(self, tmp_path: Path) -> None: + """A host predating `execute` writes no key; the container must keep its + original behaviour rather than silently withholding verdicts.""" + assert self._invoke(tmp_path, {}) is True + + +class TestContainerContextIsValidated: + """`json.loads` returns Any, so an annotation at this boundary reads like a + guarantee and enforces nothing.""" + + @staticmethod + def _run(tmp_path: Path, context: dict): + input_dir = tmp_path / "input" + input_dir.mkdir() + (input_dir / "task.yaml").write_text("task_id: t\n", encoding="utf-8") + (input_dir / "context.json").write_text(json.dumps(context), encoding="utf-8") + return runner.invoke(app, ["_run-task-internal", "--input", str(input_dir), "--output", str(tmp_path / "out")]) + + def test_a_non_string_variant_id_is_refused(self, tmp_path: Path) -> None: + result = self._run(tmp_path, {"variant_id": 7, "source_yaml": "task_id: t\n"}) + assert result.exit_code == 2 + assert "variant_id" in result.output + + def test_a_string_replicate_index_is_refused(self, tmp_path: Path) -> None: + """`"00"` would reach build_task_run_dir typed as int.""" + result = self._run(tmp_path, {"variant_id": "default", "replicate_index": "00", "source_yaml": "task_id: t\n"}) + assert result.exit_code == 2 + assert "replicate_index" in result.output + + +class TestCriterionPathsCannotEscapeTheSandbox: + """`Path('/tmp/sandbox') / '/etc/passwd'` is `/etc/passwd` — pathlib discards + the prefix on an absolute right operand — and criterion paths were the one + task-authored path in sandbox.py that skipped `_resolve_within_sandbox`. + + Defensible while a task YAML was operator-supplied; not once + `evaluate ` began rebuilding the criteria list from a shareable run + directory, which turns `file_contains` / `file_check` / `file_matches_regex` + into a pass-fail oracle over any file the grading user can read. + + The out-of-sandbox fixture is called ``outside``, NOT ``secret``: CodeQL's + sensitive-data heuristic keys on the identifier, so a local named ``secret`` + became a taint SOURCE, flowed through ``resolve_files`` into the escape + warning, and raised a high-severity `py/clear-text-logging-sensitive-data` + against production code that only logs a task-authored glob pattern. + """ + + @staticmethod + def _sandbox(tmp_path: Path): + from coder_eval.sandbox import Sandbox + + sandbox = Sandbox(SandboxConfig(driver="tempdir"), task_id="t") + work = tmp_path / "work" + work.mkdir() + sandbox.sandbox_dir = work + return sandbox, work + + def test_an_absolute_path_resolves_to_nothing(self, tmp_path: Path) -> None: + sandbox, _ = self._sandbox(tmp_path) + outside = tmp_path / "outside.txt" + outside.write_text("payload", encoding="utf-8") + + assert sandbox.resolve_files(str(outside)) == [] + + def test_a_dotdot_traversal_resolves_to_nothing(self, tmp_path: Path) -> None: + sandbox, _ = self._sandbox(tmp_path) + (tmp_path / "outside.txt").write_text("payload", encoding="utf-8") + + assert sandbox.resolve_files("../outside.txt") == [] + + def test_a_glob_cannot_escape_either(self, tmp_path: Path) -> None: + sandbox, _ = self._sandbox(tmp_path) + (tmp_path / "outside.txt").write_text("payload", encoding="utf-8") + + assert sandbox.resolve_files("../*.txt") == [] + + def test_an_ordinary_in_sandbox_path_still_resolves(self, tmp_path: Path) -> None: + """The control — the guard must not break normal grading.""" + sandbox, work = self._sandbox(tmp_path) + (work / "proof.txt").write_text("ok", encoding="utf-8") + + assert sandbox.resolve_files("proof.txt") == [work / "proof.txt"] + assert sandbox.resolve_files("*.txt") == [work / "proof.txt"] + + def test_the_escape_is_reported_once_naming_the_authored_pattern( + self, tmp_path: Path, caplog: pytest.LogCaptureFixture + ) -> None: + """One warning per criterion, naming the string the task author wrote. + + Reporting from `_within_sandbox` instead fired once per rejected glob + match — a burst of near-identical lines for a wide pattern — and named + a resolved absolute path, which is just that same string joined onto a + tempdir. + """ + sandbox, _ = self._sandbox(tmp_path) + for name in ("a.txt", "b.txt", "c.txt"): + (tmp_path / name).write_text("token", encoding="utf-8") + + with caplog.at_level(logging.WARNING, logger="coder_eval.sandbox"): + assert sandbox.resolve_files("../*.txt") == [] + + escapes = [r for r in caplog.records if "resolves outside the sandbox" in r.getMessage()] + assert len(escapes) == 1, [r.getMessage() for r in escapes] + assert "'../*.txt'" in escapes[0].getMessage() diff --git a/tests/test_detached_grading_guards.py b/tests/test_detached_grading_guards.py new file mode 100644 index 00000000..6f3c75b4 --- /dev/null +++ b/tests/test_detached_grading_guards.py @@ -0,0 +1,285 @@ +"""The guards around detached grading, each tested on the branch that fires. + +Every case here is a refusal, a skip, or a mode selection — the branches that +exist precisely because taking the other one would produce a plausible number +that is wrong. They were all shipped with coverage on the happy path only. +""" + +from __future__ import annotations + +import os +from datetime import datetime +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest +from typer.testing import CliRunner + +from coder_eval.cli import app +from coder_eval.cli.evaluate_command import run_evaluation +from coder_eval.models import ( + AgentKind, + EvaluationResult, + FileExistsCriterion, + FinalStatus, + TaskDefinition, + parse_agent_config, +) +from coder_eval.orchestrator import Orchestrator + + +runner = CliRunner() + + +# An agentless task: `type: none` runs no agent and forbids `initial_prompt` +# (there is nothing to read it), which is what makes it usable with no API key. +_AGENTLESS = """task_id: t +description: d +agent: + type: none +success_criteria: + - type: file_exists + path: proof.txt + description: x +""" + +# A simulation task needs a real agent type — the refusal under `execute` fires +# during resolution, so the agent is never created. +_SIMULATED = """task_id: t +description: d +initial_prompt: p +agent: + type: claude-code +simulation: + enabled: true + persona: a user + goal: get it done +success_criteria: + - type: file_exists + path: proof.txt + description: x +""" + + +def _task(tmp_path: Path, *, simulation: bool = False) -> Path: + path = tmp_path / "t.yaml" + path.write_text(_SIMULATED if simulation else _AGENTLESS, encoding="utf-8") + return path + + +# -------------------------------------------------------------------------- +# `execute` refuses simulation tasks +# -------------------------------------------------------------------------- + + +def test_execute_refuses_a_simulation_task_by_name(tmp_path: Path) -> None: + """The dialog loop reads criteria results to decide whether to keep talking, + so an ungraded dialog would silently change its own stopping behavior. The + refusal must name the task, or a user cannot tell which one to remove.""" + result = runner.invoke(app, ["execute", str(_task(tmp_path, simulation=True)), "--run-dir", str(tmp_path / "r")]) + + assert result.exit_code != 0 + assert "simulation" in result.output.lower() + assert "t" in result.output + + +def test_run_still_accepts_the_same_simulation_task(tmp_path: Path) -> None: + """The control: the refusal is about `execute`, not about the task.""" + task = _task(tmp_path, simulation=True) + with patch("coder_eval.cli.run_command._run_with_experiment", new=AsyncMock(return_value=(MagicMock(), 0))): + result = runner.invoke(app, ["run", str(task), "--run-dir", str(tmp_path / "r")]) + assert "does not support simulation" not in result.output + + +# -------------------------------------------------------------------------- +# The evaluate-only path refuses grade=False +# -------------------------------------------------------------------------- + + +async def test_grading_off_on_the_evaluate_only_path_is_refused(tmp_path: Path) -> None: + """No agent AND no grading is a no-op that would still write a task.json. + Refusing beats producing an empty row that looks like a result.""" + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v", grade=False) + orch.success_checker = MagicMock() + orch.result = EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2026, 1, 1), + final_status=FinalStatus.FAILURE, + iteration_count=0, + ) + + with pytest.raises(ValueError, match="meaningless on the evaluate-only path"): + await orch._evaluation_loop() + + +# -------------------------------------------------------------------------- +# --in-place / --copy +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("flag", "expect_adopt"), + [(None, False), ("--in-place", True), ("--copy", False)], + ids=["default-for-a-work-dir-is-copy", "explicit-in-place", "explicit-copy"], +) +def test_the_flag_decides_adopt_versus_setup_on_a_work_dir( + tmp_path: Path, flag: str | None, expect_adopt: bool +) -> None: + """The choice is not cosmetic: the copy path filters node_modules / dist / + build / .venv, so a criterion reading those fails as a copying artifact.""" + work = tmp_path / "work" + work.mkdir() + (work / "proof.txt").write_text("x", encoding="utf-8") + # --run-dir is not incidental: without it the grade lands in a repo-relative + # runs//, which several xdist workers race over. + args = ["evaluate", str(_task(tmp_path)), str(work), "--run-dir", str(tmp_path / "r")] + if flag: + args.append(flag) + + with ( + patch("coder_eval.sandbox.Sandbox.adopt") as adopt, + patch("coder_eval.sandbox.Sandbox.setup") as setup, + ): + runner.invoke(app, args) + + assert adopt.called is expect_adopt + assert setup.called is not expect_adopt + + +def test_run_evaluation_has_real_defaults_not_typer_sentinels(tmp_path: Path) -> None: + """`run_evaluation` exists because calling the Typer command in-process hands + every unspecified option an `OptionInfo` — and `in_place=None` became truthy, + silently flipping the copy default to in-place.""" + import inspect + + sig = inspect.signature(run_evaluation) + for name in ("work_dir", "workspace", "in_place", "run_dir"): + assert sig.parameters[name].default is None, f"{name} must default to a real None" + assert sig.parameters["preserve"].default is True + + +# -------------------------------------------------------------------------- +# The PATH round trip +# -------------------------------------------------------------------------- + + +def test_the_agents_path_is_persisted_so_a_later_grade_can_restore_it(tmp_path: Path) -> None: + """Without the persisted value a detached grade resolves `run_command` + binaries against ambient PATH and can disagree with the run it grades.""" + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v") + orch.result = EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2026, 1, 1), + final_status=FinalStatus.FAILURE, + iteration_count=0, + ) + orch.sandbox = MagicMock() + orch.agent = MagicMock() + orch.agent.get_sdk_options.return_value = {"env": {"PATH": f"{tmp_path}:/usr/bin"}} + + orch._sync_sandbox_command_path_with_agent() + + assert "command_base_path" in orch.result.environment_info + + +def test_a_restored_path_drops_entries_inside_the_graded_run(tmp_path: Path) -> None: + """The restored value is PREPENDED ahead of the host PATH and comes out of the + run's own task.json — a shareable artifact. Every entry an attacker could + have placed there must be dropped; only the run's real toolchain survives. + + The run-directory SIBLING case is the one this test used to pin the wrong way + round: it asserted such an entry was kept. The workspace is only part of the + run dir, and ``artifacts/`` and the run root travel in the same archive. + """ + run_dir = tmp_path / "run" + workspace = run_dir / "ws" + (workspace / "bin").mkdir(parents=True) + sibling = run_dir / "artifacts-shim" # inside the run dir, outside the workspace + sibling.mkdir() + toolchain = tmp_path / "toolchain" # a genuine location outside the run entirely + toolchain.mkdir() + relative = Path("evilbin") + + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + orch = Orchestrator(task=task, run_dir=run_dir, variant_id="v") + orch.sandbox = MagicMock() + orch.sandbox.sandbox_dir = workspace + + # os.pathsep, not a hardcoded ":" — the separator is ";" on Windows, where a + # colon-joined value parses as one (non-existent) entry and every assertion + # below passes vacuously against an empty result. + recorded = os.pathsep.join( + [ + str(workspace / "bin"), + str(sibling), + str(relative), + str(toolchain), + str(tmp_path / "gone"), + ] + ) + kept = orch._sanitize_restored_path(recorded) + + assert str(toolchain.resolve()) in kept, "a real out-of-run toolchain entry is the point of the restore" + assert str(workspace) not in kept, "an entry inside the graded workspace must be dropped" + assert str(sibling) not in kept, "an entry elsewhere in the run directory must be dropped too" + assert "evilbin" not in kept, "a relative entry would resolve against the grader's cwd" + assert "gone" not in kept, "a non-existent entry buys no parity" + + +# -------------------------------------------------------------------------- +# The LiteLLM cost join +# -------------------------------------------------------------------------- + + +def test_the_actual_cost_join_is_skipped_on_a_re_grade(tmp_path: Path) -> None: + """The join keys on a per-Orchestrator nonce the prior turns never carried, + so running it on a re-grade would clobber already-correct per-turn costs.""" + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + prior = EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2026, 1, 1), + final_status=FinalStatus.NOT_GRADED, + iteration_count=0, + ) + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v", prior_result=prior) + orch.result = prior + + with patch("coder_eval.litellm_cost.apply_actual_cost") as apply: + orch._join_litellm_actual_cost() + + apply.assert_not_called() diff --git a/tests/test_early_stop.py b/tests/test_early_stop.py index 5f647392..ff57c63d 100644 --- a/tests/test_early_stop.py +++ b/tests/test_early_stop.py @@ -38,7 +38,7 @@ from coder_eval.agents.claude_code_agent import ClaudeCodeAgent from coder_eval.agents.codex_agent import CodexAgent, _CodexTurnState from coder_eval.agents.registry import AgentRegistry -from coder_eval.cli.plan_command import plan_command +from coder_eval.cli.plan_command import run_plan from coder_eval.config import settings from coder_eval.criteria import CriterionRegistry, init_criteria from coder_eval.criteria.command_executed import CommandExecutedChecker @@ -1041,7 +1041,7 @@ def test_run_surface_variant_inherits_task_threshold_with_stop_early_false(self, assert early_stop_active(by_variant["smoke"]) is True def _run_plan(self, task_file: Path, exp_dir: Path) -> tuple[str, int]: - """Invoke the real plan_command against a minimal single-variant experiment. + """Invoke the real plan body against a minimal single-variant experiment. Returns the concatenated console output and the exit code (0 when plan returned normally). @@ -1058,7 +1058,7 @@ def _run_plan(self, task_file: Path, exp_dir: Path) -> tuple[str, int]: patch("coder_eval.cli.plan_command.console") as mock_console, ): try: - plan_command(task_files=[task_file], experiment=exp_file) + run_plan(task_files=[task_file], experiment=exp_file) except typer.Exit as exc: exit_code = exc.exit_code printed = " ".join(str(call) for call in mock_console.print.call_args_list) diff --git a/tests/test_evaluate_command.py b/tests/test_evaluate_command.py index 67a98046..06d26f3a 100644 --- a/tests/test_evaluate_command.py +++ b/tests/test_evaluate_command.py @@ -20,7 +20,7 @@ def test_evaluate_command_success(tmp_path): (work_dir / "app.py").write_text("print('hello')") # Import and run command - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() @@ -28,7 +28,7 @@ def test_evaluate_command_success(tmp_path): with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): # Should not raise - all criteria pass with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) assert exc_info.value.exit_code == 0 @@ -59,14 +59,14 @@ def test_evaluate_command_defaults_agent_type_when_missing(tmp_path): work_dir.mkdir() (work_dir / "app.py").write_text("print('hello')") - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) assert exc_info.value.exit_code == 0 @@ -85,7 +85,7 @@ def test_evaluate_command_maps_preserve_to_mode(tmp_path, preserve, expected_mod run_dir = tmp_path / "run" run_dir.mkdir() - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation captured: dict[str, PreservationMode] = {} @@ -105,7 +105,7 @@ async def run(self): patch("coder_eval.cli.evaluate_command.Orchestrator", _CapturingOrchestrator), pytest.raises(_StopError), ): - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir, preserve=preserve) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir, preserve=preserve) assert captured["mode"] == PreservationMode(expected_mode) @@ -119,7 +119,7 @@ def test_evaluate_command_failure(tmp_path): work_dir.mkdir() # Import and run command - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() @@ -127,7 +127,7 @@ def test_evaluate_command_failure(tmp_path): with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): # Should fail - file doesn't exist with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) assert exc_info.value.exit_code == 1 @@ -140,14 +140,14 @@ def test_evaluate_command_informational_criterion_does_not_fail_exit(tmp_path): work_dir.mkdir() (work_dir / "app.py").write_text("print('hello')") # gating criterion passes; missing.py absent - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) # The only gating criterion passed → exit 0, despite the weight=0 miss. assert exc_info.value.exit_code == 0 @@ -163,14 +163,14 @@ def test_evaluate_command_invalid_task_file(tmp_path): work_dir.mkdir() # Import and run command - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) assert exc_info.value.exit_code == 1 @@ -183,14 +183,14 @@ def test_evaluate_command_invalid_work_dir(tmp_path): work_dir = tmp_path / "nonexistent" # Import and run command - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) assert exc_info.value.exit_code == 1 @@ -205,14 +205,14 @@ def test_evaluate_command_multiple_criteria(tmp_path): (work_dir / "app.py").write_text("print('hello')") # Import and run command - from coder_eval.cli.evaluate_command import evaluate_command + from coder_eval.cli.evaluate_command import run_evaluation run_dir = tmp_path / "run" run_dir.mkdir() with patch("coder_eval.cli.console.console.print"), patch("coder_eval.logging_config.setup_logging"): with pytest.raises(typer.Exit) as exc_info: - evaluate_command(task_file=task_file, work_dir=work_dir, run_dir=run_dir) + run_evaluation(task_or_run_dir=task_file, work_dir=work_dir, run_dir=run_dir) # All 3 criteria should pass assert exc_info.value.exit_code == 0 diff --git a/tests/test_evaluate_target.py b/tests/test_evaluate_target.py new file mode 100644 index 00000000..1b908419 --- /dev/null +++ b/tests/test_evaluate_target.py @@ -0,0 +1,102 @@ +"""`resolve_evaluate_target` — the shape detection behind `coder-eval evaluate`. + +Pure and exhaustively testable by design: the command accepts two forms that +look alike on the command line, and picking the wrong one silently grades the +wrong thing. Every combination of (one arg / two args) x (run dir / plain dir / +file / missing) is enumerated here. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from coder_eval.cli.evaluate_target import ( + EvaluateMode, + EvaluateTargetError, + is_run_dir, + resolve_evaluate_target, +) + + +def _run_dir(tmp_path: Path, name: str = "run") -> Path: + d = tmp_path / name + d.mkdir() + (d / "task.json").write_text(json.dumps({"task_id": "t"}), encoding="utf-8") + return d + + +def _plain_dir(tmp_path: Path, name: str = "work") -> Path: + d = tmp_path / name + d.mkdir() + return d + + +def _file(tmp_path: Path, name: str = "task.yaml") -> Path: + f = tmp_path / name + f.write_text("task_id: t", encoding="utf-8") + return f + + +def test_is_run_dir_keys_on_task_json(tmp_path: Path) -> None: + assert is_run_dir(_run_dir(tmp_path)) + assert not is_run_dir(_plain_dir(tmp_path)) + + +# --- one argument --------------------------------------------------------- + + +def test_lone_run_dir_is_run_dir_mode(tmp_path: Path) -> None: + run = _run_dir(tmp_path) + resolved = resolve_evaluate_target(run, None) + assert resolved.mode is EvaluateMode.RUN_DIR + assert resolved.target == run + assert resolved.task_file is None, "a run dir carries its own config; nothing to supply" + + +def test_lone_task_file_is_rejected_with_the_fix(tmp_path: Path) -> None: + """A task file alone names no place to grade.""" + with pytest.raises(EvaluateTargetError, match="not a directory"): + resolve_evaluate_target(_file(tmp_path), None) + + +def test_lone_plain_dir_is_rejected_with_the_fix(tmp_path: Path) -> None: + """A plain directory alone names no criteria.""" + plain = _plain_dir(tmp_path) + with pytest.raises(EvaluateTargetError) as exc: + resolve_evaluate_target(plain, None) + # The message must name the missing piece AND the corrected command — a + # caller here is one argument away from the right invocation. + assert "task.json" in str(exc.value) + assert "" in str(exc.value) + + +# --- two arguments -------------------------------------------------------- + + +def test_task_file_plus_plain_dir_is_the_original_form(tmp_path: Path) -> None: + """The pre-existing shape must keep resolving exactly as before.""" + task, work = _file(tmp_path), _plain_dir(tmp_path) + resolved = resolve_evaluate_target(task, work) + assert resolved.mode is EvaluateMode.WORK_DIR + assert resolved.target == work + assert resolved.task_file == task + + +def test_task_file_plus_run_dir_re_grades_with_the_given_task(tmp_path: Path) -> None: + """Iterating on criteria against a run you already paid for: the run supplies + the trajectory and workspace, the explicit file supplies the criteria.""" + task, run = _file(tmp_path), _run_dir(tmp_path) + resolved = resolve_evaluate_target(task, run) + assert resolved.mode is EvaluateMode.RUN_DIR + assert resolved.target == run + assert resolved.task_file == task, "the override must survive; it is the whole point of this form" + + +def test_a_nonexistent_second_arg_stays_work_dir_mode(tmp_path: Path) -> None: + """Shape detection must not invent run-dir mode for a path that isn't there; + the command reports the missing directory itself, with a clearer message.""" + resolved = resolve_evaluate_target(_file(tmp_path), tmp_path / "nope") + assert resolved.mode is EvaluateMode.WORK_DIR diff --git a/tests/test_execute_command.py b/tests/test_execute_command.py new file mode 100644 index 00000000..9a015431 --- /dev/null +++ b/tests/test_execute_command.py @@ -0,0 +1,302 @@ +"""`coder-eval execute` — run without grading. + +Three layers, deliberately: + +* **End-to-end** against the agentless task (`agent: {type: none}`), which needs + no API key and is fully deterministic. This is the only layer that proves the + whole chain — CLI → batch → Orchestrator → task.json → run.json — actually + withholds the verdict while still executing. +* **Contrast** — the same task under `run` must still produce SUCCESS with a real + score. Without it, a totally broken `execute` (or a broken fixture) would pass + the assertions above by accident. +* **Wiring** — the two commands share one body, so a signature or `grade` drift + is caught mechanically rather than by a human noticing. +""" + +from __future__ import annotations + +import json +import re +from pathlib import Path +from typing import Any +from unittest.mock import patch + +import pytest +from typer.testing import CliRunner + +from coder_eval.cli import app +from coder_eval.models import FinalStatus, RunSummary + + +runner = CliRunner() + +# Rich styles each `--option` token in help text, and it splits the token across +# several style spans (`--junit-xml` renders as `-` + `-junit` + `-xml`, each with +# its own escape sequence). Styling is ON whenever rich thinks it is writing to a +# terminal — which includes GitHub Actions, so a bare substring check over +# `result.output` passes locally and fails only in CI. Same helper, same reason, +# as tests/test_cli_type_flag.py. +_ANSI_RE = re.compile(r"\x1b\[[0-9;]*m") + + +def _strip_ansi(s: str) -> str: + return _ANSI_RE.sub("", s) + + +# The agentless smoke task: no agent, no model call, and a pre_run that writes a +# file its criteria read back. Executing it must still write that file (proving +# the run really happened) while scoring nothing. +AGENTLESS_TASK = Path("tasks/agentless_smoke_test.yaml") + + +def _invoke(command: str, run_dir: Path) -> Any: + return runner.invoke( + app, + [command, str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--preservation-mode", "MOVE_ON_WRITE"], + ) + + +def _task_json(run_dir: Path) -> dict[str, Any]: + matches = sorted(run_dir.glob("**/task.json")) + assert len(matches) == 1, f"expected exactly one task.json under {run_dir}, got {matches}" + return json.loads(matches[0].read_text(encoding="utf-8")) + + +# -------------------------------------------------------------------------- +# The status itself +# -------------------------------------------------------------------------- + + +def test_not_graded_is_its_own_category() -> None: + """NOT_GRADED must not fold into succeeded/failed/error — each would lie.""" + assert FinalStatus.NOT_GRADED.category == "ungraded" + assert FinalStatus.NOT_GRADED.icon == "?" + + +def test_ungraded_leaves_both_sides_of_the_pass_rate() -> None: + """An all-ungraded run has NO pass rate — not a 0% one.""" + summary = RunSummary( + run_id="r", + start_time="2026-01-01T00:00:00", # type: ignore[arg-type] + end_time="2026-01-01T00:01:00", # type: ignore[arg-type] + total_duration_seconds=60.0, + tasks_run=2, + tasks_succeeded=0, + tasks_failed=0, + tasks_error=0, + tasks_not_graded=2, + task_results=[], + framework_version="test", + ) + assert summary.tasks_graded == 0 + assert summary.pass_rate is None + assert summary.error_share is None + + +def test_ungraded_does_not_dilute_a_partially_graded_run() -> None: + """One pass out of one graded task is 100%, even alongside three ungraded ones.""" + summary = RunSummary( + run_id="r", + start_time="2026-01-01T00:00:00", # type: ignore[arg-type] + end_time="2026-01-01T00:01:00", # type: ignore[arg-type] + total_duration_seconds=60.0, + tasks_run=4, + tasks_succeeded=1, + tasks_failed=0, + tasks_error=0, + tasks_not_graded=3, + task_results=[], + framework_version="test", + ) + assert summary.pass_rate == 1.0 + + +def test_task_count_invariant_counts_the_ungraded_bucket() -> None: + """The fourth bucket is part of the invariant, not a free-floating sub-counter.""" + with pytest.raises(ValueError, match="Task count invariant violated"): + RunSummary( + run_id="r", + start_time="2026-01-01T00:00:00", # type: ignore[arg-type] + end_time="2026-01-01T00:01:00", # type: ignore[arg-type] + total_duration_seconds=1.0, + tasks_run=2, + tasks_succeeded=0, + tasks_failed=0, + tasks_error=0, + tasks_not_graded=1, # 0+0+0+1 != 2 + task_results=[], + framework_version="test", + ) + + +# -------------------------------------------------------------------------- +# End to end +# -------------------------------------------------------------------------- + + +@pytest.mark.skipif(not AGENTLESS_TASK.is_file(), reason="needs a source checkout (tasks/ is not in the wheel)") +def test_execute_runs_the_task_but_grades_nothing(tmp_path: Path) -> None: + run_dir = tmp_path / "run" + result = _invoke("execute", run_dir) + + assert result.exit_code == 0, result.output + + row = _task_json(run_dir) + # The verdict is withheld ... + assert row["final_status"] == FinalStatus.NOT_GRADED.value + assert row["weighted_score"] is None, "must be None, never 0.0 — 0.0 reads as 'graded and scored zero'" + assert row["success_criteria_results"] == [] + # ... but the run itself demonstrably happened: pre_run wrote its file into + # the preserved sandbox. Without this the test would also pass if `execute` + # had simply skipped the task. + proof = sorted(run_dir.glob("**/artifacts/**/proof.txt")) + assert proof, f"pre_run did not run — no proof.txt under {run_dir}" + assert "coder-eval-ran-without-a-coder" in proof[0].read_text(encoding="utf-8") + + +@pytest.mark.skipif(not AGENTLESS_TASK.is_file(), reason="needs a source checkout (tasks/ is not in the wheel)") +def test_execute_run_json_reports_ungraded_not_failed(tmp_path: Path) -> None: + run_dir = tmp_path / "run" + assert _invoke("execute", run_dir).exit_code == 0 + + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert summary["tasks_run"] == 1 + assert summary["tasks_not_graded"] == 1 + # The whole point: an ungraded task is NOT a failure and NOT an error. + assert summary["tasks_failed"] == 0 + assert summary["tasks_error"] == 0 + assert summary["tasks_succeeded"] == 0 + assert summary["pass_rate"] is None + + +@pytest.mark.skipif(not AGENTLESS_TASK.is_file(), reason="needs a source checkout (tasks/ is not in the wheel)") +def test_run_still_grades_the_same_task(tmp_path: Path) -> None: + """The negative control: `run` must still score this task, or the assertions + above prove nothing about grading being *deliberately* skipped.""" + run_dir = tmp_path / "run" + result = _invoke("run", run_dir) + + assert result.exit_code == 0, result.output + row = _task_json(run_dir) + assert row["final_status"] == FinalStatus.SUCCESS.value + assert row["weighted_score"] == 1.0 + assert len(row["success_criteria_results"]) == 2 + + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert summary["tasks_not_graded"] == 0 + assert summary["tasks_succeeded"] == 1 + assert summary["pass_rate"] == 1.0 + + +# -------------------------------------------------------------------------- +# Wiring: one shared body, two commands +# -------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + ("command", "module", "expected_grade"), + [("run", "run_command", True), ("execute", "execute_command", False)], +) +def test_both_commands_call_the_shared_pipeline(command: str, module: str, expected_grade: bool) -> None: + """`run` and `execute` differ ONLY in `grade` — no third code path. + + Patched per module because each command imported ``run_pipeline`` into its own + namespace; patching the defining module would silently miss ``execute``. + """ + with patch(f"coder_eval.cli.{module}.run_pipeline") as pipeline: + result = runner.invoke(app, [command, "a.yaml"]) + assert result.exit_code == 0, result.output + pipeline.assert_called_once() + assert pipeline.call_args.kwargs["grade"] is expected_grade + + +def _option_names(command: str) -> set[str]: + import typer.main + + click_app = typer.main.get_command(app) + cmd = click_app.commands[command] # type: ignore[attr-defined] + return {opt for param in cmd.params for opt in getattr(param, "opts", [])} + + +# `execute` restates `run`'s Typer signature because Typer builds its parser from +# the signature and there is no way to share one. That duplication is the drift +# risk this test exists to close: a flag added to `run` must be added here too, +# or consciously listed below as a deliberate omission. +_DELIBERATELY_ABSENT_FROM_EXECUTE = { + "--junit-xml", # a report of verdicts, and there are none + "--allow-host-grading", # decides how a GRADE runs; execute never grades +} + + +def test_execute_exposes_run_flags_minus_the_refused_one() -> None: + run_opts = _option_names("run") + execute_opts = _option_names("execute") + + missing = run_opts - execute_opts - _DELIBERATELY_ABSENT_FROM_EXECUTE + assert not missing, ( + f"`run` has flag(s) {sorted(missing)} that `execute` lacks. Add them to " + "execute_command's signature, or list them in _DELIBERATELY_ABSENT_FROM_EXECUTE " + "with the reason they are refused." + ) + assert not execute_opts - run_opts, "`execute` must not grow flags of its own" + # The omissions must be real, not stale entries masking a genuine gap. + assert not _DELIBERATELY_ABSENT_FROM_EXECUTE & execute_opts + + +# -------------------------------------------------------------------------- +# The docker boundary +# -------------------------------------------------------------------------- + + +async def _staged_context(tmp_path: Path, *, grade: bool) -> dict[str, Any]: + """Stage a docker task's inputs and read back the context.json the container sees.""" + from coder_eval.isolation.docker_runner import DockerRunner + from coder_eval.models import ResolvedTask, TaskDefinition + + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent={"type": "claude-code"}, + sandbox={"driver": "docker"}, + success_criteria=[{"type": "file_exists", "path": "x.txt", "description": "x"}], + ) + rt = ResolvedTask( + task=task, + task_file=tmp_path / "t.yaml", + run_dir=tmp_path / "run", + variant_id="default", + original_task_id="t", + ) + staged = tmp_path / "input" + staged.mkdir() + await DockerRunner(rt, grade=grade)._stage_inputs(staged) + return json.loads((staged / "context.json").read_text(encoding="utf-8")) + + +@pytest.mark.parametrize("grade", [True, False]) +async def test_docker_forwards_grade_to_the_container(tmp_path: Path, grade: bool) -> None: + """`grade` is a run-level CLI decision, so it is NOT recoverable from the staged + task.yaml on the container side — it has to cross the boundary in context.json. + Without this, `execute --driver docker` would silently grade after all.""" + assert (await _staged_context(tmp_path, grade=grade))["grade"] is grade + + +# The in-container grading default is asserted behaviourally in +# tests/test_detached_grading_boundaries.py +# (`TestGradePlumbedIntoTheContainerOrchestrator`), which drives the real command +# with `Orchestrator` patched and reads the captured `grade` kwarg. The source-text +# grep that used to live here was duplicated verbatim in that file and proved +# nothing: deleting `grade=grade` at the call site left both greps green. + + +def test_execute_help_explains_the_refused_flags() -> None: + """The omission is documented in the help, not silently absent — a user who + reaches for `--junit-xml` needs to learn why it is refused, not just that it + is unrecognised. (Presence as a real *flag* is covered by the option-set test + above; here we only require the help text to mention it.)""" + result = runner.invoke(app, ["execute", "--help"]) + assert result.exit_code == 0 + output = _strip_ansi(result.output) + for flag in _DELIBERATELY_ABSENT_FROM_EXECUTE: + assert flag in output, f"execute's help should explain why {flag} is unavailable" diff --git a/tests/test_execute_evaluate_loop.py b/tests/test_execute_evaluate_loop.py new file mode 100644 index 00000000..bed20a62 --- /dev/null +++ b/tests/test_execute_evaluate_loop.py @@ -0,0 +1,480 @@ +"""The `execute` -> `evaluate` -> `aggregate` loop. + +`coder-eval execute` withholds the verdict; `coder-eval evaluate ` +supplies it later. The pair only earns its keep if it ends up where a single +`coder-eval run` would have: same status, same score, same criteria, and a +`run.json` the rest of the toolchain can read. + +Everything here runs against the agentless task — deterministic, no API key. +""" + +from __future__ import annotations + +import json +import shutil +from pathlib import Path +from typing import Any +from unittest.mock import AsyncMock, patch + +import pytest +from typer.testing import CliRunner + +from coder_eval.cli import app +from coder_eval.models import FinalStatus + + +runner = CliRunner() + +AGENTLESS_TASK = Path("tasks/agentless_smoke_test.yaml") + +pytestmark = pytest.mark.skipif( + not AGENTLESS_TASK.is_file(), reason="needs a source checkout (tasks/ is not in the wheel)" +) + + +def _task_dir(run_dir: Path) -> Path: + matches = sorted(p.parent for p in run_dir.glob("**/task.json")) + assert len(matches) == 1, f"expected exactly one task.json under {run_dir}, got {matches}" + return matches[0] + + +def _row(task_dir: Path, name: str = "task.json") -> dict[str, Any]: + return json.loads((task_dir / name).read_text(encoding="utf-8")) + + +def _invoke(args: list[str], *, expect_exit: int = 0) -> Any: + """Run a CLI command and pin its exit code. + + ``expect_exit`` is explicit rather than "0 unless it raised" because the exit + code IS the contract for a CI wrapper: a helper that always demanded 0 once + pinned a preserved-TIMEOUT row exiting 0 under "All criteria passed" as the + expected behaviour. + """ + result = runner.invoke(app, args) + assert result.exit_code == expect_exit, ( + f"{args} exited {result.exit_code}, expected {expect_exit}:\n{result.output}" + ) + return result + + +def test_execute_then_evaluate_reaches_the_same_verdict_as_run(tmp_path: Path) -> None: + """The headline guarantee, asserted against a real `run` rather than a + hardcoded expectation — so a change that breaks BOTH paths still fails.""" + direct = tmp_path / "direct" + _invoke(["run", str(AGENTLESS_TASK), "--run-dir", str(direct)]) + expected = _row(_task_dir(direct)) + + split = tmp_path / "split" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(split)]) + _invoke(["evaluate", str(_task_dir(split))]) + actual = _row(_task_dir(split)) + + assert expected["final_status"] == FinalStatus.SUCCESS.value, "the fixture must actually pass under `run`" + assert actual["final_status"] == expected["final_status"] + assert actual["weighted_score"] == expected["weighted_score"] + assert [c["criterion_type"] for c in actual["success_criteria_results"]] == [ + c["criterion_type"] for c in expected["success_criteria_results"] + ] + assert [c["score"] for c in actual["success_criteria_results"]] == [ + c["score"] for c in expected["success_criteria_results"] + ] + + +def test_evaluate_upgrades_the_row_in_place_and_keeps_the_original(tmp_path: Path) -> None: + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + assert _row(task_dir)["final_status"] == FinalStatus.NOT_GRADED.value + + _invoke(["evaluate", str(task_dir)]) + + assert _row(task_dir)["final_status"] == FinalStatus.SUCCESS.value + # The pre-grade record survives, so "this run was executed separately" stays + # auditable rather than being silently overwritten. + assert _row(task_dir, "task.execute.json")["final_status"] == FinalStatus.NOT_GRADED.value + + +def test_aggregate_rebuilds_a_graded_run_json_with_no_extra_step(tmp_path: Path) -> None: + """Grading in place is what makes the rest of the toolchain free: the + existing `aggregate` command sees the upgraded rows with no new code.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + assert json.loads((run_dir / "run.json").read_text(encoding="utf-8"))["tasks_not_graded"] == 1 + + _invoke(["evaluate", str(_task_dir(run_dir))]) + _invoke(["aggregate", str(run_dir)]) + + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert summary["tasks_not_graded"] == 0 + assert summary["tasks_succeeded"] == 1 + assert summary["pass_rate"] == 1.0 + + +def test_re_grade_carries_the_trajectory_not_an_empty_one(tmp_path: Path) -> None: + """Criteria that read the agent's tool calls (command_executed, + skill_triggered, judges with trajectory) score off `iterations`. A re-grade + that dropped them would silently fail every such criterion.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + executed = _row(task_dir) + + _invoke(["evaluate", str(task_dir)]) + graded = _row(task_dir) + + assert len(graded["iterations"]) == len(executed["iterations"]) + assert graded["iteration_count"] == executed["iteration_count"] + + +def test_evaluate_does_not_move_or_delete_the_graded_workspace(tmp_path: Path) -> None: + """Run-dir mode adopts the workspace; the caller keeps ownership.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + proof = sorted(run_dir.glob("**/artifacts/**/proof.txt")) + assert proof, "fixture precondition: execute preserved a workspace" + + _invoke(["evaluate", str(_task_dir(run_dir))]) + + assert proof[0].is_file(), "the adopted workspace was moved or deleted" + + +def test_evaluate_still_grades_a_plain_directory(tmp_path: Path) -> None: + """The original two-argument form must keep working unchanged.""" + work = tmp_path / "work" + work.mkdir() + (work / "proof.txt").write_text("coder-eval-ran-without-a-coder", encoding="utf-8") + + result = runner.invoke(app, ["evaluate", str(AGENTLESS_TASK), str(work)]) + + assert result.exit_code == 0, result.output + assert "All criteria passed" in result.output + + +def test_evaluate_rejects_workspace_flag_outside_run_dir_mode(tmp_path: Path) -> None: + work = tmp_path / "work" + work.mkdir() + result = runner.invoke(app, ["evaluate", str(AGENTLESS_TASK), str(work), "--workspace", str(work)]) + assert result.exit_code != 0 + assert "run directory only" in result.output + + +# -------------------------------------------------------------------------- +# `run --resume` over an executed run dir +# -------------------------------------------------------------------------- + + +def test_run_resume_grades_the_ungraded_rows_it_finds(tmp_path: Path) -> None: + """The whole point of the resume fix: `run --resume` over an executed run + must GRADE those rows, not report "already complete" and exit 0.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.NOT_GRADED.value + + result = _invoke(["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert "grading 1" in result.output + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.SUCCESS.value + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert summary["tasks_not_graded"] == 0 + assert summary["tasks_succeeded"] == 1 + assert summary["pass_rate"] == 1.0 + + +def test_run_resume_does_not_re_execute_the_agent(tmp_path: Path) -> None: + """Grading must reuse the trajectory on disk. Re-executing would discard the + expensive half — the reason `execute` and `run` were split at all.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + executed = _row(_task_dir(run_dir)) + + result = _invoke(["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert "running 0 remaining" in result.output, "the task was re-executed instead of graded" + graded = _row(_task_dir(run_dir)) + assert len(graded["iterations"]) == len(executed["iterations"]) + # The row still describes the TASK, not the grading pass: a re-execution + # would restamp these, and reporting the grading pass's 2s as the task's + # duration would corrupt average_duration and every harness comparison. + assert graded["started_at"] == executed["started_at"] + assert graded["duration_seconds"] == executed["duration_seconds"] + # The grading pass's own cost is kept alongside, not discarded. + assert "grading_duration_seconds" in graded["environment_info"] + # The pre-grade record is preserved by this path too, not just by `evaluate`. + assert _row(_task_dir(run_dir), "task.execute.json")["final_status"] == FinalStatus.NOT_GRADED.value + + +def test_execute_resume_treats_an_executed_row_as_done(tmp_path: Path) -> None: + """`execute --resume` owes a NOT_GRADED row nothing — it finished executing.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + + result = _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert "1 task(s) already complete" in result.output + assert "grading" not in result.output + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.NOT_GRADED.value + + +def test_a_detached_grade_does_not_re_run_pre_run_against_the_workspace(tmp_path: Path) -> None: + """`run()` calls the pre/post-run hooks unconditionally, with cwd = the + sandbox. On an ADOPTED sandbox that sandbox is the agent's own output, and + several in-tree tasks stage fixtures there (`cp -a /app/[!.]* "$PWD/"`), so + re-running them would overwrite the deliverables before the criteria read + them — changing the verdict and destroying preserved artifacts.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + proof = sorted(run_dir.glob("**/artifacts/**/proof.txt"))[0] + # Mark the agent's file. The fixture's pre_run rewrites proof.txt from + # scratch, so a re-run would wipe this marker. + proof.write_text("coder-eval-ran-without-a-coder AND-THE-AGENT-EDITED-THIS", encoding="utf-8") + + _invoke(["evaluate", str(task_dir)]) + + assert "AND-THE-AGENT-EDITED-THIS" in proof.read_text(encoding="utf-8"), ( + "pre_run re-ran against the adopted workspace and overwrote the agent's work" + ) + # The hooks' recorded outcomes are carried over rather than lost. + assert _row(task_dir)["pre_run_results"], "the execute phase's pre_run results were dropped" + + +def test_run_resume_exits_non_zero_when_it_cannot_grade(tmp_path: Path) -> None: + """`run` was asked for a verdict. If grading fails, reporting exit 0 tells CI + the suite is fine when nothing was actually scored.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + # Remove the workspace so the re-grade has nothing to grade against. + shutil.rmtree(run_dir / "default" / "agentless_smoke_test" / "00" / "artifacts", ignore_errors=True) + row = _task_dir(run_dir) / "task.json" + record = json.loads(row.read_text(encoding="utf-8")) + record["sandbox_path"] = str(tmp_path / "gone") + row.write_text(json.dumps(record), encoding="utf-8") + + result = runner.invoke(app, ["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert result.exit_code != 0, "a run that graded nothing must not report success" + + +def test_execute_still_exits_zero_with_every_row_ungraded(tmp_path: Path) -> None: + """The other side of the rule above: under `execute` an ungraded row is the + expected outcome, not a failure of the command.""" + run_dir = tmp_path / "r" + result = runner.invoke(app, ["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + assert result.exit_code == 0, result.output + + +def test_execute_to_run_resume_emits_no_config_drift_warning(tmp_path: Path) -> None: + """`grade` is exempt from the fingerprint diff: this flow is supported, and + the warning's "keeps their original-config results" text is wrong for it.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + + result = _invoke(["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert "run config changed" not in result.output + + +def test_run_resume_keeps_the_row_regradeable_when_grading_crashes(tmp_path: Path) -> None: + """A grading crash is not a verdict about the run. Folding the ORIGINAL + ungraded row back keeps the task re-gradeable — writing ERROR over it would + not, since ERROR is "complete" for both commands.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + + with patch( + "coder_eval.orchestration.regrade.regrade_in_place", + new=AsyncMock(side_effect=RuntimeError("checker exploded")), + ): + result = runner.invoke(app, ["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert result.exit_code != 0, "a resume that graded nothing must not report success" + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.NOT_GRADED.value + # The reason is durable, not console-only. It lands in run.json rather than + # task.json: task.json stays the pristine execute record, which is what keeps + # the row re-gradeable below. + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert "checker exploded" in str(summary["task_results"]) + + # And the row really is still re-gradeable. + _invoke(["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.SUCCESS.value + + +def test_run_resume_reports_a_failing_verdict_and_exits_non_zero(tmp_path: Path) -> None: + """The other resume gate: grading that SUCCEEDS but fails the criteria.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + # Remove the file the criteria read, so the grade legitimately fails. + for proof in run_dir.glob("**/artifacts/**/proof.txt"): + proof.unlink() + + result = runner.invoke(app, ["run", str(AGENTLESS_TASK), "--run-dir", str(run_dir), "--resume"]) + + assert result.exit_code != 0 + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.FAILURE.value + summary = json.loads((run_dir / "run.json").read_text(encoding="utf-8")) + assert summary["tasks_failed"] == 1 + assert summary["tasks_not_graded"] == 0 + + +def test_evaluate_grades_the_directory_named_by_workspace(tmp_path: Path) -> None: + """--workspace exists for a verifier that built its own /app; nothing else + asserted it actually grades that directory rather than the run's artifacts.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + elsewhere = tmp_path / "elsewhere" + elsewhere.mkdir() + (elsewhere / "proof.txt").write_text("coder-eval-ran-without-a-coder", encoding="utf-8") + # Make the run's own artifacts FAIL, so a pass can only come from --workspace. + for proof in run_dir.glob("**/artifacts/**/proof.txt"): + proof.unlink() + + _invoke(["evaluate", str(_task_dir(run_dir)), "--workspace", str(elsewhere)]) + + assert _row(_task_dir(run_dir))["final_status"] == FinalStatus.SUCCESS.value + + +def test_evaluate_refuses_to_re_grade_a_run_that_errored(tmp_path: Path) -> None: + """Grading may only move NOT_GRADED to a verdict. An ERROR / TIMEOUT run is + an execution fact this pass neither repeated nor observed — laundering it + into SUCCESS would report a crashed run as a pass. + + The exit code has to agree. It exited 0 under "All criteria passed! ✓" for a + row `run.json` counts as failed, because the gate read the criteria tally + rather than the outcome — so a CI wrapper shelling `coder-eval evaluate` + went green on a timed-out run.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + row = _row(task_dir) + row["final_status"] = FinalStatus.TIMEOUT.value + (task_dir / "task.json").write_text(json.dumps(row), encoding="utf-8") + + result = _invoke(["evaluate", str(task_dir)], expect_exit=1) + + assert _row(task_dir)["final_status"] == FinalStatus.TIMEOUT.value + assert "All criteria passed" not in result.output + assert FinalStatus.TIMEOUT.value in result.output + + +def test_an_inherited_error_still_renders_its_criteria_and_keeps_the_status(tmp_path: Path) -> None: + """The other arm of the same confusion. A PRESERVED ERROR is not a grading + crash: the ERROR branch fired anyway, printed the ORIGINAL run's crash + message as though grading had failed, claimed the row was "left ungraded" + (it was not — the restored record still reads ERROR), and discarded a verdict + it had just computed.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + row = _row(task_dir) + row["final_status"] = FinalStatus.ERROR.value + row["error_message"] = "agent crashed during the original run" + (task_dir / "task.json").write_text(json.dumps(row), encoding="utf-8") + + result = _invoke(["evaluate", str(task_dir)], expect_exit=1) + + assert "Criteria Results" in result.output, "the computed verdict was thrown away" + assert "left ungraded" not in result.output, "grading did not crash; saying so is false" + assert _row(task_dir)["final_status"] == FinalStatus.ERROR.value + + +def test_grading_the_same_run_twice_reaches_the_same_verdict(tmp_path: Path) -> None: + """Idempotence. A second grade must see the same workspace the first did — + it catches both a pre_run that mutated the tree and a lost sandbox_path.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + + _invoke(["evaluate", str(task_dir)]) + first = _row(task_dir) + _invoke(["evaluate", str(task_dir)]) + second = _row(task_dir) + + assert second["final_status"] == first["final_status"] + assert second["weighted_score"] == first["weighted_score"] + assert second["sandbox_path"] == first["sandbox_path"], "the artifacts pointer must survive a re-grade" + # The pre-grade record is still the ORIGINAL ungraded one, not the first grade's. + assert _row(task_dir, "task.execute.json")["final_status"] == FinalStatus.NOT_GRADED.value + + +# -------------------------------------------------------------------------- +# `execute` withholds the verdict, never the facts of the run +# -------------------------------------------------------------------------- + + +def test_execute_records_max_turns_exhausted_exactly_as_run_does(tmp_path: Path) -> None: + """`max_turns_exhausted` is a fact about the RUN, not a verdict. + + It used to be captured AFTER the grading switch's early return, so under + `execute` it was never recorded at all: the row finalized NOT_GRADED and the + command exited 0 where `run` reported MAX_TURNS_EXHAUSTED and exited 1 — for + identical agent output. `_seed_from_prior_result` cannot restore a fact the + execute phase never captured, so a later `evaluate` inherited the wrong + terminal status too. + """ + from coder_eval.streaming.collector import EventCollector + + # One turn that reports the cap was hit, on both paths. + original = EventCollector.build_turn_record + + def _exhausted(self, *args: Any, **kwargs: Any): + record = original(self, *args, **kwargs) + record.max_turns_exhausted = True + return record + + def _run(command: str, run_dir: Path) -> Any: + with patch.object(EventCollector, "build_turn_record", _exhausted): + return runner.invoke(app, [command, str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + + graded_dir = tmp_path / "graded" + _run("run", graded_dir) + graded = _row(_task_dir(graded_dir)) + + executed_dir = tmp_path / "executed" + _run("execute", executed_dir) + executed = _row(_task_dir(executed_dir)) + + assert graded["max_turns_exhausted"] is True, "the fixture must actually exhaust turns under `run`" + assert executed["max_turns_exhausted"] is True, ( + "`execute` dropped a fact about the run. Only the verdict is withheld." + ) + # The FACT is recorded; the STATUS is not decided. `run` returns SUCCESS for + # a max-turns trajectory whose criteria pass and only falls through to + # MAX_TURNS_EXHAUSTED when they fail — so the status is not knowable without + # grading, and claiming it here made it both terminal and permanent + # (MAX_TURNS_EXHAUSTED is an execution fact, which the detached grade may + # never overturn). + assert executed["final_status"] == FinalStatus.NOT_GRADED.value + + # The parity that matters: grading the executed run must land exactly where + # `run` did. Asserting only the executed half is what let the divergence ship. + _invoke(["evaluate", str(_task_dir(executed_dir))]) + regraded = _row(_task_dir(executed_dir)) + + assert regraded["final_status"] == graded["final_status"] + assert regraded["weighted_score"] == graded["weighted_score"] + assert regraded["max_turns_exhausted"] is True, "the fact must survive the grade too" + + +def test_a_detached_grade_keeps_the_runs_api_routing_not_the_graders(tmp_path: Path) -> None: + """`_seed_from_prior_result`'s contract is that the PRIOR run wins on + environment_info. The route recorder ran after the seeding and overwrote + `api_routing` with the grading host's, leaving a self-contradictory record — + a direct route named beside the run's stale bedrock fields.""" + run_dir = tmp_path / "r" + _invoke(["execute", str(AGENTLESS_TASK), "--run-dir", str(run_dir)]) + task_dir = _task_dir(run_dir) + + before = _row(task_dir)["environment_info"] + before["api_routing"] = "a_backend_this_host_does_not_use" + row = _row(task_dir) + row["environment_info"] = before + (task_dir / "task.json").write_text(json.dumps(row), encoding="utf-8") + + _invoke(["evaluate", str(task_dir)]) + after = _row(task_dir)["environment_info"] + + assert after["api_routing"] == "a_backend_this_host_does_not_use", ( + "the grade overwrote the RUN's recorded routing with the grading host's" + ) + assert after.get("graded_by_api_routing"), "the grader's own route must still be recorded, just not in place" diff --git a/tests/test_experiment_reports.py b/tests/test_experiment_reports.py index 3e212da5..71f7d139 100644 --- a/tests/test_experiment_reports.py +++ b/tests/test_experiment_reports.py @@ -1708,3 +1708,102 @@ def test_experiment_report_snapshot_replicates(self): ) md = ExperimentReportGenerator.generate_experiment_report(result) assert_matches_snapshot(md, "experiment_replicates.md") + + +class TestUngradedRenderingInMarkdown: + """The Markdown twin of `TestUngradedRenderingInHtml`. + + Five ungraded edits landed on `reports_html.py` with tests and the identical + five on `reports_experiment.py` with none, so the two renderers could drift + apart while the suite stayed green. These also cover the four new public + `reports_stats` helpers, which no test called at all — including + `is_env_table_key`, a behaviour change (three key families newly hidden from + BOTH Environment tables) that shipped unasserted in either direction. + """ + + @staticmethod + def _ungraded_result(): + from coder_eval.models import ExperimentResult, VariantAggregate, VariantResult + from coder_eval.models.experiment import TaskExperimentSummary + + return ExperimentResult( + experiment_id="exec-only", + description="an execute run, not yet graded", + variant_ids=["a"], + task_summaries=[ + TaskExperimentSummary( + task_id="t", + variant_results=[ + VariantResult( + variant_id="a", + task_id="t", + weighted_score=None, + final_status="NOT_GRADED", + duration_seconds=12.0, + total_tokens=900, + ) + ], + best_variant="a", + is_tie=True, + score_spread=0.0, + ) + ], + variant_aggregates={ + "a": VariantAggregate( + variant_id="a", + tasks_run=1, + tasks_succeeded=0, + tasks_failed=0, + tasks_error=0, + tasks_not_graded=1, + average_score=None, + average_duration=12.0, + ) + }, + total_duration_seconds=12.0, + ) + + def test_the_report_names_the_fourth_bucket_and_publishes_no_zero(self): + from coder_eval.reports_experiment import ExperimentReportGenerator + + md = ExperimentReportGenerator.generate_experiment_report(self._ungraded_result()) + + assert "Not Graded" in md, "the fourth bucket must be named, or an execute run reads as 0 of 1" + # The score CELL, not the whole document — the spread column legitimately + # renders 0.000 for a single-variant experiment. + score_row = next(line for line in md.splitlines() if line.startswith("| t |")) + assert "n/a" in score_row, score_row + assert "0.000" not in score_row.split("|")[2], "an unmeasured row rendered as a scored zero" + + def test_duration_and_tokens_survive_an_ungraded_row(self): + """Only the SCORE is missing. Dropping the row whole made an all-ungraded + experiment render `Avg Duration | N/A` with Tokens absent entirely — the + bug `collect_variant_series`' own comment describes.""" + from coder_eval.reports_stats import collect_variant_series + + series = collect_variant_series(self._ungraded_result())["a"] + + assert series.scores == [] + assert series.durations == [12.0] + assert series.tokens == [900.0] + + def test_format_score_distinguishes_unmeasured_from_zero(self): + from coder_eval.reports_stats import UNGRADED_SCORE_TEXT, format_score + + assert format_score(None) == UNGRADED_SCORE_TEXT + assert format_score(0.0) == "0.000" + assert format_score(None) != format_score(0.0) + + @pytest.mark.parametrize( + "key", ["installed_tools", "command_base_path", "reference_digest", "graded_by_api_routing"] + ) + def test_env_table_hides_the_noise_keys(self, key): + from coder_eval.reports_stats import is_env_table_key + + assert not is_env_table_key(key) + + def test_env_table_keeps_ordinary_keys(self): + from coder_eval.reports_stats import is_env_table_key + + assert is_env_table_key("coder_eval") + assert is_env_table_key("api_routing") diff --git a/tests/test_experiment_runner.py b/tests/test_experiment_runner.py index 1ee6f08d..a9259fa5 100644 --- a/tests/test_experiment_runner.py +++ b/tests/test_experiment_runner.py @@ -750,3 +750,84 @@ def test_all_replicates_errored_gives_zero_duration(self): vr = result.task_summaries[0].variant_results[0] assert abs(vr.duration_seconds - 0.0) < 1e-9 assert vr.final_status == FinalStatus.ERROR + + +class TestAggregationCountsErroredRowsAsMisses: + """An ERRORED row must count as 0.0, not vanish from both sides. + + ``weighted_score is None`` and ``final_status.category == "ungraded"`` look + interchangeable and are not: an ERROR / BUILD_FAILED row also has no score. + Filtering on the score dropped it from the numerator AND the denominator, so + a nightly where one image build failed reported a HIGHER headline score than + a clean one, and A/B comparisons were biased toward whichever variant errored + more. Only an ungraded row leaves both sides — nothing measured it. + """ + + @staticmethod + def _result(task_id: str, variant_id: str, status: str, score: float | None) -> EvaluationResult: + return EvaluationResult( + task_id=task_id, + task_description="d", + variant_id=variant_id, + agent_type="claude-code", + started_at=datetime.now(), + final_status=status, # type: ignore[arg-type] + weighted_score=score, + duration_seconds=1.0, + iteration_count=1, + environment_info={}, + ) + + def _aggregate(self, rows: list[tuple[str, str, str, float | None]]) -> ExperimentResult: + return aggregate_results( + experiment_id="e", + description="d", + variant_ids=sorted({vid for _, vid, _, _ in rows}), + task_results=[ + TaskResult( + task_id=task_id, + variant_id=variant_id, + result=self._result(task_id, variant_id, status, score), + duration=1.0, + ) + for task_id, variant_id, status, score in rows + ], + total_duration=1.0, + ) + + def test_an_errored_row_drags_the_average_down_like_a_miss(self): + clean = self._aggregate([("a", "v", "SUCCESS", 1.0), ("b", "v", "FAILURE", 0.0)]) + errored = self._aggregate([("a", "v", "SUCCESS", 1.0), ("b", "v", "ERROR", None)]) + + assert clean.variant_aggregates["v"].average_score == 0.5 + assert errored.variant_aggregates["v"].average_score == 0.5, ( + "an infrastructure-failure night must not score HIGHER than a clean one" + ) + + def test_a_build_failure_is_a_miss_too(self): + result = self._aggregate([("a", "v", "SUCCESS", 1.0), ("b", "v", "BUILD_FAILED", None)]) + assert result.variant_aggregates["v"].average_score == 0.5 + + def test_an_ungraded_row_leaves_both_sides(self): + result = self._aggregate([("a", "v", "SUCCESS", 1.0), ("b", "v", "NOT_GRADED", None)]) + assert result.variant_aggregates["v"].average_score == 1.0 + + def test_a_fully_ungraded_variant_has_no_average(self): + result = self._aggregate([("a", "v", "NOT_GRADED", None)]) + assert result.variant_aggregates["v"].average_score is None + + def test_score_spread_still_sees_a_fully_errored_arm(self): + """A task where variant B fully errored reported spread 0.0 instead of + the real gap, because B had no score to compare.""" + result = self._aggregate([("a", "x", "SUCCESS", 0.8), ("a", "y", "ERROR", None)]) + assert result.task_summaries[0].score_spread == pytest.approx(0.8) + + def test_no_arbitrary_winner_when_nothing_was_scored(self): + """`variants[0]` named whichever arm the input happened to list first, + with `is_tie=False` asserting it was a real result — so swapping the + inputs flipped the reported winner.""" + one = self._aggregate([("a", "x", "NOT_GRADED", None), ("a", "y", "NOT_GRADED", None)]) + other = self._aggregate([("a", "y", "NOT_GRADED", None), ("a", "x", "NOT_GRADED", None)]) + + assert one.task_summaries[0].best_variant == other.task_summaries[0].best_variant + assert one.task_summaries[0].is_tie is True diff --git a/tests/test_litellm_cost.py b/tests/test_litellm_cost.py index ea23c8c7..ce40a43b 100644 --- a/tests/test_litellm_cost.py +++ b/tests/test_litellm_cost.py @@ -257,6 +257,8 @@ def test_joins_on_litellm_route(self, tmp_path, monkeypatch): _cost_correlation_run_id=run_id, _cost_attempt_nonce="att1", _log_task_id="calc", + # A live run, not a re-grade: the join applies (see _join_litellm_actual_cost). + prior_result=None, ) orch_mod.Orchestrator._join_litellm_actual_cost(fake) assert fake.result.iterations[0].token_usage.total_cost_usd == 0.09 # static 0.5 overridden @@ -270,6 +272,8 @@ def test_join_never_raises_on_bad_log(self, tmp_path, monkeypatch): _cost_correlation_run_id="R", _cost_attempt_nonce="att1", _log_task_id="calc", + # A live run, not a re-grade: the join applies (see _join_litellm_actual_cost). + prior_result=None, ) orch_mod.Orchestrator._join_litellm_actual_cost(fake) # missing file → no-op, no raise assert fake.result.iterations[0].token_usage.total_cost_usd == 0.5 @@ -294,6 +298,8 @@ def test_run_total_rederives_from_actual_after_join(self, tmp_path, monkeypatch) _cost_correlation_run_id=run_id, _cost_attempt_nonce="att1", _log_task_id="calc", + # A live run, not a re-grade: the join applies (see _join_litellm_actual_cost). + prior_result=None, ) orch_mod.Orchestrator._join_litellm_actual_cost(fake) orch_mod.Orchestrator._aggregate_token_usage(fake) diff --git a/tests/test_plan_command.py b/tests/test_plan_command.py index 16a8d663..f8796fb9 100644 --- a/tests/test_plan_command.py +++ b/tests/test_plan_command.py @@ -6,7 +6,7 @@ import pytest import typer -from coder_eval.cli.plan_command import plan_command +from coder_eval.cli.plan_command import run_plan from coder_eval.models import ( AgentConfig, ExperimentDefinition, @@ -73,7 +73,7 @@ def test_plan_shows_na_when_agent_is_none(self, tmp_path: Path) -> None: patch(f"{_EXP}.resolve_task_for_variant", return_value=(resolved_task, {}, 1)), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) printed = [str(call) for call in mock_console.print.call_args_list] agent_lines = [p for p in printed if "Agent:" in p] @@ -98,7 +98,7 @@ def test_plan_shows_agent_type_when_present(self, tmp_path: Path) -> None: patch(f"{_EXP}.resolve_task_for_variant", return_value=(resolved_task, {}, 1)), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) printed = [str(call) for call in mock_console.print.call_args_list] agent_lines = [p for p in printed if "Agent:" in p] @@ -124,7 +124,7 @@ def test_plan_shows_deferred_when_agent_type_is_none(self, tmp_path: Path) -> No patch(f"{_EXP}.resolve_task_for_variant", return_value=(resolved_task, {}, 1)), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) printed = [str(call) for call in mock_console.print.call_args_list] deferred_lines = [p for p in printed if "deferred" in p] @@ -155,7 +155,7 @@ def test_plan_with_experiment_flag_shows_experiment_info(self, tmp_path: Path) - patch(f"{_EXP}.DEFAULT_EXPERIMENT_PATH", exp_file), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file], experiment=exp_file) + run_plan(task_files=[task_file], experiment=exp_file) printed = " ".join(str(call) for call in mock_console.print.call_args_list) assert "test-exp" in printed @@ -185,7 +185,7 @@ def test_plan_with_experiment_shows_resolved_agent_per_variant(self, tmp_path: P patch(f"{_EXP}.DEFAULT_EXPERIMENT_PATH", exp_file), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file], experiment=exp_file) + run_plan(task_files=[task_file], experiment=exp_file) printed = " ".join(str(call) for call in mock_console.print.call_args_list) assert "sonnet-4" in printed @@ -211,7 +211,7 @@ def test_plan_with_default_experiment(self, tmp_path: Path) -> None: patch(f"{_EXP}.resolve_task_for_variant", return_value=(resolved_task, {}, 1)), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) printed = " ".join(str(call) for call in mock_console.print.call_args_list) assert "test-exp" in printed @@ -231,7 +231,7 @@ def test_plan_warns_when_task_timeout_cannot_extend_single_iteration(self, tmp_p patch(f"{_EXP}.resolve_task_for_variant", return_value=(resolved_task, {}, 1)), patch("coder_eval.cli.plan_command.console") as mock_console, ): - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) printed = " ".join(str(call) for call in mock_console.print.call_args_list) assert "A larger task_timeout cannot extend the agent's single iteration" in printed @@ -254,7 +254,7 @@ def test_plan_exits_when_default_experiment_missing(self, tmp_path: Path) -> Non patch("coder_eval.cli.plan_command.console") as mock_console, ): with pytest.raises(typer.Exit) as exc_info: - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) assert exc_info.value.exit_code == 1 @@ -280,7 +280,7 @@ def test_plan_reports_invalid_task(self, tmp_path: Path) -> None: patch("coder_eval.cli.plan_command.console") as mock_console, ): with pytest.raises(typer.Exit) as exc_info: - plan_command(task_files=[task_file]) + run_plan(task_files=[task_file]) assert exc_info.value.exit_code == 1 @@ -305,7 +305,7 @@ def test_plan_exits_on_explicit_experiment_load_failure(self, tmp_path: Path) -> patch("coder_eval.cli.plan_command.console") as mock_console, ): with pytest.raises(typer.Exit) as exc_info: - plan_command(task_files=[task_file], experiment=exp_file) + run_plan(task_files=[task_file], experiment=exp_file) assert exc_info.value.exit_code == 1 diff --git a/tests/test_post_run.py b/tests/test_post_run.py index a55f0e14..a5b5687f 100644 --- a/tests/test_post_run.py +++ b/tests/test_post_run.py @@ -119,7 +119,7 @@ def _make_orchestrator(task: TaskDefinition, tmp_path: Path) -> Orchestrator: async def test_post_run_skipped_when_empty(tmp_path): task = _make_task(post_run=[]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -142,7 +142,7 @@ async def test_post_run_skipped_when_no_sandbox(tmp_path): async def test_post_run_command_success(tmp_path): task = _make_task(post_run=[PostRunCommand(command="echo '{\"ok\": true}'")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -161,7 +161,7 @@ async def test_post_run_command_failure_does_not_affect_result(tmp_path): ) orch = _make_orchestrator(task, tmp_path) orch.result.final_status = FinalStatus.SUCCESS - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -178,7 +178,7 @@ async def test_post_run_command_with_pipes(tmp_path): """Shell commands support pipes and redirects.""" task = _make_task(post_run=[PostRunCommand(command="echo hello world | tr a-z A-Z")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -192,7 +192,7 @@ async def test_post_run_command_with_pipes(tmp_path): async def test_post_run_command_timeout(tmp_path): task = _make_task(post_run=[PostRunCommand(command="sleep 10", timeout=1)]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -211,7 +211,7 @@ async def test_post_run_multiple_commands(tmp_path): ] ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() @@ -229,7 +229,7 @@ async def test_post_run_cwd_is_sandbox(tmp_path): task = _make_task(post_run=[PostRunCommand(command='python3 -c "import os; print(os.getcwd())"')]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = sandbox_dir await orch._run_post_run_commands() @@ -243,7 +243,7 @@ async def test_post_run_streams_stdout_to_logger(tmp_path, caplog): """Each line of stdout is forwarded to the orchestrator logger as it is read.""" task = _make_task(post_run=[PostRunCommand(command="python3 -c \"print('line-one'); print('line-two')\"")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with caplog.at_level(logging.INFO, logger="coder_eval.orchestrator"): @@ -263,7 +263,7 @@ async def test_post_run_streams_stderr_as_warning(tmp_path, caplog): """Stderr lines are forwarded at WARNING level (separate from stdout).""" task = _make_task(post_run=[PostRunCommand(command="python3 -c \"import sys; print('boom', file=sys.stderr)\"")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with caplog.at_level(logging.WARNING, logger="coder_eval.orchestrator"): @@ -279,7 +279,7 @@ async def test_post_run_output_truncated(tmp_path): # Generate output larger than the limit task = _make_task(post_run=[PostRunCommand(command="python3 -c \"print('x' * 200_000)\"")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_post_run_commands() diff --git a/tests/test_pre_run.py b/tests/test_pre_run.py index 6febaaa7..947039ed 100644 --- a/tests/test_pre_run.py +++ b/tests/test_pre_run.py @@ -150,7 +150,7 @@ def _make_orchestrator(task: TaskDefinition, tmp_path: Path) -> Orchestrator: async def test_pre_run_skipped_when_empty(tmp_path): task = _make_task(pre_run=[]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -173,7 +173,7 @@ async def test_pre_run_skipped_when_no_sandbox(tmp_path): async def test_pre_run_command_success(tmp_path): task = _make_task(pre_run=[PreRunCommand(command="echo '{\"ok\": true}'")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -189,7 +189,7 @@ async def test_pre_run_command_success(tmp_path): async def test_pre_run_command_with_pipes(tmp_path): task = _make_task(pre_run=[PreRunCommand(command="echo hello world | tr a-z A-Z")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -208,7 +208,7 @@ async def test_pre_run_multiple_commands(tmp_path): ] ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -225,7 +225,7 @@ async def test_pre_run_cwd_is_sandbox(tmp_path): task = _make_task(pre_run=[PreRunCommand(command='python3 -c "import os; print(os.getcwd())"')]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = sandbox_dir await orch._run_pre_run_commands() @@ -238,7 +238,7 @@ async def test_pre_run_cwd_is_sandbox(tmp_path): async def test_pre_run_streams_stdout_to_logger(tmp_path, caplog): task = _make_task(pre_run=[PreRunCommand(command="python3 -c \"print('line-one'); print('line-two')\"")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with caplog.at_level(logging.INFO, logger="coder_eval.orchestrator"): @@ -263,7 +263,7 @@ async def test_pre_run_streams_stderr_as_warning(tmp_path, caplog): ] ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with caplog.at_level(logging.WARNING, logger="coder_eval.orchestrator"): @@ -277,7 +277,7 @@ async def test_pre_run_streams_stderr_as_warning(tmp_path, caplog): async def test_pre_run_output_truncated(tmp_path): task = _make_task(pre_run=[PreRunCommand(command="python3 -c \"print('x' * 200_000)\"")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -296,7 +296,7 @@ async def test_pre_run_failure_raises_when_fail_on_error_true(tmp_path): pre_run=[PreRunCommand(command='python3 -c "import sys; sys.exit(1)"')], ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with pytest.raises(RuntimeError, match="Pre-run command failed"): @@ -307,7 +307,7 @@ async def test_pre_run_failure_raises_when_fail_on_error_true(tmp_path): async def test_pre_run_timeout_raises_when_fail_on_error_true(tmp_path): task = _make_task(pre_run=[PreRunCommand(command="sleep 10", timeout=1)]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with pytest.raises(RuntimeError, match="timed out after 1s"): @@ -320,7 +320,7 @@ async def test_pre_run_failure_result_captured_before_raise(tmp_path): pre_run=[PreRunCommand(command='python3 -c "import sys; sys.exit(2)"')], ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with pytest.raises(RuntimeError): @@ -339,7 +339,7 @@ async def test_pre_run_subsequent_commands_skipped_after_abort(tmp_path): ], ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with pytest.raises(RuntimeError): @@ -362,7 +362,7 @@ async def test_pre_run_failure_does_not_raise_when_fail_on_error_false(tmp_path) ], ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() @@ -376,7 +376,7 @@ async def test_pre_run_spawn_exception_raises_when_fail_on_error_true(tmp_path): """Generic exceptions from create_subprocess_shell propagate as RuntimeError when fail_on_error=True.""" task = _make_task(pre_run=[PreRunCommand(command="echo hi")]) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path with ( @@ -399,7 +399,7 @@ async def test_pre_run_spawn_exception_does_not_raise_when_fail_on_error_false(t ] ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path real_shell = __import__("asyncio").create_subprocess_shell @@ -428,7 +428,7 @@ async def test_pre_run_timeout_does_not_raise_when_fail_on_error_false(tmp_path) ], ) orch = _make_orchestrator(task, tmp_path) - orch.sandbox = AsyncMock() + orch.sandbox = AsyncMock(was_adopted=False) orch.sandbox.sandbox_dir = tmp_path await orch._run_pre_run_commands() diff --git a/tests/test_regrade.py b/tests/test_regrade.py new file mode 100644 index 00000000..be8da625 --- /dev/null +++ b/tests/test_regrade.py @@ -0,0 +1,323 @@ +"""``orchestration/regrade.py`` — the refusals, not the happy path. + +The end-to-end loop test covers a successful re-grade of the agentless task. What +it cannot cover is every branch that REFUSES to grade, and those are the ones that +matter: each exists because grading anyway would publish a plausible number that +is wrong. The reference-digest guard in particular shipped as dead code (nothing +wrote the key it read) precisely because the only test that reached it used a +fixture with no reference at all. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path + +import pytest + +from coder_eval.models import ( + AgentKind, + EvaluationResult, + FileExistsCriterion, + FinalStatus, + RunCommandCriterion, + TaskConfigRecord, + TaskDefinition, + parse_agent_config, +) +from coder_eval.orchestration.regrade import ( + RegradeError, + back_up_pre_grade_record, + default_workspace, + load_prior_result, + task_from_prior, + verify_reference_unchanged, +) +from coder_eval.path_utils import PRE_GRADE_JSON_FILENAME, TASK_JSON_FILENAME, digest_tree + + +def _task(*, reference: dict[str, str] | None = None, command: str | None = None) -> TaskDefinition: + criteria: list[object] = [FileExistsCriterion(path="x.txt", description="x")] + if command is not None: + criteria.append(RunCommandCriterion(command=command, description="run it")) + return TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + reference=reference, # type: ignore[arg-type] + success_criteria=criteria, # type: ignore[arg-type] + ) + + +def _result(**kwargs: object) -> EvaluationResult: + from datetime import datetime + + base: dict[str, object] = { + "task_id": "t", + "task_description": "d", + "variant_id": "v", + "agent_type": AgentKind.CLAUDE_CODE, + "started_at": datetime(2020, 1, 1), + "final_status": FinalStatus.NOT_GRADED, + "iteration_count": 1, + } + base.update(kwargs) + return EvaluationResult(**base) # type: ignore[arg-type] + + +# -------------------------------------------------------------------------- +# load_prior_result +# -------------------------------------------------------------------------- + + +def test_missing_task_json_is_a_regrade_error(tmp_path: Path) -> None: + with pytest.raises(RegradeError, match="Cannot read"): + load_prior_result(tmp_path) + + +def test_unparseable_task_json_is_a_regrade_error(tmp_path: Path) -> None: + (tmp_path / TASK_JSON_FILENAME).write_text("{not json", encoding="utf-8") + with pytest.raises(RegradeError, match="not a readable EvaluationResult"): + load_prior_result(tmp_path) + + +# -------------------------------------------------------------------------- +# task_from_prior — which task gets graded +# -------------------------------------------------------------------------- + + +def test_no_task_config_refuses_rather_than_guessing(tmp_path: Path) -> None: + with pytest.raises(RegradeError, match="carries no task_config"): + task_from_prior(_result(), tmp_path) + + +def test_resolved_config_wins_over_the_source_yaml(tmp_path: Path) -> None: + """`resolved` is post-merge, so it carries variant overrides / -D / dataset + expansion. Re-reading the YAML would grade a DIFFERENT task.""" + source = tmp_path / "t.yaml" + source.write_text("task_id: from-yaml\n", encoding="utf-8") + resolved = _task().model_dump(mode="json") + resolved["task_id"] = "from-resolved" + prior = _result(task_config=TaskConfigRecord(resolved=resolved, source_yaml="raw", source_file=str(source))) + + task, _ = task_from_prior(prior, tmp_path) + + assert task.task_id == "from-resolved" + + +def test_unusable_resolved_config_with_no_source_refuses(tmp_path: Path) -> None: + prior = _result(task_config=TaskConfigRecord(resolved={"nonsense": True}, source_yaml="raw", source_file=None)) + with pytest.raises(RegradeError, match="no longer validates"): + task_from_prior(prior, tmp_path) + + +def test_source_fallback_is_loud(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + """A quiet fallback would silently grade a task other than the one that ran.""" + source = tmp_path / "t.yaml" + source.write_text( + "task_id: from-yaml\ndescription: d\ninitial_prompt: p\n" + + "success_criteria:\n - type: file_exists\n path: x.txt\n description: x\n", + encoding="utf-8", + ) + prior = _result( + task_config=TaskConfigRecord(resolved={"nonsense": True}, source_yaml="raw", source_file=str(source)) + ) + + with caplog.at_level(logging.WARNING): + task, _ = task_from_prior(prior, tmp_path) + + assert task.task_id == "from-yaml" + assert "NOT reapplied" in caplog.text + + +def test_shell_commands_from_a_run_dir_config_are_refused_by_default(tmp_path: Path) -> None: + """A run dir is a shareable artifact, and rebuilding from it decides what the + grader executes with the grader's credentials. + + A warning is not a control — it is printed as the command is already being + prepared. So the default is REFUSAL, and the message names both the commands + and the way to accept them.""" + resolved = _task(command="echo surprising").model_dump(mode="json") + prior = _result(task_config=TaskConfigRecord(resolved=resolved, source_yaml="raw", source_file=None)) + + with pytest.raises(RegradeError) as exc: + task_from_prior(prior, tmp_path) + + assert "echo surprising" in str(exc.value) + assert "--allow-recorded-commands" in str(exc.value) + + +def test_the_opt_in_accepts_them_and_still_names_them(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + """Accepted is not the same as invisible: the opt-in still logs what will run.""" + resolved = _task(command="echo surprising").model_dump(mode="json") + prior = _result(task_config=TaskConfigRecord(resolved=resolved, source_yaml="raw", source_file=None)) + + with caplog.at_level(logging.WARNING): + task, _ = task_from_prior(prior, tmp_path, allow_recorded_commands=True) + + assert task.task_id + assert "echo surprising" in caplog.text + + +def test_a_config_with_no_shell_needs_no_opt_in(tmp_path: Path) -> None: + """The common case — execute then evaluate your own file/JSON criteria — is + unaffected, or the gate would just be turned off.""" + resolved = _task().model_dump(mode="json") + prior = _result(task_config=TaskConfigRecord(resolved=resolved, source_yaml="raw", source_file=None)) + + task, _ = task_from_prior(prior, tmp_path) + assert task.task_id + + +# -------------------------------------------------------------------------- +# default_workspace +# -------------------------------------------------------------------------- + + +def test_recorded_sandbox_path_wins_when_it_still_exists(tmp_path: Path) -> None: + workspace = tmp_path / "ws" + workspace.mkdir() + assert default_workspace(tmp_path, _result(sandbox_path=str(workspace))) == workspace + + +def test_falls_back_to_the_single_artifacts_child(tmp_path: Path) -> None: + child = tmp_path / "artifacts" / "t" + child.mkdir(parents=True) + assert default_workspace(tmp_path, _result(sandbox_path="/gone")) == child + + +def test_flat_artifacts_dir_is_itself_the_workspace(tmp_path: Path) -> None: + artifacts = tmp_path / "artifacts" + artifacts.mkdir() + (artifacts / "file.txt").write_text("x", encoding="utf-8") + assert default_workspace(tmp_path, _result()) == artifacts + + +def test_no_workspace_at_all_refuses(tmp_path: Path) -> None: + with pytest.raises(RegradeError, match="No workspace to grade"): + default_workspace(tmp_path, _result()) + + +# -------------------------------------------------------------------------- +# verify_reference_unchanged — the anti-cheat guard +# -------------------------------------------------------------------------- + + +def _reference_task(tmp_path: Path) -> tuple[TaskDefinition, Path, Path]: + task_file = tmp_path / "t.yaml" + task_file.write_text("x", encoding="utf-8") + reference = tmp_path / "ref" + reference.mkdir() + (reference / "answer.py").write_text("print('right')\n", encoding="utf-8") + return _task(reference={"directory": "ref"}), task_file, reference + + +def test_an_edited_reference_refuses_the_grade(tmp_path: Path) -> None: + """The headline guarantee. Without it, an answer key edited between execute + and grade scores the agent's old work against a new one.""" + task, task_file, reference = _reference_task(tmp_path) + prior = _result(environment_info={"reference_digest": digest_tree(reference)}) + verify_reference_unchanged(prior, task, task_file) # unchanged: fine + + (reference / "answer.py").write_text("print('different')\n", encoding="utf-8") + + with pytest.raises(RegradeError, match="digest mismatch"): + verify_reference_unchanged(prior, task, task_file) + + +def test_a_vanished_reference_refuses_rather_than_grading_without_one(tmp_path: Path) -> None: + task, task_file, reference = _reference_task(tmp_path) + prior = _result(environment_info={"reference_digest": digest_tree(reference)}) + for p in reference.iterdir(): + p.unlink() + reference.rmdir() + + with pytest.raises(RegradeError): + verify_reference_unchanged(prior, task, task_file) + + +def test_a_run_without_a_recorded_digest_says_so(tmp_path: Path, caplog: pytest.LogCaptureFixture) -> None: + """Silence here is what let the guard ship as dead code for a release.""" + task, task_file, _ = _reference_task(tmp_path) + + with caplog.at_level(logging.WARNING): + verify_reference_unchanged(_result(), task, task_file) + + assert "cannot be verified" in caplog.text + + +def test_a_task_with_no_reference_is_not_checked(tmp_path: Path) -> None: + verify_reference_unchanged(_result(), _task(), tmp_path / "t.yaml") + + +# -------------------------------------------------------------------------- +# back_up_pre_grade_record +# -------------------------------------------------------------------------- + + +def test_the_pre_grade_record_is_written_once(tmp_path: Path) -> None: + """A second grade must not overwrite the ORIGINAL execute record with an + already-graded one — that is the only evidence the run was ungraded.""" + (tmp_path / TASK_JSON_FILENAME).write_text('{"round": 1}', encoding="utf-8") + back_up_pre_grade_record(tmp_path) + (tmp_path / TASK_JSON_FILENAME).write_text('{"round": 2}', encoding="utf-8") + back_up_pre_grade_record(tmp_path) + + assert json.loads((tmp_path / PRE_GRADE_JSON_FILENAME).read_text(encoding="utf-8")) == {"round": 1} + + +def test_backup_is_a_no_op_with_nothing_to_back_up(tmp_path: Path) -> None: + back_up_pre_grade_record(tmp_path) + assert not (tmp_path / PRE_GRADE_JSON_FILENAME).exists() + + +def test_a_failed_backup_never_fails_the_grade(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """The audit copy is a convenience; the verdict is the deliverable.""" + (tmp_path / TASK_JSON_FILENAME).write_text("{}", encoding="utf-8") + + def _boom(*_args: object, **_kwargs: object) -> None: + raise OSError("read-only file system") + + monkeypatch.setattr(Path, "write_text", _boom) + back_up_pre_grade_record(tmp_path) # must not raise + + +# -------------------------------------------------------------------------- +# Dataset-row ids contain "/" +# -------------------------------------------------------------------------- + + +def test_a_dataset_row_workspace_resolves_by_task_id_not_by_child_count(tmp_path: Path) -> None: + """Preservation writes artifacts/, and a dataset row's task_id is + "/". "The single child of artifacts/" therefore resolves to + artifacts/ — one level too high — and every path-relative criterion + then fails as a locating artifact rather than as a verdict.""" + workspace = tmp_path / "artifacts" / "suite" / "row-1" + workspace.mkdir(parents=True) + + resolved = default_workspace(tmp_path, _result(task_id="suite/row-1")) + + assert resolved == workspace + + +def test_an_ambiguous_artifacts_dir_refuses_rather_than_guessing(tmp_path: Path) -> None: + artifacts = tmp_path / "artifacts" + (artifacts / "a").mkdir(parents=True) + (artifacts / "b").mkdir() + + with pytest.raises(RegradeError, match="Pass --workspace"): + default_workspace(tmp_path, _result(task_id="neither")) + + +def test_a_sandbox_path_outside_the_run_dir_is_refused(tmp_path: Path) -> None: + """`sandbox_path` is an unvalidated absolute path out of the run's own + task.json, and criteria execute with cwd there and may mutate it.""" + outside = tmp_path / "elsewhere" + outside.mkdir() + run_dir = tmp_path / "run" + run_dir.mkdir() + + with pytest.raises(RegradeError, match="outside the run directory"): + default_workspace(run_dir, _result(sandbox_path=str(outside))) diff --git a/tests/test_reports.py b/tests/test_reports.py index 17253769..1dc85ae2 100644 --- a/tests/test_reports.py +++ b/tests/test_reports.py @@ -1307,3 +1307,60 @@ def test_generate_markdown_renders_both_markers_when_both_fire(): assert "max_turns exhausted" in report_md assert "expected_turns exceeded" in report_md assert "7/5" in report_md + + +class TestAnUnmeasuredRunPublishesNoRate: + """`tasks_graded` is the complement of the ungraded bucket, so it keeps ERROR + rows — right under `run`, where an errored row was attempted and genuinely + missed. Under `coder-eval execute` no criterion runs on ANY row, yet a + crashed one lands in the `error` bucket, so it stayed in the denominator on + its own: a 100-task execute night with 5 crashes published `pass_rate 0.0` + and `error_share 1.0`, a measured-looking total failure for a run that was + never measured at all. + """ + + @staticmethod + def _summary(*, succeeded: int, failed: int, error: int, not_graded: int) -> RunSummary: + return RunSummary( + run_id="2026-01-01_00-00-00", + start_time=datetime(2026, 1, 1), + end_time=datetime(2026, 1, 1), + total_duration_seconds=0.0, + tasks_run=succeeded + failed + error + not_graded, + tasks_succeeded=succeeded, + tasks_failed=failed, + tasks_error=error, + tasks_not_graded=not_graded, + task_results=[], + framework_version="0.1.0", + environment_info={}, + ) + + def test_an_execute_night_with_a_crash_reports_no_rate(self): + summary = self._summary(succeeded=0, failed=0, error=5, not_graded=95) + + assert summary.pass_rate is None + assert summary.error_share is None + + def test_the_markdown_says_so_rather_than_rendering_zero_percent(self): + summary = self._summary(succeeded=0, failed=0, error=5, not_graded=95) + + md = ReportGenerator.generate_markdown(summary) + + assert "n/a" in md + assert "0.0%" not in md, "a never-measured run rendered a measured-looking 0.0% pass rate" + + def test_a_partly_graded_run_still_reports_a_rate(self): + """The carve-out is only for "nothing was measured". Once ANY row reached + a verdict, an errored row is a genuine miss and belongs in the + denominator — the erroring-bonus this rule exists to prevent.""" + summary = self._summary(succeeded=3, failed=1, error=1, not_graded=5) + + assert summary.pass_rate == 3 / 5 + assert summary.error_share == 1 / 5 + + def test_an_ordinary_graded_run_is_untouched(self): + summary = self._summary(succeeded=2, failed=1, error=1, not_graded=0) + + assert summary.pass_rate == 0.5 + assert summary.error_share == 0.25 diff --git a/tests/test_reports_html.py b/tests/test_reports_html.py index 77e5886a..b468d8a0 100644 --- a/tests/test_reports_html.py +++ b/tests/test_reports_html.py @@ -37,16 +37,21 @@ ) -_CATEGORY_TO_CLASS = {"succeeded": "success", "failed": "failure", "error": "error"} +# "ungraded" is the one category whose badge is legitimately neutral: the row +# carries no verdict, so it must render as neither green nor red. Listing it +# explicitly (rather than dropping the negative assertion below) keeps the guard +# that no OTHER member falls through to the neutral fallback. +_CATEGORY_TO_CLASS = {"succeeded": "success", "failed": "failure", "error": "error", "ungraded": "neutral"} @pytest.mark.parametrize("status", list(FinalStatus)) def test_status_badge_dispatches_on_category(status: FinalStatus): - """Every FinalStatus member renders a non-neutral, category-correct badge.""" + """Every FinalStatus member renders a category-correct badge, neutral only when intended.""" badge = _status_badge(status) expected_cls = _CATEGORY_TO_CLASS[status.category] assert f'class="badge {expected_cls}"' in badge - assert "neutral" not in badge + if expected_cls != "neutral": + assert "neutral" not in badge # The human-readable label is the status value. assert status.value in badge @@ -1229,16 +1234,20 @@ def test_generation_metrics_breaks_down_crashed_partials(): FinalStatus.MAX_TURNS_EXHAUSTED: "failure", FinalStatus.TOKEN_BUDGET_EXCEEDED: "failure", FinalStatus.COST_BUDGET_EXCEEDED: "failure", + # Neutral on purpose — an ungraded row has no verdict to colour. See + # _CATEGORY_TO_CLASS above. + FinalStatus.NOT_GRADED: "neutral", } @pytest.mark.parametrize("status", list(FinalStatus)) def test_status_badge_maps_every_member_to_its_category(status: FinalStatus): - """Every FinalStatus member gets a non-neutral badge matching its category.""" + """Every FinalStatus member gets a badge matching its category, neutral only when intended.""" badge = _status_badge(status) expected_cls = _EXPECTED_BADGE_CLASS[status] assert f'class="badge {expected_cls}"' in badge - assert "neutral" not in badge + if expected_cls != "neutral": + assert "neutral" not in badge assert status.value in badge @@ -1252,3 +1261,79 @@ def test_status_badge_unknown_string_is_neutral(): badge = _status_badge("LEGACY_UNKNOWN") assert 'class="badge neutral"' in badge assert "LEGACY_UNKNOWN" in badge + + +class TestUngradedRenderingInHtml: + """The four ungraded changes to reports_html, only one of which had a test. + + An all-ungraded ExperimentResult exercises the variant tile, the variant + table's denominator, `_variant_stddev_lines`' None filter and the per-task + comparison's switch to `format_score` in one pass. + """ + + @staticmethod + def _all_ungraded(variant_ids: list[str]) -> ExperimentResult: + summaries = [ + TaskExperimentSummary( + task_id="t0", + variant_results=[ + VariantResult( + variant_id=vid, + task_id="t0", + weighted_score=None, + final_status=FinalStatus.NOT_GRADED, + duration_seconds=1.0, + total_tokens=100, + iteration_count=1, + total_assistant_turns=1, + ) + for vid in variant_ids + ], + best_variant=variant_ids[0], + is_tie=True, + score_spread=0.0, + ) + ] + aggregates = { + vid: VariantAggregate( + variant_id=vid, + tasks_run=3, + tasks_succeeded=0, + tasks_failed=0, + tasks_error=0, + tasks_not_graded=3, + average_score=None, + average_duration=12.0, + ) + for vid in variant_ids + } + return ExperimentResult( + experiment_id="exp-ungraded", + description="d", + variant_ids=variant_ids, + task_summaries=summaries, + variant_aggregates=aggregates, + total_duration_seconds=30.0, + ) + + def test_the_variant_tile_names_the_fourth_bucket(self): + """Rendered only when non-zero, so an off-by-one on `> 0` is invisible on + a graded run — this is the only place it shows.""" + html = HTMLReportGenerator.generate_experiment_html(self._all_ungraded(["v1"]), None) + assert "Not Graded" in html + + def test_a_graded_run_does_not_grow_the_tile(self): + html = HTMLReportGenerator.generate_experiment_html(_experiment_result(["v1"]), None) + assert "Not Graded" not in html + + def test_the_aggregate_table_accounts_for_every_task(self): + """Tasks Run / Succeeded / Failed / Errors stopped summing to tasks_run + on an ungraded run, with nothing on the page to say where the rest went.""" + html = HTMLReportGenerator.generate_experiment_html(self._all_ungraded(["v1", "v2"]), None) + assert "Not Graded" in html + + def test_no_surface_publishes_a_fabricated_zero(self): + """A run nothing measured must read `n/a`, never `0.0` or `0.0%`.""" + html = HTMLReportGenerator.generate_experiment_html(self._all_ungraded(["v1"]), None) + assert "n/a" in html + assert "0.0%" not in html diff --git a/tests/test_resume.py b/tests/test_resume.py index 6f46084c..54bf8bbe 100644 --- a/tests/test_resume.py +++ b/tests/test_resume.py @@ -56,9 +56,19 @@ def _resolved(run_root, task_id: str) -> ResolvedTask: ) -def _write_task_json(rt: ResolvedTask, status: FinalStatus) -> None: - """Write a finalized task.json into rt.run_dir, like the orchestrator does.""" +def _write_task_json(rt: ResolvedTask, status: FinalStatus, *, graded: bool | None = None) -> None: + """Write a finalized task.json into rt.run_dir, like the orchestrator does. + + ``graded`` decides whether the row carries a verdict, and it is not the same + question as ``status``. That is the whole point of the resume partition: a + row is owed a grade when it was executed but never scored, and an ``execute`` + row that also tripped a run limit lands with an execution-fact status AND no + score. Defaulting it from the status alone would make the fixture unable to + express the case the partition exists to route. + """ rt.run_dir.mkdir(parents=True, exist_ok=True) + if graded is None: + graded = status.category != "ungraded" result = EvaluationResult( task_id=rt.task.task_id, task_description=rt.task.description, @@ -66,7 +76,9 @@ def _write_task_json(rt: ResolvedTask, status: FinalStatus) -> None: agent_type=AgentKind.CLAUDE_CODE, started_at=datetime.now(), final_status=status, - weighted_score=1.0 if status == FinalStatus.SUCCESS else 0.0, + # None, never 0.0, when nothing was measured: a laundered zero here would + # make the fixture indistinguishable from a genuinely-scored miss. + weighted_score=(1.0 if status == FinalStatus.SUCCESS else 0.0) if graded else None, duration_seconds=12.5, iteration_count=1, environment_info={}, @@ -84,7 +96,7 @@ def test_partition_splits_finalized_from_pending(tmp_path): partial.run_dir.mkdir(parents=True, exist_ok=True) (partial.run_dir / "task.json").write_text(json.dumps({"task_id": "partial_task"}), encoding="utf-8") - to_run, prior_results, prior_resolved = partition_for_resume([done, pending, partial]) + to_run, _to_grade, prior_results, prior_resolved = partition_for_resume([done, pending, partial]) assert {rt.task.task_id for rt in to_run} == {"pending_task", "partial_task"} assert [tr.task_id for tr in prior_results] == ["done_task"] @@ -94,10 +106,102 @@ def test_partition_splits_finalized_from_pending(tmp_path): assert prior_results[0].duration == 12.5 +def test_partition_sends_ungraded_rows_to_grading_not_to_rerun(tmp_path): + """`run --resume` owes a NOT_GRADED row a GRADE, not another agent run. + + A NOT_GRADED row (written by `coder-eval execute`) carries a final status, so + the naive "has any final status" test calls it complete — which made + `run --resume` report "already complete", grade nothing, and exit 0. + """ + ungraded = _resolved(tmp_path, "ungraded_task") + graded = _resolved(tmp_path, "graded_task") + _write_task_json(ungraded, FinalStatus.NOT_GRADED) + _write_task_json(graded, FinalStatus.SUCCESS) + + part = partition_for_resume([ungraded, graded], grade=True) + + assert [rt.task.task_id for rt in part.to_grade] == ["ungraded_task"] + assert part.to_run == [], "an executed row must not be re-executed — that discards the agent spend" + assert [tr.task_id for tr in part.prior_results] == ["graded_task"] + + +def test_partition_treats_ungraded_as_done_for_execute(tmp_path): + """`execute --resume` owes a NOT_GRADED row nothing: it finished executing.""" + ungraded = _resolved(tmp_path, "ungraded_task") + _write_task_json(ungraded, FinalStatus.NOT_GRADED) + + part = partition_for_resume([ungraded], grade=False) + + assert part.to_grade == [] + assert part.to_run == [] + assert [tr.task_id for tr in part.prior_results] == ["ungraded_task"] + + +@pytest.mark.parametrize( + "status", + [FinalStatus.TIMEOUT, FinalStatus.TOKEN_BUDGET_EXCEEDED, FinalStatus.COST_BUDGET_EXCEEDED], +) +def test_an_execute_row_that_also_tripped_a_run_limit_still_owes_a_grade(tmp_path, status): + """The case routing-by-category missed entirely. + + A run limit aborts the run BEFORE grading, so under `coder-eval execute` such + a row lands with an execution-fact status — category `error`/`failed`, not + `ungraded` — and no verdict at all. Keying the partition on the category + filed it under "already complete", so it stayed permanently `weighted_score: + null` with an empty criteria vector in run.json, the rollup and the + evalboard — while `evaluate ` graded the identical bytes happily. + Two entry points into one feature disagreeing about the same record. + """ + row = _resolved(tmp_path, "limited_task") + _write_task_json(row, status, graded=False) + + part = partition_for_resume([row], grade=True) + + assert [rt.task.task_id for rt in part.to_grade] == ["limited_task"] + assert part.to_run == [], "the agent already ran; re-running it discards that spend" + + +@pytest.mark.parametrize("status", [FinalStatus.FAILURE, FinalStatus.ERROR, FinalStatus.TIMEOUT]) +def test_partition_still_never_retries_failures(tmp_path, status): + """The carve-out must not become a general 'retry bad rows' rule. + + Resume has never retried failures (delete a task's task.json to force that), + and both commands must keep treating a row that CARRIES a verdict as + complete — which is why these rows are written graded. + """ + failed = _resolved(tmp_path, "failed_task") + _write_task_json(failed, status, graded=True) + + for grade in (True, False): + part = partition_for_resume([failed], grade=grade) + assert part.to_run == [], f"grade={grade} re-ran a {status.value} row" + assert part.to_grade == [], f"grade={grade} tried to re-grade a {status.value} row" + assert len(part.prior_results) == 1 + + +def test_grade_flag_is_exempt_from_the_resume_drift_warning(tmp_path): + """`execute` then `run --resume` is a supported flow, not a config mistake. + + The warning's text ("already-finalized tasks keep their original-config + results") is actively wrong for it: those rows are re-graded with the current + config, which is the whole point. + """ + executed = BatchRunConfig(run_dir=tmp_path, grade=False) + write_run_fingerprint(tmp_path, compute_run_fingerprint(executed, "exp1", "direct", None)) + prior = read_run_fingerprint(tmp_path) + + grading = BatchRunConfig(run_dir=tmp_path, grade=True) + assert fingerprint_diff(prior, compute_run_fingerprint(grading, "exp1", "direct", None)) == {} + + # ...but a real difference alongside it is still reported. + both = BatchRunConfig(run_dir=tmp_path, grade=True, overrides={"agent.model": "opus"}) + assert "overrides" in fingerprint_diff(prior, compute_run_fingerprint(both, "exp1", "direct", None)) + + def test_partition_no_run_dir_yields_all_pending(tmp_path): """--resume on a fresh dir degrades to a normal run (everything to_run).""" tasks = [_resolved(tmp_path, f"t{i}") for i in range(3)] - to_run, prior_results, prior_resolved = partition_for_resume(tasks) + to_run, _to_grade, prior_results, prior_resolved = partition_for_resume(tasks) assert len(to_run) == 3 assert prior_results == [] assert prior_resolved == [] @@ -114,7 +218,7 @@ async def test_run_batch_folds_prior_into_run_json(tmp_path): _write_task_json(p_success, FinalStatus.SUCCESS) _write_task_json(p_fail, FinalStatus.FAILURE) - _, prior_results, prior_resolved = partition_for_resume([p_success, p_fail]) + _, _to_grade, prior_results, prior_resolved = partition_for_resume([p_success, p_fail]) assert len(prior_results) == 2 # Nothing left to run — exercises the merge + run.json write in isolation. diff --git a/tests/test_route_seam_exhaustiveness.py b/tests/test_route_seam_exhaustiveness.py index 3d04d762..422ee8ea 100644 --- a/tests/test_route_seam_exhaustiveness.py +++ b/tests/test_route_seam_exhaustiveness.py @@ -92,8 +92,16 @@ def test_record_route_environment_info_handles_every_route(): pyright can't check the isinstance chain. Also covers the simulator_route -> simulator_routing seam directly (not just via a shared alias with route).""" for r in _INSTANCES: + # prior_result=None: a detached grade takes a different branch that + # records the GRADING host's route under graded_by_* instead of + # overwriting the run's. This test covers the ordinary run. fake = SimpleNamespace( - route=r, eval_route=r, simulator_route=r, result=SimpleNamespace(environment_info={}), agent=None + route=r, + eval_route=r, + simulator_route=r, + result=SimpleNamespace(environment_info={}), + agent=None, + prior_result=None, ) Orchestrator._record_route_environment_info(fake) # type: ignore[arg-type] env = fake.result.environment_info diff --git a/tests/test_run_command_junit.py b/tests/test_run_command_junit.py index b5ac1de0..7f8e5341 100644 --- a/tests/test_run_command_junit.py +++ b/tests/test_run_command_junit.py @@ -26,7 +26,7 @@ async def _invoke( status: str, failed: bool, ) -> None: - summary = Mock(tasks_failed=1 if failed else 0, tasks_error=0) + summary = Mock(tasks_failed=1 if failed else 0, tasks_error=0, tasks_not_graded=0) async def _fake(*_args, **_kwargs): # Mirror production: run.json is persisted inside _run_with_experiment. diff --git a/tests/test_run_metrics.py b/tests/test_run_metrics.py index 6ac5baff..6866a87f 100644 --- a/tests/test_run_metrics.py +++ b/tests/test_run_metrics.py @@ -5,7 +5,9 @@ **The denominator.** ``pass_rate`` used to be ``succeeded / (run - error)``, which paid a bonus for erroring: the more a run fell over, the smaller its denominator got, up to the degenerate case of a run rendering as a perfect score while passing -a handful of rows. Every surface now divides by ``tasks_run``. +a handful of rows. Every surface now divides by ``tasks_graded`` — every +dispatched task except the ones that were never measured at all (``coder-eval +execute`` leaves rows ``NOT_GRADED``, and those leave BOTH sides of the rate). **The bill.** Cost was summed over whatever rows happened to carry one, so a run whose model was missing from the rate card, or whose turns were killed before the diff --git a/tests/test_sandbox_adopt.py b/tests/test_sandbox_adopt.py new file mode 100644 index 00000000..39fafd9b --- /dev/null +++ b/tests/test_sandbox_adopt.py @@ -0,0 +1,108 @@ +"""`Sandbox.adopt` — grade a workspace in place instead of copying it. + +The behavior that matters is what adopt does NOT do. `setup()` materializes a +workspace (copies templates in, writes shims, builds a venv); `adopt()` takes one +that already exists and only derives the environment around it. A regression that +made adopt materialize anything would overwrite the very files being graded, and +would do so silently — the criteria would still run, just against different +content. So each assertion below pins one thing staying untouched. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from coder_eval.models import SandboxConfig +from coder_eval.sandbox import Sandbox + + +def _workspace(tmp_path: Path) -> Path: + """A workspace shaped like an agent's output: real files plus build output.""" + ws = tmp_path / "ws" + (ws / "node_modules" / "pkg").mkdir(parents=True) + (ws / "dist").mkdir() + (ws / "src.py").write_text("print('hi')", encoding="utf-8") + (ws / "node_modules" / "pkg" / "index.js").write_text("module.exports=1", encoding="utf-8") + (ws / "dist" / "bundle.js").write_text("bundled", encoding="utf-8") + return ws + + +def _sandbox(**cfg: object) -> Sandbox: + return Sandbox(SandboxConfig(**cfg), task_id="t") # type: ignore[arg-type] + + +def test_adopt_uses_the_directory_itself(tmp_path: Path) -> None: + ws = _workspace(tmp_path) + sandbox = _sandbox() + assert sandbox.adopt(ws) == ws.resolve() + assert sandbox.sandbox_dir == ws.resolve(), "adopt must not create a copy" + + +def test_adopt_exposes_files_the_copy_path_filters_out(tmp_path: Path) -> None: + """The bug this fixes. `_should_ignore_template_file` drops node_modules, + dist, build and .venv, so on the copy path a criterion like + `test -f dist/bundle.js` fails as a COPYING artifact rather than as a + verdict on the agent's work.""" + ws = _workspace(tmp_path) + sandbox = _sandbox() + sandbox.adopt(ws) + assert sandbox.sandbox_dir is not None + assert (sandbox.sandbox_dir / "dist" / "bundle.js").is_file() + assert (sandbox.sandbox_dir / "node_modules" / "pkg" / "index.js").is_file() + + +def test_adopt_writes_nothing_into_the_workspace(tmp_path: Path) -> None: + """No shims, no venv, no template files — the graded tree is exactly as found.""" + ws = _workspace(tmp_path) + before = {p.relative_to(ws) for p in ws.rglob("*")} + _sandbox().adopt(ws) + assert {p.relative_to(ws) for p in ws.rglob("*")} == before + + +def test_adopt_never_owns_the_directory(tmp_path: Path) -> None: + """cleanup() must not delete a directory the caller handed us.""" + ws = _workspace(tmp_path) + sandbox = _sandbox() + sandbox.adopt(ws) + assert sandbox.is_persistent + sandbox.cleanup() + assert ws.is_dir(), "cleanup deleted an adopted workspace" + assert (ws / "src.py").is_file() + + +def test_adopt_discovers_an_existing_venv_without_creating_one(tmp_path: Path) -> None: + """The execute phase already built the venv; re-creating it would mutate the + graded tree. Discovery keeps `run_command` criteria on the agent's PATH.""" + ws = _workspace(tmp_path) + (ws / ".venv" / "bin").mkdir(parents=True) + sandbox = _sandbox(python={"env_packages": []}) + sandbox.adopt(ws) + assert sandbox.venv_dir == ws.resolve() / ".venv" + + +def test_adopt_leaves_venv_unset_when_there_is_none(tmp_path: Path) -> None: + sandbox = _sandbox(python={"env_packages": []}) + sandbox.adopt(_workspace(tmp_path)) + assert sandbox.venv_dir is None + + +def test_adopt_rejects_the_docker_driver(tmp_path: Path) -> None: + """A container workspace is not reachable from the host, so adopting one + would silently grade whatever happens to sit at that host path.""" + sandbox = _sandbox(driver="docker") + with pytest.raises(RuntimeError, match="host-side only"): + sandbox.adopt(_workspace(tmp_path)) + + +def test_adopt_rejects_a_missing_directory(tmp_path: Path) -> None: + with pytest.raises(RuntimeError, match="not an existing directory"): + _sandbox().adopt(tmp_path / "nope") + + +def test_adopt_rejects_a_file(tmp_path: Path) -> None: + f = tmp_path / "a.txt" + f.write_text("x", encoding="utf-8") + with pytest.raises(RuntimeError, match="not an existing directory"): + _sandbox().adopt(f) diff --git a/tests/test_seed_from_prior_result.py b/tests/test_seed_from_prior_result.py new file mode 100644 index 00000000..6d88c05e --- /dev/null +++ b/tests/test_seed_from_prior_result.py @@ -0,0 +1,300 @@ +"""``Orchestrator._seed_from_prior_result`` — the detached grade's fidelity contract. + +A detached grade (``evaluate `` / ``run --resume``) recomputes the +verdict but must not recompute the RUN. Every field it carries over is a fact the +agent phase established and this pass cannot re-derive; every field it does not +carry is either recomputed from the trajectory or deliberately dropped. + +The end-to-end tests in ``test_execute_evaluate_loop.py`` exercise this through a +fixture where most of these fields hold their defaults, so deleting a carry line +leaves them green. These tests set every field to a distinctive value instead, and +the partition below fails closed when a new field is added to ``EvaluationResult`` +without a decision about it. +""" + +from __future__ import annotations + +from datetime import datetime +from pathlib import Path + +import pytest + +from coder_eval.models import ( + AgentKind, + CommandExecutedCriterion, + CriterionResult, + EarlyStopInfo, + EarlyStopReason, + EvaluationResult, + FileExistsCriterion, + FinalStatus, + PostRunResult, + SimulationTelemetry, + TaskDefinition, + TurnRecord, + parse_agent_config, +) +from coder_eval.orchestrator import Orchestrator + + +# Every field on EvaluationResult, partitioned by what a detached grade does with +# it. No catch-all: a new field fails the parity test below until it is listed, +# which is the same fail-closed shape as _STATUS_CATEGORIES in models/enums.py. +CARRIED = { + "started_at", + "iterations", + "iteration_count", + "early_stop", + "max_turns_exhausted", + "error_message", + "error_details", + "error_log_tail", + "sdk_options", + "agent_config", + "expected_commands", + "simulation", + "pre_run_results", + "post_run_results", + "sandbox_path", + "environment_info", +} + +# Recomputed by this pass — carrying them would defeat the point. +RECOMPUTED = { + # The verdict itself: exactly what the grading pass produces. + "final_status", + "weighted_score", + "success_criteria_results", + "post_failure_criteria_results", + # Derived from `iterations`, which IS carried — so seeding the trajectory + # reproduces these exactly without copying them. + "model_used", + "command_stats", + "total_token_usage", + "total_assistant_turns", + "actual_commands", + "commands_efficiency", + # Identity, supplied by the caller from the task being graded. + "task_id", + "task_description", + "variant_id", + "agent_type", + "task_config", + # Timing: `duration_seconds` is restored from the prior result in + # _finalize_result (after its own timing write), and `completed_at` marks + # when the row reached its final state, which the grade genuinely changes. + "duration_seconds", + "completed_at", +} + + +def _prior() -> EvaluationResult: + """A prior result with a distinctive value in every carried field.""" + return EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2020, 1, 1, 0, 0, 0), + final_status=FinalStatus.NOT_GRADED, + iteration_count=7, + # Distinctive, NOT the model default. `iterations` and `simulation` were + # left at `[]` / `None`, so their parametrized sensors asserted `[] == []` + # and `None == None` — verified by deleting the `simulation` carry line in + # orchestrator.py and getting a byte-identical green suite, i.e. nothing + # anywhere guarded it and a re-graded simulation run would silently lose + # its dialog record. + iterations=[TurnRecord(iteration=1, user_input="prior prompt", agent_output="prior reply")], + simulation=SimulationTelemetry(n_trials=3, replicate_index=2, stop_reason="stop_token", total_turns=4), + max_turns_exhausted=True, + error_message="prior message", + error_details={"where": "prior"}, + error_log_tail="prior tail", + sdk_options={"opt": "prior"}, + agent_config=parse_agent_config(type=AgentKind.CLAUDE_CODE, model="prior-model"), + expected_commands=11, + pre_run_results=[PostRunResult(command="prior-pre", exit_code=0)], + post_run_results=[PostRunResult(command="prior-post", exit_code=0)], + sandbox_path="/prior/workspace", + environment_info={"installed_tools": "prior", "coder_eval": "1.0.0-run"}, + early_stop=EarlyStopInfo( + reason=EarlyStopReason.CRITERION_FAILED, + deciding_criterion_type="skill_triggered", + deciding_criterion_description="the armed criterion", + sdk_turn_index=0, + tool_call_index=1, + elapsed_seconds=1.0, + gate_threshold=1.0, + ), + ) + + +def _seeded(tmp_path: Path) -> tuple[Orchestrator, EvaluationResult]: + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + prior = _prior() + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v", prior_result=prior) + orch.result = EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2030, 1, 1, 0, 0, 0), + final_status=FinalStatus.FAILURE, + iteration_count=0, + environment_info={"installed_tools": "grader", "coder_eval": "9.9.9-grader"}, + ) + orch._seed_from_prior_result() + assert orch.result is not None + return orch, prior + + +def test_field_partition_covers_every_evaluation_result_field() -> None: + """The sensor. A field added to EvaluationResult must be classified as + carried or recomputed before this passes — otherwise it silently defaults on + every detached grade, which is exactly how `agent_config` was being lost.""" + classified = CARRIED | RECOMPUTED + fields = set(EvaluationResult.model_fields) + assert not fields - classified, ( + f"Unclassified EvaluationResult field(s): {sorted(fields - classified)}. Decide whether " + "_seed_from_prior_result must carry them, then add them to CARRIED or RECOMPUTED." + ) + assert not classified - fields, f"Stale entries: {sorted(classified - fields)}" + + +@pytest.mark.parametrize("field", sorted(CARRIED - {"environment_info"})) +def test_every_carried_field_reaches_the_regrade(field: str, tmp_path: Path) -> None: + orch, prior = _seeded(tmp_path) + assert orch.result is not None + # Anti-vacuity FIRST. This module's docstring claims every carried field is + # set to a distinctive value, and for two of sixteen it was not — so the + # assertion below compared a default against itself and would have passed + # with the production carry line deleted. Checking the fixture differs from + # the model default makes that failure mode impossible to reintroduce + # silently: a new CARRIED entry left at its default fails HERE, naming the + # fixture, rather than passing and pinning nothing. + default = EvaluationResult.model_fields[field].get_default(call_default_factory=True) + assert getattr(prior, field) != default, ( + f"_prior() leaves `{field}` at its model default, so the comparison below is vacuous — " + "it would pass with the carry line deleted. Give the fixture a distinctive value." + ) + assert getattr(orch.result, field) == getattr(prior, field), ( + f"_seed_from_prior_result dropped `{field}`; the graded row would report its default " + "instead of what the run actually did." + ) + + +def test_early_stop_is_carried_because_it_selects_the_gate(tmp_path: Path) -> None: + """Called out separately because it is the one carried field that changes the + VERDICT: gate selection is FIRED-ONLY, so a dropped early_stop re-grades a + truncated trajectory under the full-run strict-AND gate.""" + orch, _ = _seeded(tmp_path) + assert orch.result is not None and orch.result.early_stop is not None + assert orch.result.early_stop.reason is EarlyStopReason.CRITERION_FAILED + + +def test_grader_environment_is_kept_beside_the_run_s_not_over_it(tmp_path: Path) -> None: + orch, _ = _seeded(tmp_path) + assert orch.result is not None + # The run's own capture wins: a report showing the grader's tool versions as + # the run's is worse than one showing neither. + assert orch.result.environment_info["installed_tools"] == "prior" + # The grader is recorded as FLAT scalars, and only where it differs. Nesting + # a whole env capture here renders as a Python dict repr in the HTML report + # and violates the evalboard's declared value type. + assert orch.result.environment_info["graded_by_coder_eval"] == "9.9.9-grader" + assert all(not isinstance(v, dict) or k == "installed_tools" for k, v in orch.result.environment_info.items()), ( + "environment_info is consumed as a flat map" + ) + + +def test_the_evaluate_only_path_selects_the_same_gate_as_the_agent_path(tmp_path: Path) -> None: + """C1: gate selection is FIRED-ONLY, and a detached grade reaches the verdict + through the evaluate-only branch. That branch used to call + ``all_criteria_passed`` unconditionally, so re-grading an early-stopped run + applied the full-run strict-AND gate to a truncated trajectory and could flip + SUCCESS into FAILURE. Both paths must go through ``_select_gate``.""" + import inspect + + source = inspect.getsource(Orchestrator._evaluation_loop) + assert source.count("_select_gate()") == 2, ( + "Both the evaluate-only branch and the agent branch must select the gate through " + "_select_gate(); a second hand-written selection is how the seeded early_stop " + "came to be carried but never read." + ) + assert "all_criteria_passed" not in source, "gate selection belongs in _select_gate, not inline" + + # A truncated run: the ARMED criterion passed, the unarmed one never had the + # chance to. The armed gate says SUCCESS; strict-AND says FAILURE. That + # difference IS the flipped verdict, and it is decided purely by early_stop. + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[ + CommandExecutedCriterion( + description="armed", + tool_name="Read", + require_success=True, + stop_early={"on_pass": "stop"}, # type: ignore[arg-type] + ), + FileExistsCriterion(path="x.txt", description="unarmed"), + ], + ) + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v", prior_result=_prior()) + orch.result = _prior() + orch.result.success_criteria_results = [ + CriterionResult(criterion_type="command_executed", description="armed", score=1.0), + CriterionResult(criterion_type="file_exists", description="unarmed", score=0.0), + ] + + assert orch._select_gate() is True, "an early-stopped run gates on the armed subset" + orch.result.early_stop = None + assert orch._select_gate() is False, "a run that completed naturally gates strict-AND" + + +def test_grading_cannot_overturn_an_execution_fact() -> None: + """H5: a detached grade re-runs the criteria over a trajectory it did not + produce. It may move NOT_GRADED to a verdict; it must not turn a run that + timed out or crashed into a pass.""" + assert not FinalStatus.NOT_GRADED.is_execution_fact + assert not FinalStatus.SUCCESS.is_execution_fact + assert not FinalStatus.FAILURE.is_execution_fact + for status in ( + FinalStatus.ERROR, + FinalStatus.TIMEOUT, + FinalStatus.BUILD_FAILED, + FinalStatus.MAX_TURNS_EXHAUSTED, + FinalStatus.TOKEN_BUDGET_EXCEEDED, + FinalStatus.COST_BUDGET_EXCEEDED, + ): + assert status.is_execution_fact, f"{status} describes the run, so grading must preserve it" + + +def test_seeding_is_a_no_op_without_a_prior_result(tmp_path: Path) -> None: + task = TaskDefinition( + task_id="t", + description="d", + initial_prompt="p", + agent=parse_agent_config(type=AgentKind.CLAUDE_CODE), + success_criteria=[FileExistsCriterion(path="x.txt", description="x")], + ) + orch = Orchestrator(task=task, run_dir=tmp_path, variant_id="v") + orch.result = EvaluationResult( + task_id="t", + task_description="d", + variant_id="v", + agent_type=AgentKind.CLAUDE_CODE, + started_at=datetime(2030, 1, 1), + final_status=FinalStatus.FAILURE, + iteration_count=0, + ) + orch._seed_from_prior_result() + assert orch.result.started_at == datetime(2030, 1, 1) + assert orch.result.iterations == [] diff --git a/tests/test_suite_rollup.py b/tests/test_suite_rollup.py index 425ee7d7..7adb9c7b 100644 --- a/tests/test_suite_rollup.py +++ b/tests/test_suite_rollup.py @@ -6,6 +6,8 @@ from datetime import datetime from pathlib import Path +import pytest + from coder_eval.models import ( AgentKind, ClassificationCriterionResult, @@ -210,7 +212,11 @@ def test_error_count_per_criterion(self, tmp_path: Path) -> None: def test_empty_suite(self, tmp_path: Path) -> None: rollup = _compute_suite_rollup("s", "v1", [], tmp_path) assert rollup.rows_total == 0 - assert rollup.pass_rate == 0.0 + # None, not 0.0. A suite that measured nothing has no pass rate, and + # 0.0 publishes "0.0%" — indistinguishable from a suite where every row + # failed. Same rule as RunSummary.pass_rate and VariantAggregate. + assert rollup.pass_rate is None + assert rollup.rows_graded == 0 assert rollup.average_weighted_score is None @@ -746,3 +752,74 @@ def test_missing_aggregator_threshold_check_matches_injected_metric(self, tmp_pa assert cr_check.actual_value == agg.metrics["completion_rate"] assert cr_check.passed is False assert rollup.passed is False + + +def _row(row_id: str, final_status: FinalStatus, weighted_score: float | None) -> TaskResult: + """A one-criterion suite row. An ungraded row carries NO criteria results — + that is what `execute` leaves behind, and grading them is the whole point.""" + criteria = [] if weighted_score is None else [("file_exists", weighted_score, None)] + return _make_row( + suite_id="s", + row_id=row_id, + final_status=final_status, + weighted_score=weighted_score, + criteria=criteria, + ) + + +class TestSuiteRollupUngradedBucket: + """The fourth bucket on the suite surface, which shipped with no coverage. + + An ungraded row (`coder-eval execute`) leaves BOTH sides of the suite pass + rate — it is not a pass and not a failure — and it carries no failure reason, + so it must not appear in `failed_samples` either. The row-count invariant + counts it, so a row landing outside all four buckets fails loudly instead of + silently vanishing from the rollup. + """ + + def test_ungraded_rows_leave_both_sides_of_the_pass_rate(self, tmp_path: Path) -> None: + rows = [ + _row("r1", FinalStatus.SUCCESS, 1.0), + _row("r2", FinalStatus.NOT_GRADED, None), + _row("r3", FinalStatus.NOT_GRADED, None), + ] + rollup = _compute_suite_rollup("s", "v1", rows, tmp_path) + + assert rollup.rows_total == 3 + assert rollup.rows_not_graded == 2 + assert rollup.rows_graded == 1 + # 1 of 1 graded, not 1 of 3. + assert rollup.pass_rate == 1.0 + + def test_a_fully_ungraded_suite_has_no_pass_rate(self, tmp_path: Path) -> None: + rows = [_row(f"r{i}", FinalStatus.NOT_GRADED, None) for i in range(3)] + rollup = _compute_suite_rollup("s", "v1", rows, tmp_path) + + assert rollup.rows_graded == 0 + assert rollup.pass_rate is None + assert "n/a" in _render_suite_markdown(rollup) + + def test_ungraded_rows_are_not_collected_as_failed_samples(self, tmp_path: Path) -> None: + """`failed_samples` is documented as failed/errored rows. An ungraded row + has no failure reasons to show, so listing it contradicts the same + function's pass-rate rule two blocks up.""" + rows = [ + _row("r1", FinalStatus.FAILURE, 0.0), + _row("r2", FinalStatus.NOT_GRADED, None), + ] + rollup = _compute_suite_rollup("s", "v1", rows, tmp_path) + + assert [s.row_id for s in rollup.failed_samples] == ["r1"] + + def test_a_row_outside_every_bucket_fails_the_invariant(self) -> None: + with pytest.raises(ValueError, match="Suite row count invariant violated"): + SuiteRollup( + suite_id="s", + variant_id="v", + rows_total=3, + rows_passed=1, + rows_failed=0, + rows_error=0, + rows_not_graded=1, # 1+0+0+1 != 3 + pass_rate=1.0, + ) diff --git a/tests/test_ungraded_reporting.py b/tests/test_ungraded_reporting.py new file mode 100644 index 00000000..db2b9933 --- /dev/null +++ b/tests/test_ungraded_reporting.py @@ -0,0 +1,254 @@ +"""How an ungraded row renders on every reporting surface. + +`coder-eval execute` leaves rows `NOT_GRADED`, and each generator had to learn a +fourth category. Only the HTML badge got an assertion when that landed, so the +JUnit `` element, the Markdown "Not Graded" bullets, and — most +importantly — the switched pass-rate DENOMINATOR were all shipped untested. The +denominator is the "gate that turns a gap into a score" shape: a printed rate +whose divisor changed, with nothing tripping it. + +`VariantAggregate` is here too, mirroring the four `RunSummary` cases in +`test_execute_command.py`: the two models now carry the same formula, and only +one of them was tested. +""" + +from __future__ import annotations + +from datetime import datetime +from pathlib import Path +from typing import Any + +import pytest + +from coder_eval.models import FinalStatus, RunSummary, VariantAggregate, VariantResult +from coder_eval.orchestration.experiment import _mean_graded_score, _pick_worst_status + + +def _summary(**kwargs: object) -> RunSummary: + base: dict[str, object] = { + "run_id": "r", + "start_time": datetime(2026, 1, 1), + "end_time": datetime(2026, 1, 1, 0, 1), + "total_duration_seconds": 60.0, + "tasks_run": 2, + "tasks_succeeded": 1, + "tasks_failed": 0, + "tasks_error": 0, + "tasks_not_graded": 1, + "task_results": [], + "framework_version": "test", + } + base.update(kwargs) + return RunSummary(**base) # type: ignore[arg-type] + + +def _aggregate(**kwargs: object) -> VariantAggregate: + base: dict[str, object] = { + "variant_id": "v", + "tasks_run": 2, + "tasks_succeeded": 1, + "tasks_failed": 0, + "tasks_error": 0, + "tasks_not_graded": 1, + "average_score": 1.0, + "average_duration": 1.0, + } + base.update(kwargs) + return VariantAggregate(**base) # type: ignore[arg-type] + + +# -------------------------------------------------------------------------- +# VariantAggregate — the twin of the RunSummary cases in test_execute_command +# -------------------------------------------------------------------------- + + +def test_variant_aggregate_counts_the_ungraded_bucket_in_its_invariant() -> None: + with pytest.raises(ValueError, match="Task count invariant violated"): + _aggregate(tasks_run=3) # 1 + 0 + 0 + 1 != 3 + + +def test_variant_aggregate_pass_rate_divides_by_graded_not_run() -> None: + """One pass out of one GRADED task is 100%, even beside an ungraded one.""" + assert _aggregate().tasks_graded == 1 + assert _aggregate().pass_rate == 1.0 + + +def test_variant_aggregate_has_no_pass_rate_when_nothing_was_graded() -> None: + agg = _aggregate(tasks_run=2, tasks_succeeded=0, tasks_not_graded=2) + assert agg.pass_rate is None, "0/0 is unknown, not 0%" + + +def test_variant_aggregate_serializes_its_denominator() -> None: + """A consumer that cannot read `tasks_graded` re-derives the rate and drifts.""" + import json + + assert json.loads(_aggregate().model_dump_json())["tasks_graded"] == 1 + + +def test_mean_graded_score_ignores_ungraded_rows() -> None: + def _vr(score: float | None, status: FinalStatus) -> VariantResult: + return VariantResult( + variant_id="v", task_id="t", weighted_score=score, final_status=status, duration_seconds=1.0 + ) + + rows = [_vr(1.0, FinalStatus.SUCCESS), _vr(None, FinalStatus.NOT_GRADED), _vr(None, FinalStatus.NOT_GRADED)] + assert _mean_graded_score(rows) == 1.0 + # Nothing graded -> no mean at all. 0.0 would read as "measured and scored zero". + assert _mean_graded_score(rows[1:]) is None + + +def test_ungraded_loses_to_every_real_outcome_when_picking_the_worst_status() -> None: + """`_pick_worst_status` reports the replicate set's worst outcome. An ungraded + replicate is not an outcome, so it must never mask a real one.""" + assert _pick_worst_status([FinalStatus.NOT_GRADED, FinalStatus.SUCCESS]) is FinalStatus.SUCCESS + assert _pick_worst_status([FinalStatus.NOT_GRADED, FinalStatus.FAILURE]) is FinalStatus.FAILURE + assert _pick_worst_status([FinalStatus.NOT_GRADED]) is FinalStatus.NOT_GRADED + + +# -------------------------------------------------------------------------- +# Markdown +# -------------------------------------------------------------------------- + + +def test_markdown_pass_rate_uses_the_graded_denominator() -> None: + from coder_eval.reports import _pass_rate_lines + + text = "\n".join(_pass_rate_lines(_summary())) + + assert "1/1" in text, "the denominator is tasks_graded, not tasks_run" + assert "1/2" not in text + + +def test_markdown_reports_no_rate_at_all_for_a_fully_ungraded_run() -> None: + from coder_eval.reports import _pass_rate_lines + + text = "\n".join(_pass_rate_lines(_summary(tasks_succeeded=0, tasks_not_graded=2))) + + assert "n/a" in text + assert "0.0%" not in text, "a run that was never measured has no rate, not a 0% one" + + +def test_markdown_reports_no_rate_change_for_an_ordinary_graded_run() -> None: + """The regression guard: adding the fourth bucket must not alter any surface + of a run that has none.""" + from coder_eval.reports import _pass_rate_lines + + text = "\n".join(_pass_rate_lines(_summary(tasks_run=2, tasks_succeeded=1, tasks_failed=1, tasks_not_graded=0))) + + assert "1/2" in text + assert "n/a" not in text + + +# -------------------------------------------------------------------------- +# JUnit +# -------------------------------------------------------------------------- + + +def _junit_for(status: FinalStatus, tmp_path: Path) -> Any: + import json + + # defusedxml on the test side, matching tests/test_reports_junit.py. + from defusedxml.ElementTree import parse as parse_xml + + from coder_eval.reports_junit import write_junit_xml + + run_dir = tmp_path / "run" + task_dir = run_dir / "default" / "t" / "00" + task_dir.mkdir(parents=True) + row = { + "task_id": "t", + "task_description": "d", + "variant_id": "default", + "agent_type": "claude-code", + "started_at": "2026-01-01T00:00:00", + "final_status": status.value, + "iteration_count": 1, + "duration_seconds": 1.0, + } + (task_dir / "task.json").write_text(json.dumps(row), encoding="utf-8") + (run_dir / "run.json").write_text( + json.dumps( + { + "run_id": "r", + "start_time": "2026-01-01T00:00:00", + "end_time": "2026-01-01T00:01:00", + "total_duration_seconds": 60.0, + "tasks_run": 1, + "tasks_succeeded": 1 if status is FinalStatus.SUCCESS else 0, + "tasks_failed": 0, + "tasks_error": 0, + "tasks_not_graded": 1 if status is FinalStatus.NOT_GRADED else 0, + "task_results": [{"task_id": "t", "variant_id": "default", "status": status.value, "duration": 1.0}], + "framework_version": "test", + } + ), + encoding="utf-8", + ) + written = write_junit_xml(run_dir, tmp_path / "junit.xml") + return parse_xml(written).getroot() + + +def test_junit_marks_an_ungraded_row_skipped_not_failed(tmp_path: Path) -> None: + root = _junit_for(FinalStatus.NOT_GRADED, tmp_path) + + skipped = root.findall(".//testcase/skipped") + assert len(skipped) == 1, "an ungraded row is not a verdict, so it is " + assert "not graded" in (skipped[0].get("message") or "") + assert not root.findall(".//testcase/failure"), "an ungraded row must not read as a failure" + + +def test_junit_counts_are_derived_from_the_children(tmp_path: Path) -> None: + root = _junit_for(FinalStatus.NOT_GRADED, tmp_path) + suite = root.find(".//testsuite") + assert suite is not None + assert suite.get("skipped") == "1" + assert suite.get("failures") == "0" + assert suite.get("errors") == "0" + + +def test_junit_still_passes_an_ordinary_graded_row(tmp_path: Path) -> None: + root = _junit_for(FinalStatus.SUCCESS, tmp_path) + assert not root.findall(".//testcase/skipped") + assert not root.findall(".//testcase/failure") + + +# -------------------------------------------------------------------------- +# The end-of-run console summary +# -------------------------------------------------------------------------- + + +def _summary_output(summary: RunSummary, tmp_path: Path) -> str: + from coder_eval.cli.console import console + from coder_eval.cli.run_helpers import print_execution_summary + + with console.capture() as captured: + print_execution_summary(tmp_path, summary) + return captured.get() + + +def test_summary_reports_an_ungraded_run_as_executed_not_failed(tmp_path: Path) -> None: + text = _summary_output(_summary(tasks_succeeded=0, tasks_not_graded=2), tmp_path) + assert "2/2 executed, not graded" in text + assert "0/0 succeeded" not in text, "a fully ungraded run has no succeeded ratio to report" + + +def test_summary_points_at_the_grading_form_that_keeps_the_trajectory(tmp_path: Path) -> None: + """`evaluate ` grades a bare directory with NO + trajectory, so command_executed / skill_triggered / trajectory judges score + differently from what `run` would have produced.""" + text = _summary_output(_summary(tasks_succeeded=0, tasks_not_graded=2), tmp_path) + assert "--resume" in text or "" in text + assert "evaluate " not in text + + +def test_summary_still_reports_a_ratio_for_an_empty_run(tmp_path: Path) -> None: + """Both counters are falsy for tasks_run == 0; independent `if`s would print + no Results line at all, where it previously printed 0/0.""" + empty = _summary(tasks_run=0, tasks_succeeded=0, tasks_not_graded=0) + assert "0/0 succeeded" in _summary_output(empty, tmp_path) + + +def test_summary_is_unchanged_for_an_ordinary_graded_run(tmp_path: Path) -> None: + text = _summary_output(_summary(tasks_run=2, tasks_succeeded=1, tasks_failed=1, tasks_not_graded=0), tmp_path) + assert "1/2 succeeded" in text + assert "not graded" not in text