feat: structured, durable evidence for every check a pipeline gate runs

CLAUDE.md

@@ -180,6 +180,7 @@ codeoid mcp refresh <session>                       # apply MCP servers changed
 codeoid mcp approve <session> <server> [--path <f>] [--yes <fingerprint>]  # review, then approve a project's MCP server for every backend (#395)
 codeoid mcp adopt <server> [--session <s>] [--url <u>]  # codeoid signs in to a server from Claude's config (#390)
 codeoid mcp signin <server> [--callback-url <u>]        # sign in to an OAuth registry server
+codeoid evidence <id> [--full]   # one recorded check: what ran, on which tree, what it found
 codeoid routine add <name> --prompt … [--cron|--every|--at] [--check <cmd>]  # scheduled agent runs
 codeoid routine ls | show | runs | run [--wait] | edit | pause | resume | rm | cancel <runId> | token | preview
 ```

docs/pipeline-run.md

@@ -74,6 +74,13 @@ The only thing it saves is one auto-attach, which the client does for free.
   On a phase without `findings:` a `review` gate behaves as before (passes; the human is the reviewer).
 - **`skill` gates remain optional automated verdicts (later slice).**
   Until implemented they are simply absent, not fake.
+- **Every check a gate executes leaves evidence** (docs/loop-engineering-design.md §7).
+  `command` gates and execution probes record what ran, its result, its exit status or signal, bounded output excerpts, and the full output as files beside the daemon database (`<dbdir>/evidence/`).
+  An execution probe of type `test` (or `verify`) that exits 0 after running no tests is `inconclusive`, not a pass; a `command` gate is judged on its exit status alone.
+  A missing command is `error`; a check past its time bound is stopped with everything it spawned and recorded as a timed-out `fail`; aborting the pipeline or stopping the daemon stops its running checks.
+  Checks run with the same stripped environment as an agent's own commands (no daemon credentials), and stored excerpts are redacted; artifacts are readable only by the daemon's user and are pruned after 30 days or past 2 GiB in total.
+  The halted phase shows the latest check's output in the cockpit, `codeoid pipeline show` lists each phase's evidence ids, and `codeoid evidence <id>` prints one record.
+  Pipeline evidence names no pinned tree yet; it records what ran in the workdir, not which revision it certified.
 
 ---
 

packages/protocol/src/evidence.ts

@@ -0,0 +1,55 @@
+/**
+ * Evidence (docs/loop-engineering-design.md §7): the daemon's record of one
+ * executed check — what ran, on which tree, with what result — independent of
+ * any model's account of it.
+ *
+ * `evidence.get` reads one record. Its scope follows the record's owner: a
+ * pipeline's evidence needs `pipeline:read` and the pipeline must be visible to
+ * the caller. An id the caller cannot see is `not_found`, never `forbidden`, so
+ * ids reveal nothing across tenants.
+ */
+
+export type EvidenceResultWire = "pass" | "fail" | "inconclusive" | "error" | "cancelled";
+
+export interface EvidenceArtifactWire {
+  name: "stdout" | "stderr";
+  sha256: string;
+  bytes: number;
+  truncated: boolean;
+}
+
+export interface EvidenceWire {
+  id: string;
+  ownerKind: "pipeline" | "loop";
+  ownerId: string;
+  criterionId: string;
+  candidateDigest: string | null;
+  verifierDigest: string;
+  isolation: "process" | "sandbox";
+  command: string;
+  result: EvidenceResultWire;
+  reason: string;
+  exitCode: number | null;
+  signal: string | null;
+  timedOut: boolean;
+  /** ISO timestamps. */
+  startedAt: string;
+  endedAt: string;
+  environment: { platform: string; arch: string; runtime: string; envKeys: string[] };
+  stdoutExcerpt: string;
+  stderrExcerpt: string;
+  /** Full output streams held by the daemon; paths are not exposed. */
+  artifacts: EvidenceArtifactWire[];
+}
+
+export interface EvidenceGetMsg {
+  type: "evidence.get";
+  id: string;
+  evidenceId: string;
+}
+
+export interface EvidenceRecordMsg {
+  type: "evidence.record";
+  requestId: string;
+  evidence: EvidenceWire;
+}

packages/protocol/src/index.ts

@@ -11,3 +11,4 @@ export * from "./accounts.js";
 export * from "./endpoints.js";
 export * from "./mcp-oauth.js";
 export * from "./routines.js";
+export * from "./evidence.js";

packages/protocol/src/schemas.test.ts

@@ -197,6 +197,7 @@ const samples: { [T in ClientTypes]: Extract<ClientMessage, { type: T }> } = {
   "routine.delete": { type: "routine.delete", id: "rt3", routine: "nightly" },
   "routine.list": { type: "routine.list", id: "rt4" },
   "routine.get": { type: "routine.get", id: "rt5", routine: "nightly", runs: 5 },
+  "evidence.get": { type: "evidence.get", id: "ev1", evidenceId: "ev_123" },
   "routine.run": { type: "routine.run", id: "rt6", routine: "nightly", payload: "PR #12" },
   "routine.cancel_run": { type: "routine.cancel_run", id: "rt7", runId: "run-1" },
   "routine.runs": { type: "routine.runs", id: "rt8", routine: "nightly", limit: 20, before: "2026-10-16T00:00:00.000Z" },

packages/protocol/src/schemas.ts

@@ -849,6 +849,10 @@ const routinePatchSchema = z.object(
   },
 );
 
+// ── Evidence (docs/loop-engineering-design.md §7) ─────────────────────────────
+
+export const evidenceGetSchema = z.object({ ...base, type: z.literal("evidence.get"), evidenceId: idField });
+
 export const routineCreateSchema = z.object({ ...base, type: z.literal("routine.create"), spec: routineSpecSchema });
 export const routineUpdateSchema = z.object({
   ...base,
@@ -1157,6 +1161,7 @@ export const clientMessageSchema = z.discriminatedUnion("type", [
   routineRunsSchema,
   routineTokenSchema,
   routinePreviewSchema,
+  evidenceGetSchema,
   compareListSchema,
   compareKeepSchema,
   mcpOAuthBeginSchema,

packages/protocol/src/types.ts

@@ -84,6 +84,7 @@ import type {
   RoutineTokenResultMsg,
   RoutineUpdateMsg,
 } from "./routines.js";
+import type { EvidenceGetMsg, EvidenceRecordMsg } from "./evidence.js";
 import type {
   McpAdoptMsg,
   McpAdoptResultMsg,
@@ -1177,6 +1178,7 @@ export type ClientMessage =
   | RoutineRunsMsg
   | RoutineTokenMsg
   | RoutinePreviewMsg
+  | EvidenceGetMsg
   | McpOAuthBeginMsg
   | McpOAuthCompleteMsg
   | McpOAuthDisconnectMsg
@@ -2648,6 +2650,9 @@ export interface PipelinePhaseWire {
   /** Human revise notes accumulated on this phase (newest last) — the client
    *  renders the revision history. */
   feedback?: string[];
+  /** Evidence ids recorded by this phase's gates, oldest first — read each
+   *  with `evidence.get`. Absent when no gate executed a check. */
+  evidence?: string[];
   /** The findings loop's state for a phase that declares `findings:` (the
    *  engine's review → fix → re-review loop): counts plus a markdown ledger of
    *  every round. Optional (additive) — absent on phases without the loop and
@@ -2975,6 +2980,7 @@ export type DaemonMessage =
   | RoutineRunsResultMsg
   | RoutineTokenResultMsg
   | RoutinePreviewResultMsg
+  | EvidenceRecordMsg
   | McpOAuthBeginResultMsg
   | McpOAuthCompleteResultMsg
   | McpOAuthDisconnectResultMsg

src/cli.ts

@@ -1708,6 +1708,14 @@ pack
   .description("Uninstall a pack")
   .action((id: string) => withClient((c) => c.packRemove(id)));
 
+// ── Evidence (docs/loop-engineering-design.md §7) ───────────────────────────
+
+program
+  .command("evidence <id>")
+  .description("Show one evidence record: what a check ran, on which tree, and what it found")
+  .option("--full", "Print the whole stored excerpt instead of its last 40 lines")
+  .action((id: string, opts: { full?: boolean }) => withClient((c) => c.evidenceShow(id, opts)));
+
 // ── Pipeline runs (docs/pipeline-run.md) ────────────────────────────────────
 
 const pipeline = program

src/daemon/evidence/index.ts

@@ -0,0 +1,6 @@
+export { EvidenceService, ephemeralCheck, type CheckRun, type CheckRunner, type EvidenceContext } from "./service.js";
+export { EvidenceStore, type EvidenceTenant } from "./store.js";
+export { runCheck, verifierDigest } from "./runner.js";
+export { detectZeroTests } from "./parsers.js";
+export { evidenceToWire } from "./wire.js";
+export type * from "./types.js";

src/daemon/evidence/parsers.ts

@@ -0,0 +1,110 @@
+/**
+ * Recognize a test run that exited 0 without testing anything.
+ *
+ * A green exit after selecting no tests is the commonest false pass: a filter
+ * that matches nothing, a renamed directory, a runner pointed at the wrong
+ * package. §7 of the loop design says an empty test selection is not a pass, so
+ * a check marked `expectTests` that prints one of these is `inconclusive`.
+ *
+ * Patterns are each runner's own summary line for "nothing ran". They are
+ * deliberately specific: a false match turns a real pass into an inconclusive
+ * result, which costs an attempt but never certifies anything, so the bias is
+ * toward missing an exotic runner rather than misreading a common one.
+ *
+ * The scan sees every line of the output, not an excerpt: go and cargo decide
+ * from counts across the whole run, and the line that shows tests DID run can
+ * sit in the middle of a long log.
+ */
+
+interface ZeroTestPattern {
+  runner: string;
+  pattern: RegExp;
+}
+
+const ZERO_TEST_PATTERNS: readonly ZeroTestPattern[] = [
+  // bun test: "Ran 0 tests across 0 files." / "No tests found!".
+  { runner: "bun", pattern: /\bRan 0 tests\b/ },
+  { runner: "bun", pattern: /\bNo tests found!/ },
+  // jest: "No tests found, exiting with code 0" (with --passWithNoTests).
+  { runner: "jest", pattern: /\bNo tests found, exiting with code 0\b/ },
+  // vitest: "No test files found, exiting with code 0".
+  { runner: "vitest", pattern: /\bNo test files found, exiting with code 0\b/ },
+  // pytest: "no tests ran in 0.01s" (exit 5 normally, 0 with some plugins).
+  { runner: "pytest", pattern: /\bno tests ran\b/i },
+  // node --test: "# tests 0".
+  { runner: "node", pattern: /^# tests 0$/ },
+  // mocha: "0 passing".
+  { runner: "mocha", pattern: /^\s*0 passing\b/ },
+];
+
+/** Line-at-a-time scan of a check's output (both streams). */
+export class ZeroTestScan {
+  #matched: string | null = null;
+  // go: "?   pkg [no test files]" per package; "ok"/"FAIL"/"---" when tests ran.
+  #goNoTestFiles = false;
+  #goRan = false;
+  // cargo: "running N tests" per test binary.
+  #cargoRunningLines = 0;
+  #cargoNonZero = false;
+
+  line(text: string): void {
+    if (this.#matched === null) {
+      for (const { runner, pattern } of ZERO_TEST_PATTERNS) {
+        if (pattern.test(text)) {
+          this.#matched = runner;
+          break;
+        }
+      }
+    }
+    if (/\[no test files\]/.test(text)) this.#goNoTestFiles = true;
+    if (/^(ok|FAIL|---)\s/.test(text)) this.#goRan = true;
+    const cargo = /^running (\d+) tests?$/.exec(text);
+    if (cargo) {
+      this.#cargoRunningLines++;
+      if (Number(cargo[1]) > 0) this.#cargoNonZero = true;
+    }
+  }
+
+  /** The runner whose "nothing ran" summary was seen, or null. */
+  result(): string | null {
+    if (this.#matched) return this.#matched;
+    if (this.#goNoTestFiles && !this.#goRan) return "go";
+    if (this.#cargoRunningLines > 0 && !this.#cargoNonZero) return "cargo";
+    return null;
+  }
+}
+
+/** Splits a byte stream into lines for a ZeroTestScan, carrying partial lines
+ *  across chunks. Lines past a length bound are scanned truncated. */
+export class LineSplitter {
+  static readonly MAX_LINE = 4_096;
+  #partial = "";
+  readonly #onLine: (line: string) => void;
+
+  constructor(onLine: (line: string) => void) {
+    this.#onLine = onLine;
+  }
+
+  push(text: string): void {
+    const parts = (this.#partial + text).split("\n");
+    this.#partial = parts.pop() ?? "";
+    if (this.#partial.length > LineSplitter.MAX_LINE) this.#partial = this.#partial.slice(0, LineSplitter.MAX_LINE);
+    for (const p of parts) this.#onLine(p.replace(/\r$/, "").slice(0, LineSplitter.MAX_LINE));
+  }
+
+  end(): void {
+    if (this.#partial) this.#onLine(this.#partial.replace(/\r$/, ""));
+    this.#partial = "";
+  }
+}
+
+/**
+ * The runner whose "nothing ran" summary appears in `output`, or null when the
+ * output shows tests ran (or shows nothing recognizable — absence of a known
+ * summary is not evidence of an empty selection).
+ */
+export function detectZeroTests(output: string): string | null {
+  const scan = new ZeroTestScan();
+  for (const line of output.split("\n")) scan.line(line.replace(/\r$/, ""));
+  return scan.result();
+}

src/daemon/evidence/redact.test.ts

@@ -0,0 +1,47 @@
+import { describe, expect, test } from "bun:test";
+import { redactOutput } from "./redact";
+
+describe("redactOutput", () => {
+  test("keeps what a worker needs to repair a failure", () => {
+    const log = [
+      "exit code: 1",
+      "--- PASS: TestParseHeader (0.00s)",
+      "--- FAIL: TestTokenizerHandlesUnicode (0.01s)",
+      "FAIL src/daemon/pipeline/session-manager.routines.test.ts",
+      "    at Object.<anonymous> (/home/u/repo/src/very_long_component_name_for_testing.ts:42:7)",
+      "commit 3f9c2a1b8e7d6c5b4a3928170f6e5d4c3b2a1908",
+      "expected 2 received 3",
+    ].join("\n");
+    expect(redactOutput(log)).toBe(log);
+  });
+
+  test("masks known credential shapes and secret-named values", () => {
+    const out = redactOutput(
+      [
+        "Authorization: Bearer abcdefgh.ijklmnop-qrstuv",
+        "export GITHUB_TOKEN=ghp_0123456789abcdefghijABCDEFGHIJ",
+        "using key sk-ant-api03-0123456789abcdefghijklmn",
+        "DATABASE_PASSWORD='hunter2-is-secret'",
+        "connecting to postgres://app:s3cretpass@db.internal:5432/app",
+        "aws AKIAABCDEFGHIJKLMNOP",
+      ].join("\n"),
+    );
+    for (const secret of ["ijklmnop", "ghp_0123", "sk-ant-api03", "hunter2", "s3cretpass", "AKIAABCDEF"]) {
+      expect(out).not.toContain(secret);
+    }
+    expect(out).toContain("db.internal:5432");
+  });
+
+  test("a PEM block longer than any scan window is masked whole", () => {
+    const body = Array.from({ length: 60 }, (_, i) => `MIIBVwIBADANBgkqhkiG9w0BAQEFAASCAT8wggE7AgEAAkEA${String(i).padStart(4, "0")}`).join("\n");
+    const out = redactOutput(`before\n-----BEGIN RSA PRIVATE KEY-----\n${body}\n-----END RSA PRIVATE KEY-----\nafter`);
+    expect(out).not.toContain("MIIBVwIBADAN");
+    expect(out).toContain("before");
+    expect(out).toContain("after");
+  });
+
+  test("a secret deep inside a very long line is still masked", () => {
+    const line = `${"x".repeat(1_790)} password=hunter2Secret!x ${"y".repeat(500)}`;
+    expect(redactOutput(line)).not.toContain("hunter2Secret");
+  });
+});

src/daemon/evidence/redact.ts

@@ -0,0 +1,53 @@
+/**
+ * Redact credentials from a check's captured output before it is stored as an
+ * excerpt or shown to a model. Test output echoes tokens more often than one
+ * would like: `set -x`, a failing HTTP fixture, a connection string in an error.
+ *
+ * These rules are written for logs, not configuration. Test output is mostly
+ * file paths, test names, hashes, and exit codes, and a worker repairing a
+ * failure needs all of them, so nothing is masked for merely looking random:
+ * only known credential shapes and values bound to keys that name a secret.
+ * The whole text is scanned at once (every pattern is linear), so a PEM block
+ * or a long line is never split between two scans.
+ */
+
+const MASK = "…[redacted]";
+
+/** Key names whose values are secrets, in `key=value` / `key: value` form. */
+const SECRET_KEY =
+  "(?:[A-Za-z0-9_.-]{0,40}[_.-])?(?:password|passwd|pwd|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key|client[_-]?secret|auth[_-]?token|credentials?|session[_-]?key)";
+
+const RULES: ReadonlyArray<[RegExp, string | ((...m: string[]) => string)]> = [
+  // PEM blocks, terminated or cut off by the excerpt.
+  [/-----BEGIN [A-Z0-9 ]{1,40}-----[\s\S]*?(?:-----END [A-Z0-9 ]{1,40}-----|$)/g, MASK],
+  // JWTs.
+  [/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}(?:\.[A-Za-z0-9_-]*)?/g, MASK],
+  // scheme://user:password@host — keep the scheme and host.
+  [/\b([a-z][a-z0-9+.-]{0,20}:\/\/)[^/\s:@]{0,200}:[^/\s@]{1,500}@/gi, `$1${MASK}@`],
+  // Authorization headers and schemes.
+  [/\b(authorization|proxy-authorization)(\s*[:=]\s*)(?:(bearer|basic|token|digest)\s+)?[^\s"',;]{4,}/gi, (_m, k, sep, scheme) => `${k}${sep}${scheme ? `${scheme} ` : ""}${MASK}`],
+  [/\b(bearer|basic)\s+[A-Za-z0-9._~+/=-]{8,}/gi, `$1 ${MASK}`],
+  // Well-known token formats.
+  [/\b(?:sk|pk|rk)-(?:ant-|proj-|live-|test-)?[A-Za-z0-9_-]{16,}/g, MASK],
+  [/\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{20,}/g, MASK],
+  [/\bgithub_pat_[A-Za-z0-9_]{20,}/g, MASK],
+  [/\bglpat-[A-Za-z0-9_-]{16,}/g, MASK],
+  [/\bxox[abposr]-[A-Za-z0-9-]{10,}/g, MASK],
+  [/\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g, MASK],
+  [/\bAIza[0-9A-Za-z_-]{30,}/g, MASK],
+  [/\b(?:zid_sk|hf_sk|npm)_[A-Za-z0-9_-]{16,}/g, MASK],
+  // key=value, key: value, "key": "value" — only for keys that name a secret.
+  [
+    new RegExp(`(^|[^A-Za-z0-9])(${SECRET_KEY})(["']?\\s*[:=]\\s*)("[^"\\n]{1,500}"|'[^'\\n]{1,500}'|[^\\s"',;)}\\]]{1,500})`, "gim"),
+    (_m, pre, key, sep) => `${pre}${key}${sep}${MASK}`,
+  ],
+];
+
+export function redactOutput(text: string): string {
+  if (!text) return text;
+  let out = text;
+  for (const [pattern, replacement] of RULES) {
+    out = out.replace(pattern, replacement as string);
+  }
+  return out;
+}