From a3034ae026aa306569957db94bd79e710ec8ba87 Mon Sep 17 00:00:00 2001 From: Yash Datta Date: Sun, 25 Oct 2026 08:04:21 +0800 Subject: [PATCH 1/3] feat: structured, durable evidence for every check a pipeline gate runs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first slice of loops (docs/loop-engineering-design.md §7, milestone A). A check's verdict is what a command did, captured by the daemon: result (pass / fail / inconclusive / error / cancelled), exit status or signal, bounded excerpts, the full output as artifact files, the verifier's digest, and the environment — persisted in the daemon DB. - evidence/runner.ts runs each check as its own process group, so a timeout, cancel, or the check's own exit kills everything it spawned; a test run that selects no tests is inconclusive, never a pass. - Pipeline command gates and execution probes record evidence, carry the failing output into the gate reason, and list their evidence on the phase. - evidence.get reads one record (pipeline:read, tenant- and owner-scoped; an id the caller cannot see is not_found); codeoid evidence prints it; the halted phase in the web cockpit shows the latest check's output. Co-Authored-By: Claude Opus 5.5 (1M context) --- CLAUDE.md | 1 + docs/pipeline-run.md | 5 + packages/protocol/src/evidence.ts | 55 +++++ packages/protocol/src/index.ts | 1 + packages/protocol/src/schemas.test.ts | 1 + packages/protocol/src/schemas.ts | 5 + packages/protocol/src/types.ts | 6 + src/cli.ts | 8 + src/daemon/evidence/index.ts | 6 + src/daemon/evidence/parsers.ts | 65 +++++ src/daemon/evidence/runner.test.ts | 168 +++++++++++++ src/daemon/evidence/runner.ts | 293 +++++++++++++++++++++++ src/daemon/evidence/service.test.ts | 75 ++++++ src/daemon/evidence/service.ts | 92 +++++++ src/daemon/evidence/store.ts | 177 ++++++++++++++ src/daemon/evidence/types.ts | 111 +++++++++ src/daemon/evidence/wire.ts | 31 +++ src/daemon/pipeline/engine.ts | 26 +- src/daemon/pipeline/gate-probes.test.ts | 23 ++ src/daemon/pipeline/gate-probes.ts | 75 +++--- src/daemon/pipeline/interface.ts | 8 + src/daemon/pipeline/manager.ts | 5 +- src/daemon/pipeline/pack.test.ts | 30 +++ src/daemon/pipeline/pack.ts | 36 ++- src/daemon/pipeline/wiring.ts | 5 +- src/daemon/session-manager.ts | 35 +++ src/terminal/client.ts | 10 + src/terminal/evidence-format.test.ts | 43 ++++ src/terminal/evidence-format.ts | 34 +++ src/terminal/pipeline-format.ts | 4 + src/tests/pipeline-handlers.test.ts | 77 ++++++ src/tests/protocol.test.ts | 2 + web/src/components/EvidenceView.test.tsx | 59 +++++ web/src/components/EvidenceView.tsx | 86 +++++++ web/src/components/PipelineRunner.tsx | 5 + web/src/state/evidence.ts | 30 +++ 36 files changed, 1631 insertions(+), 62 deletions(-) create mode 100644 packages/protocol/src/evidence.ts create mode 100644 src/daemon/evidence/index.ts create mode 100644 src/daemon/evidence/parsers.ts create mode 100644 src/daemon/evidence/runner.test.ts create mode 100644 src/daemon/evidence/runner.ts create mode 100644 src/daemon/evidence/service.test.ts create mode 100644 src/daemon/evidence/service.ts create mode 100644 src/daemon/evidence/store.ts create mode 100644 src/daemon/evidence/types.ts create mode 100644 src/daemon/evidence/wire.ts create mode 100644 src/terminal/evidence-format.test.ts create mode 100644 src/terminal/evidence-format.ts create mode 100644 web/src/components/EvidenceView.test.tsx create mode 100644 web/src/components/EvidenceView.tsx create mode 100644 web/src/state/evidence.ts diff --git a/CLAUDE.md b/CLAUDE.md index 5a27ad46..cc18da74 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -180,6 +180,7 @@ codeoid mcp refresh # apply MCP servers changed codeoid mcp approve [--path ] [--yes ] # review, then approve a project's MCP server for every backend (#395) codeoid mcp adopt [--session ] [--url ] # codeoid signs in to a server from Claude's config (#390) codeoid mcp signin [--callback-url ] # sign in to an OAuth registry server +codeoid evidence [--full] # one recorded check: what ran, on which tree, what it found codeoid routine add --prompt … [--cron|--every|--at] [--check ] # scheduled agent runs codeoid routine ls | show | runs | run [--wait] | edit | pause | resume | rm | cancel | token | preview ``` diff --git a/docs/pipeline-run.md b/docs/pipeline-run.md index c770a9e3..a1e9c795 100644 --- a/docs/pipeline-run.md +++ b/docs/pipeline-run.md @@ -74,6 +74,11 @@ The only thing it saves is one auto-attach, which the client does for free. On a phase without `findings:` a `review` gate behaves as before (passes; the human is the reviewer). - **`skill` gates remain optional automated verdicts (later slice).** Until implemented they are simply absent, not fake. +- **Every check a gate executes leaves evidence** (docs/loop-engineering-design.md §7). + `command` gates and execution probes record what ran, its result, its exit status or signal, bounded output excerpts, and the full output as files beside the daemon database (`/evidence/`). + A check that exits 0 after running no tests is `inconclusive`, not a pass; a missing command is `error`; a check past its time bound is stopped with everything it spawned and recorded as a timed-out `fail`. + The halted phase shows the latest check's output in the cockpit, `codeoid pipeline show` lists each phase's evidence ids, and `codeoid evidence ` prints one record. + Pipeline evidence names no pinned tree yet; it records what ran in the workdir, not which revision it certified. --- diff --git a/packages/protocol/src/evidence.ts b/packages/protocol/src/evidence.ts new file mode 100644 index 00000000..2905daa9 --- /dev/null +++ b/packages/protocol/src/evidence.ts @@ -0,0 +1,55 @@ +/** + * Evidence (docs/loop-engineering-design.md §7): the daemon's record of one + * executed check — what ran, on which tree, with what result — independent of + * any model's account of it. + * + * `evidence.get` reads one record. Its scope follows the record's owner: a + * pipeline's evidence needs `pipeline:read` and the pipeline must be visible to + * the caller. An id the caller cannot see is `not_found`, never `forbidden`, so + * ids reveal nothing across tenants. + */ + +export type EvidenceResultWire = "pass" | "fail" | "inconclusive" | "error" | "cancelled"; + +export interface EvidenceArtifactWire { + name: "stdout" | "stderr"; + sha256: string; + bytes: number; + truncated: boolean; +} + +export interface EvidenceWire { + id: string; + ownerKind: "pipeline" | "loop"; + ownerId: string; + criterionId: string; + candidateDigest: string | null; + verifierDigest: string; + isolation: "process" | "sandbox"; + command: string; + result: EvidenceResultWire; + reason: string; + exitCode: number | null; + signal: string | null; + timedOut: boolean; + /** ISO timestamps. */ + startedAt: string; + endedAt: string; + environment: { platform: string; arch: string; runtime: string; envKeys: string[] }; + stdoutExcerpt: string; + stderrExcerpt: string; + /** Full output streams held by the daemon; paths are not exposed. */ + artifacts: EvidenceArtifactWire[]; +} + +export interface EvidenceGetMsg { + type: "evidence.get"; + id: string; + evidenceId: string; +} + +export interface EvidenceRecordMsg { + type: "evidence.record"; + requestId: string; + evidence: EvidenceWire; +} diff --git a/packages/protocol/src/index.ts b/packages/protocol/src/index.ts index 317d630c..ad75c6ac 100644 --- a/packages/protocol/src/index.ts +++ b/packages/protocol/src/index.ts @@ -11,3 +11,4 @@ export * from "./accounts.js"; export * from "./endpoints.js"; export * from "./mcp-oauth.js"; export * from "./routines.js"; +export * from "./evidence.js"; diff --git a/packages/protocol/src/schemas.test.ts b/packages/protocol/src/schemas.test.ts index 80e7039d..b1932ab7 100644 --- a/packages/protocol/src/schemas.test.ts +++ b/packages/protocol/src/schemas.test.ts @@ -197,6 +197,7 @@ const samples: { [T in ClientTypes]: Extract } = { "routine.delete": { type: "routine.delete", id: "rt3", routine: "nightly" }, "routine.list": { type: "routine.list", id: "rt4" }, "routine.get": { type: "routine.get", id: "rt5", routine: "nightly", runs: 5 }, + "evidence.get": { type: "evidence.get", id: "ev1", evidenceId: "ev_123" }, "routine.run": { type: "routine.run", id: "rt6", routine: "nightly", payload: "PR #12" }, "routine.cancel_run": { type: "routine.cancel_run", id: "rt7", runId: "run-1" }, "routine.runs": { type: "routine.runs", id: "rt8", routine: "nightly", limit: 20, before: "2026-10-16T00:00:00.000Z" }, diff --git a/packages/protocol/src/schemas.ts b/packages/protocol/src/schemas.ts index d36d1c68..92be6847 100644 --- a/packages/protocol/src/schemas.ts +++ b/packages/protocol/src/schemas.ts @@ -849,6 +849,10 @@ const routinePatchSchema = z.object( }, ); +// ── Evidence (docs/loop-engineering-design.md §7) ───────────────────────────── + +export const evidenceGetSchema = z.object({ ...base, type: z.literal("evidence.get"), evidenceId: idField }); + export const routineCreateSchema = z.object({ ...base, type: z.literal("routine.create"), spec: routineSpecSchema }); export const routineUpdateSchema = z.object({ ...base, @@ -1157,6 +1161,7 @@ export const clientMessageSchema = z.discriminatedUnion("type", [ routineRunsSchema, routineTokenSchema, routinePreviewSchema, + evidenceGetSchema, compareListSchema, compareKeepSchema, mcpOAuthBeginSchema, diff --git a/packages/protocol/src/types.ts b/packages/protocol/src/types.ts index a557242d..e48e0325 100644 --- a/packages/protocol/src/types.ts +++ b/packages/protocol/src/types.ts @@ -84,6 +84,7 @@ import type { RoutineTokenResultMsg, RoutineUpdateMsg, } from "./routines.js"; +import type { EvidenceGetMsg, EvidenceRecordMsg } from "./evidence.js"; import type { McpAdoptMsg, McpAdoptResultMsg, @@ -1177,6 +1178,7 @@ export type ClientMessage = | RoutineRunsMsg | RoutineTokenMsg | RoutinePreviewMsg + | EvidenceGetMsg | McpOAuthBeginMsg | McpOAuthCompleteMsg | McpOAuthDisconnectMsg @@ -2648,6 +2650,9 @@ export interface PipelinePhaseWire { /** Human revise notes accumulated on this phase (newest last) — the client * renders the revision history. */ feedback?: string[]; + /** Evidence ids recorded by this phase's gates, oldest first — read each + * with `evidence.get`. Absent when no gate executed a check. */ + evidence?: string[]; /** The findings loop's state for a phase that declares `findings:` (the * engine's review → fix → re-review loop): counts plus a markdown ledger of * every round. Optional (additive) — absent on phases without the loop and @@ -2975,6 +2980,7 @@ export type DaemonMessage = | RoutineRunsResultMsg | RoutineTokenResultMsg | RoutinePreviewResultMsg + | EvidenceRecordMsg | McpOAuthBeginResultMsg | McpOAuthCompleteResultMsg | McpOAuthDisconnectResultMsg diff --git a/src/cli.ts b/src/cli.ts index 250ff53f..b2da46d0 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -1708,6 +1708,14 @@ pack .description("Uninstall a pack") .action((id: string) => withClient((c) => c.packRemove(id))); +// ── Evidence (docs/loop-engineering-design.md §7) ─────────────────────────── + +program + .command("evidence ") + .description("Show one evidence record: what a check ran, on which tree, and what it found") + .option("--full", "Print the whole stored excerpt instead of its last 40 lines") + .action((id: string, opts: { full?: boolean }) => withClient((c) => c.evidenceShow(id, opts))); + // ── Pipeline runs (docs/pipeline-run.md) ──────────────────────────────────── const pipeline = program diff --git a/src/daemon/evidence/index.ts b/src/daemon/evidence/index.ts new file mode 100644 index 00000000..19916844 --- /dev/null +++ b/src/daemon/evidence/index.ts @@ -0,0 +1,6 @@ +export { EvidenceService, ephemeralCheck, type CheckRun, type CheckRunner, type EvidenceContext } from "./service.js"; +export { EvidenceStore, type EvidenceTenant } from "./store.js"; +export { runCheck, verifierDigest } from "./runner.js"; +export { detectZeroTests } from "./parsers.js"; +export { evidenceToWire } from "./wire.js"; +export type * from "./types.js"; diff --git a/src/daemon/evidence/parsers.ts b/src/daemon/evidence/parsers.ts new file mode 100644 index 00000000..f6cd350c --- /dev/null +++ b/src/daemon/evidence/parsers.ts @@ -0,0 +1,65 @@ +/** + * Recognize a test run that exited 0 without testing anything. + * + * A green exit after selecting no tests is the commonest false pass: a filter + * that matches nothing, a renamed directory, a runner pointed at the wrong + * package. §7 of the loop design says an empty test selection is not a pass, so + * a check marked `expectTests` that prints one of these is `inconclusive`. + * + * Patterns are each runner's own summary line for "nothing ran", matched on + * the combined output. They are deliberately specific: a false match turns a + * real pass into an inconclusive result, which costs an attempt but never + * certifies anything, so the bias is toward missing an exotic runner rather + * than misreading a common one. + */ + +interface ZeroTestPattern { + runner: string; + pattern: RegExp; +} + +const ZERO_TEST_PATTERNS: readonly ZeroTestPattern[] = [ + // bun test: "Ran 0 tests across 0 files." / "0 pass\n 0 fail" with no tests. + { runner: "bun", pattern: /\bRan 0 tests\b/ }, + { runner: "bun", pattern: /\bNo tests found!/ }, + // jest: "No tests found, exiting with code 0" (with --passWithNoTests). + { runner: "jest", pattern: /\bNo tests found, exiting with code 0\b/ }, + // vitest: "No test files found, exiting with code 0". + { runner: "vitest", pattern: /\bNo test files found, exiting with code 0\b/ }, + // pytest: "no tests ran in 0.01s" (exit 5 normally, 0 with some plugins). + { runner: "pytest", pattern: /\bno tests ran\b/i }, + // cargo: every "running N tests" line says 0 — checked separately below. + // node --test: "# tests 0". + { runner: "node", pattern: /^# tests 0$/m }, + // mocha: "0 passing". + { runner: "mocha", pattern: /^\s*0 passing\b/m }, +]; + +/** go test prints "? pkg [no test files]" per package and "ok pkg" for each + * package whose tests ran. Nothing ran when there are no-test-file lines and + * no ok/FAIL lines. */ +function goRanNothing(output: string): boolean { + if (!/\[no test files\]/.test(output)) return false; + return !/^(ok|FAIL|---)\s/m.test(output); +} + +/** cargo test prints "running N tests" per test binary; nothing ran when there + * is at least one such line and every one says 0. */ +function cargoRanNothing(output: string): boolean { + const counts = [...output.matchAll(/^running (\d+) tests?$/gm)].map((m) => Number(m[1])); + return counts.length > 0 && counts.every((n) => n === 0); +} + +/** + * The runner whose "nothing ran" summary appears in `output`, or null when the + * output shows tests ran (or shows nothing recognizable — absence of a known + * summary is not evidence of an empty selection). + */ +export function detectZeroTests(output: string): string | null { + for (const { runner, pattern } of ZERO_TEST_PATTERNS) { + if (pattern.test(output)) return runner; + } + if (goRanNothing(output)) return "go"; + if (cargoRanNothing(output)) return "cargo"; + return null; +} diff --git a/src/daemon/evidence/runner.test.ts b/src/daemon/evidence/runner.test.ts new file mode 100644 index 00000000..f89d4112 --- /dev/null +++ b/src/daemon/evidence/runner.test.ts @@ -0,0 +1,168 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { detectZeroTests } from "./parsers"; +import { EXCERPT_HEAD_BYTES, EXCERPT_TAIL_BYTES, runCheck, verifierDigest } from "./runner"; +import type { CheckSpec } from "./types"; + +let dir: string; +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "codeoid-evidence-")); +}); +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +const spec = (command: string, extra: Partial = {}): CheckSpec => ({ + id: "c1", + command, + timeoutMs: 10_000, + ...extra, +}); +const run = (s: CheckSpec, signal?: AbortSignal) => + runCheck(s, { cwd: dir, artifactDir: join(dir, "art"), signal, killGraceMs: 200 }); + +describe("classification", () => { + test("exit 0 is a pass; non-zero is a fail with the code", async () => { + expect((await run(spec("true"))).result).toBe("pass"); + const failed = await run(spec("exit 3")); + expect(failed.result).toBe("fail"); + expect(failed.exitCode).toBe(3); + }); + + test("a missing command is an error, not a fail and never a pass", async () => { + const out = await run(spec("definitely-not-a-binary-xyz")); + expect(out.result).toBe("error"); + expect(out.reason).toContain("not found"); + }); + + test("a directory that does not exist is an error", async () => { + const out = await runCheck(spec("true"), { cwd: join(dir, "missing"), artifactDir: join(dir, "art") }); + expect(out.result).toBe("error"); + }); + + test("a test run that selected no tests is inconclusive", async () => { + const out = await run(spec("echo 'Ran 0 tests across 0 files.'", { expectTests: true })); + expect(out.result).toBe("inconclusive"); + }); + + test("the same output from a non-test check is a pass", async () => { + expect((await run(spec("echo 'Ran 0 tests across 0 files.'"))).result).toBe("pass"); + }); + + test("a check killed by a signal is a fail naming it", async () => { + const out = await run(spec("kill -TERM $$")); + expect(out.result).toBe("fail"); + expect(out.signal).toBe("SIGTERM"); + }); +}); + +describe("process ownership", () => { + test("a timeout kills the whole group and records a timed-out fail", async () => { + const marker = join(dir, "child-alive"); + // The child would create the marker after 2s if it survived the kill. + const out = await run(spec(`(sleep 2; touch ${marker}) & sleep 30`, { timeoutMs: 300 })); + expect(out.result).toBe("fail"); + expect(out.timedOut).toBe(true); + await Bun.sleep(2_500); + expect(existsSync(marker)).toBe(false); + }); + + test("a descendant left running after the check exits is killed", async () => { + const marker = join(dir, "orphan-alive"); + const out = await run(spec(`(sleep 1; touch ${marker}) & exit 0`)); + expect(out.result).toBe("pass"); + await Bun.sleep(1_500); + expect(existsSync(marker)).toBe(false); + }); + + test("cancelling stops the check and records cancelled", async () => { + const ac = new AbortController(); + setTimeout(() => ac.abort(), 200); + const out = await run(spec("sleep 30"), ac.signal); + expect(out.result).toBe("cancelled"); + }); + + test("an already-cancelled signal never starts the check", async () => { + const ac = new AbortController(); + ac.abort(); + const marker = join(dir, "ran"); + const out = await run(spec(`touch ${marker}`), ac.signal); + expect(out.result).toBe("cancelled"); + expect(existsSync(marker)).toBe(false); + }); +}); + +describe("output", () => { + test("excerpts keep the head and tail; the artifact keeps everything", async () => { + const out = await run(spec("i=0; while [ $i -lt 4000 ]; do echo line-$i; i=$((i+1)); done")); + expect(out.stdoutExcerpt.startsWith("line-0\n")).toBe(true); + expect(out.stdoutExcerpt).toContain("line-3999"); + expect(out.stdoutExcerpt).toContain("bytes omitted"); + expect(out.stdoutExcerpt.length).toBeLessThan(EXCERPT_HEAD_BYTES + EXCERPT_TAIL_BYTES + 200); + const stdout = out.artifacts.find((a) => a.name === "stdout")!; + const full = readFileSync(stdout.path, "utf8"); + expect(full.split("\n").filter(Boolean)).toHaveLength(4000); + expect(stdout.bytes).toBe(Buffer.byteLength(full)); + expect(stdout.truncated).toBe(false); + }); + + test("an artifact over its cap is truncated and says so", async () => { + const out = await runCheck(spec("head -c 5000 /dev/zero"), { + cwd: dir, + artifactDir: join(dir, "art"), + maxArtifactBytes: 1000, + }); + const stdout = out.artifacts.find((a) => a.name === "stdout")!; + expect(stdout.bytes).toBe(1000); + expect(stdout.truncated).toBe(true); + }); + + test("stderr is captured separately", async () => { + const out = await run(spec("echo out; echo err >&2; exit 1")); + expect(out.stdoutExcerpt.trim()).toBe("out"); + expect(out.stderrExcerpt.trim()).toBe("err"); + }); +}); + +describe("verifier digest", () => { + test("pins the definition but not env values", () => { + const a = verifierDigest(spec("bun test", { env: { TOKEN: "one" } })); + const b = verifierDigest(spec("bun test", { env: { TOKEN: "two" } })); + const c = verifierDigest(spec("bun test --bail", { env: { TOKEN: "one" } })); + expect(a).toBe(b); + expect(a).not.toBe(c); + }); + + test("env values never reach the outcome", async () => { + const out = await run(spec("true", { env: { SECRET_TOKEN: "hunter2" } })); + expect(JSON.stringify(out)).not.toContain("hunter2"); + expect(out.environment.envKeys).toEqual(["SECRET_TOKEN"]); + }); +}); + +describe("detectZeroTests", () => { + test.each([ + ["bun", "Ran 0 tests across 0 files. [5.00ms]"], + ["jest", "No tests found, exiting with code 0"], + ["vitest", "No test files found, exiting with code 0"], + ["pytest", "============ no tests ran in 0.01s ============"], + ["go", "? \texample.com/pkg\t[no test files]\n? \texample.com/b\t[no test files]"], + ["cargo", "running 0 tests\n\ntest result: ok. 0 passed\nrunning 0 tests"], + ["node", "# tests 0\n# pass 0"], + ["mocha", " 0 passing (2ms)"], + ])("%s empty selection", (runner, output) => { + expect(detectZeroTests(output)).toBe(runner); + }); + + test.each([ + ["bun", "Ran 12 tests across 3 files."], + ["go", "? \tpkg/a\t[no test files]\nok \tpkg/b\t0.01s"], + ["cargo", "running 0 tests\nrunning 4 tests"], + ["mocha", " 10 passing (2ms)"], + ["unknown runner", "all good"], + ])("%s with tests run is not flagged", (_name, output) => { + expect(detectZeroTests(output)).toBeNull(); + }); +}); diff --git a/src/daemon/evidence/runner.ts b/src/daemon/evidence/runner.ts new file mode 100644 index 00000000..e073d6b4 --- /dev/null +++ b/src/daemon/evidence/runner.ts @@ -0,0 +1,293 @@ +/** + * Run one check and capture what it did (docs/loop-engineering-design.md §7). + * + * The runner owns the process it starts: the check runs as the leader of its + * own process group, so a timeout, a cancel, or the check's own exit can kill + * everything it spawned — a test server left behind by `sh -c` included. Output + * is kept twice: bounded excerpts for display and worker feedback, and the full + * streams as artifact files (up to a cap) so nothing a reviewer needs is lost. + * + * Classification is the runner's job, not the caller's: exit 0 is `pass` only + * when it actually established something (see parsers.ts), a timeout is + * `fail`, and anything that stopped the check from running is `error`. + */ + +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { createWriteStream, mkdirSync, type WriteStream } from "node:fs"; +import { join } from "node:path"; +import { detectZeroTests } from "./parsers"; +import type { CheckOutcome, CheckSpec, EvidenceArtifact, EvidenceEnvironment, EvidenceResult } from "./types"; + +/** Excerpt kept from each end of a stream for display and worker feedback. */ +export const EXCERPT_HEAD_BYTES = 4 * 1024; +export const EXCERPT_TAIL_BYTES = 12 * 1024; +/** Default cap on one stream's artifact file. */ +export const DEFAULT_MAX_ARTIFACT_BYTES = 64 * 1024 * 1024; +/** Between SIGTERM and SIGKILL when the runner stops a check. */ +export const DEFAULT_KILL_GRACE_MS = 2_000; +/** How long to wait for output pipes to drain after the check exits, before + * giving up on a descendant that escaped the process group and still holds them. */ +const DRAIN_WAIT_MS = 2_000; + +export interface RunCheckOptions { + /** Directory the check runs in. */ + cwd: string; + /** Directory the full output streams are written to (created if missing). */ + artifactDir: string; + /** Aborting stops the check and records `cancelled`. */ + signal?: AbortSignal; + killGraceMs?: number; + maxArtifactBytes?: number; +} + +/** sha256 of a check's definition. Two runs with the same digest ran the same + * check; env values are excluded so a rotated secret doesn't look like a new + * verifier, and never reaches a stored record. */ +export function verifierDigest(spec: CheckSpec): string { + const def = { + command: spec.command, + timeoutMs: spec.timeoutMs, + envKeys: Object.keys(spec.env ?? {}).sort(), + expectTests: spec.expectTests === true, + }; + return createHash("sha256").update(JSON.stringify(def)).digest("hex"); +} + +function environmentOf(spec: CheckSpec): EvidenceEnvironment { + return { + platform: process.platform, + arch: process.arch, + runtime: typeof Bun !== "undefined" ? `bun ${Bun.version}` : `node ${process.version}`, + envKeys: Object.keys(spec.env ?? {}).sort(), + }; +} + +/** One output stream: written to its artifact file up to a cap, hashed, and + * kept as a head + rolling tail for the excerpt. */ +class CapturedStream { + readonly #name: "stdout" | "stderr"; + readonly #path: string; + readonly #file: WriteStream; + readonly #hash = createHash("sha256"); + readonly #max: number; + #bytes = 0; + #written = 0; + #head: Buffer[] = []; + #headBytes = 0; + #tail: Buffer[] = []; + #tailBytes = 0; + + constructor(name: "stdout" | "stderr", dir: string, max: number) { + this.#name = name; + this.#path = join(dir, `${name}.log`); + this.#file = createWriteStream(this.#path); + this.#max = max; + } + + push(chunk: Buffer): void { + this.#bytes += chunk.length; + if (this.#written < this.#max) { + const room = this.#max - this.#written; + const part = chunk.length <= room ? chunk : chunk.subarray(0, room); + this.#file.write(part); + this.#hash.update(part); + this.#written += part.length; + } + let rest = chunk; + if (this.#headBytes < EXCERPT_HEAD_BYTES) { + const take = rest.subarray(0, EXCERPT_HEAD_BYTES - this.#headBytes); + this.#head.push(take); + this.#headBytes += take.length; + rest = rest.subarray(take.length); + if (rest.length === 0) return; + } + this.#tail.push(rest); + this.#tailBytes += rest.length; + while (this.#tailBytes - (this.#tail[0]?.length ?? 0) >= EXCERPT_TAIL_BYTES) { + this.#tailBytes -= this.#tail.shift()!.length; + } + } + + /** The excerpt: the head, then — if output was longer — a marker and the tail. */ + excerpt(): string { + const head = Buffer.concat(this.#head).toString("utf8"); + if (this.#tailBytes === 0) return head; + let tail = Buffer.concat(this.#tail); + let skipped = this.#bytes - this.#headBytes - tail.length; + if (tail.length > EXCERPT_TAIL_BYTES) { + skipped += tail.length - EXCERPT_TAIL_BYTES; + tail = tail.subarray(tail.length - EXCERPT_TAIL_BYTES); + } + const marker = skipped > 0 ? `\n… [${skipped} bytes omitted — full output in the ${this.#name} artifact] …\n` : ""; + return head + marker + tail.toString("utf8"); + } + + /** Flush the artifact file and describe it. */ + async close(): Promise { + await new Promise((res) => this.#file.end(res)); + return { + name: this.#name, + path: this.#path, + sha256: this.#hash.digest("hex"), + bytes: this.#written, + truncated: this.#bytes > this.#written, + }; + } +} + +/** Signal the check's whole process group; a group that is already gone is fine. */ +function killGroup(pid: number, signal: NodeJS.Signals): void { + try { + process.kill(-pid, signal); + } catch { + // ESRCH: every member already exited. + } +} + +/** + * Run `spec` in `opts.cwd` and classify the result. Never throws for a check + * that failed to run — that is `error` evidence — so a caller always gets a + * record to store. + */ +export async function runCheck(spec: CheckSpec, opts: RunCheckOptions): Promise { + const startedAt = Date.now(); + const base = { + criterionId: spec.id, + verifierDigest: verifierDigest(spec), + environment: environmentOf(spec), + isolation: "process" as const, + command: spec.command, + }; + const finish = ( + fields: Pick & + Partial>, + ): CheckOutcome => ({ + ...base, + stdoutExcerpt: "", + stderrExcerpt: "", + artifacts: [], + startedAt, + endedAt: Date.now(), + ...fields, + }); + + if (opts.signal?.aborted) { + return finish({ result: "cancelled", reason: "cancelled before it started", exitCode: null, signal: null, timedOut: false }); + } + + try { + mkdirSync(opts.artifactDir, { recursive: true }); + } catch (err) { + return finish({ + result: "error", + reason: `could not create the artifact directory: ${errText(err)}`, + exitCode: null, + signal: null, + timedOut: false, + }); + } + + const max = opts.maxArtifactBytes ?? DEFAULT_MAX_ARTIFACT_BYTES; + const out = new CapturedStream("stdout", opts.artifactDir, max); + const err = new CapturedStream("stderr", opts.artifactDir, max); + const grace = opts.killGraceMs ?? DEFAULT_KILL_GRACE_MS; + + const child = spawn("sh", ["-c", spec.command], { + cwd: opts.cwd, + env: { ...process.env, ...(spec.env ?? {}) }, + stdio: ["ignore", "pipe", "pipe"], + // Its own process group, so killGroup reaches everything it spawns. + detached: true, + }); + child.stdout?.on("data", (c: Buffer) => out.push(c)); + child.stderr?.on("data", (c: Buffer) => err.push(c)); + // Listened for from the start: `close` can fire in the same tick as `exit`. + const closed = new Promise((res) => child.once("close", () => res())); + + let stoppedBy: "timeout" | "cancel" | null = null; + const stop = (why: "timeout" | "cancel"): void => { + if (stoppedBy || child.pid === undefined) return; + stoppedBy = why; + killGroup(child.pid, "SIGTERM"); + setTimeout(() => killGroup(child.pid!, "SIGKILL"), grace).unref(); + }; + const timer = setTimeout(() => stop("timeout"), spec.timeoutMs); + const onAbort = (): void => stop("cancel"); + opts.signal?.addEventListener("abort", onAbort, { once: true }); + + const exit = await new Promise<{ code: number | null; signal: NodeJS.Signals | null; spawnError?: Error }>( + (res) => { + child.once("error", (e) => res({ code: null, signal: null, spawnError: e })); + child.once("exit", (code, signal) => res({ code, signal })); + }, + ); + clearTimeout(timer); + opts.signal?.removeEventListener("abort", onAbort); + + // The check is over; anything it left running in its group goes too. + if (child.pid !== undefined) killGroup(child.pid, "SIGKILL"); + let drainTimer: ReturnType | undefined; + const drained = new Promise((res) => { + drainTimer = setTimeout(res, DRAIN_WAIT_MS); + }); + await Promise.race([closed, drained]); + clearTimeout(drainTimer); + child.stdout?.destroy(); + child.stderr?.destroy(); + + const artifacts = [await out.close(), await err.close()]; + const stdoutExcerpt = out.excerpt(); + const stderrExcerpt = err.excerpt(); + const captured = { stdoutExcerpt, stderrExcerpt, artifacts }; + const signal = exit.signal ?? null; + + if (exit.spawnError) { + return finish({ + result: "error", + reason: `could not start the check: ${exit.spawnError.message}`, + exitCode: null, + signal: null, + timedOut: false, + ...captured, + }); + } + if (stoppedBy === "cancel") { + return finish({ result: "cancelled", reason: "cancelled while running", exitCode: exit.code, signal, timedOut: false, ...captured }); + } + if (stoppedBy === "timeout") { + return finish({ + result: "fail", + reason: `did not finish within ${spec.timeoutMs}ms and was stopped`, + exitCode: exit.code, + signal, + timedOut: true, + ...captured, + }); + } + return finish({ ...classifyExit(spec, exit.code, signal, `${stdoutExcerpt}\n${stderrExcerpt}`), ...captured }); +} + +function classifyExit( + spec: CheckSpec, + code: number | null, + signal: NodeJS.Signals | null, + output: string, +): Pick { + const at = { exitCode: code, signal, timedOut: false }; + const result = (r: EvidenceResult, reason: string) => ({ result: r, reason, ...at }); + if (signal) return result("fail", `was killed by ${signal}`); + if (code === 126 || code === 127) { + return result("error", code === 127 ? "command not found (exit 127)" : "command not executable (exit 126)"); + } + if (code !== 0) return result("fail", `exited with code ${code}`); + if (spec.expectTests) { + const runner = detectZeroTests(output); + if (runner) return result("inconclusive", `exited 0 but ran no tests (${runner} reported an empty selection)`); + } + return result("pass", "exited 0"); +} + +function errText(err: unknown): string { + return err instanceof Error ? err.message : String(err); +} diff --git a/src/daemon/evidence/service.test.ts b/src/daemon/evidence/service.test.ts new file mode 100644 index 00000000..469508b6 --- /dev/null +++ b/src/daemon/evidence/service.test.ts @@ -0,0 +1,75 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { Database } from "bun:sqlite"; +import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { EvidenceService, ephemeralCheck } from "./service"; +import { EvidenceStore } from "./store"; + +const TENANT = { accountId: "acct-1", projectId: "proj-1" }; +const OTHER = { accountId: "acct-2", projectId: "proj-1" }; + +let dir: string; +beforeEach(() => { + dir = mkdtempSync(join(tmpdir(), "codeoid-evidence-svc-")); +}); +afterEach(() => { + rmSync(dir, { recursive: true, force: true }); +}); + +const ctx = { tenant: TENANT, ownerKind: "pipeline" as const, ownerId: "pipe-1" }; + +describe("EvidenceService", () => { + test("a recorded failure survives reopening the database", async () => { + const dbPath = join(dir, "db.sqlite"); + const svc = new EvidenceService(new EvidenceStore(dbPath), join(dir, "evidence")); + const { record } = await svc.run({ id: "exit:test", command: "echo boom >&2; exit 1", timeoutMs: 5_000 }, dir, { + ...ctx, + candidateDigest: "tree-abc", + }); + expect(record?.result).toBe("fail"); + + const reopened = new EvidenceStore(new Database(dbPath)); + const back = reopened.get(record!.id, TENANT)!; + expect(back.result).toBe("fail"); + expect(back.candidateDigest).toBe("tree-abc"); + expect(back.stderrExcerpt.trim()).toBe("boom"); + expect(back.exitCode).toBe(1); + const stderr = back.artifacts.find((a) => a.name === "stderr")!; + expect(readFileSync(stderr.path, "utf8").trim()).toBe("boom"); + }); + + test("another tenant cannot read a record, and an unknown id looks the same", async () => { + const svc = new EvidenceService(new EvidenceStore(":memory:"), join(dir, "evidence")); + const { record } = await svc.run({ id: "c", command: "true", timeoutMs: 5_000 }, dir, ctx); + expect(svc.get(record!.id, TENANT)?.id).toBe(record!.id); + expect(svc.get(record!.id, OTHER)).toBeNull(); + expect(svc.get("ev_missing", TENANT)).toBeNull(); + }); + + test("records list per owner, in order, within the tenant", async () => { + const svc = new EvidenceService(new EvidenceStore(":memory:"), join(dir, "evidence")); + const first = await svc.run({ id: "a", command: "true", timeoutMs: 5_000 }, dir, ctx); + const second = await svc.run({ id: "b", command: "false", timeoutMs: 5_000 }, dir, ctx); + await svc.run({ id: "c", command: "true", timeoutMs: 5_000 }, dir, { ...ctx, ownerId: "pipe-2" }); + expect(svc.listForOwner("pipeline", "pipe-1", TENANT).map((r) => r.id)).toEqual([first.record!.id, second.record!.id]); + expect(svc.listForOwner("pipeline", "pipe-1", OTHER)).toEqual([]); + }); + + test("an owner id cannot place artifacts outside the artifact root", async () => { + const root = join(dir, "evidence"); + const svc = new EvidenceService(new EvidenceStore(":memory:"), root); + const { record } = await svc.run({ id: "c", command: "echo hi", timeoutMs: 5_000 }, dir, { + ...ctx, + ownerId: "../../escape", + }); + for (const a of record!.artifacts) expect(a.path.startsWith(`${root}/`)).toBe(true); + }); + + test("an unrecorded check leaves no files behind", async () => { + const { outcome, record } = await ephemeralCheck({ id: "c", command: "echo hi", timeoutMs: 5_000 }, dir); + expect(record).toBeUndefined(); + expect(outcome.stdoutExcerpt.trim()).toBe("hi"); + for (const a of outcome.artifacts) expect(existsSync(a.path)).toBe(false); + }); +}); diff --git a/src/daemon/evidence/service.ts b/src/daemon/evidence/service.ts new file mode 100644 index 00000000..b2b40643 --- /dev/null +++ b/src/daemon/evidence/service.ts @@ -0,0 +1,92 @@ +/** + * Runs checks and records what they did. The one entry point pipelines' gates + * and (later) loops use, so every executed check leaves the same evidence. + */ + +import { randomUUID } from "node:crypto"; +import { rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { runCheck } from "./runner.js"; +import type { EvidenceStore, EvidenceTenant } from "./store.js"; +import type { CheckOutcome, CheckSpec, EvidenceOwnerKind, EvidenceRecord } from "./types.js"; + +/** Who the evidence is for and what it judged. */ +export interface EvidenceContext { + tenant: EvidenceTenant; + ownerKind: EvidenceOwnerKind; + ownerId: string; + /** The tree the check judged, when the caller pinned one. */ + candidateDigest?: string | null; + signal?: AbortSignal; +} + +/** A check run through the service: its outcome, and the stored record when + * the run was recorded. */ +export interface CheckRun { + outcome: CheckOutcome; + record?: EvidenceRecord; +} + +/** The narrow surface a gate needs: run a check in a directory. */ +export type CheckRunner = (spec: CheckSpec, cwd: string) => Promise; + +export class EvidenceService { + readonly #store: EvidenceStore; + readonly #artifactRoot: string; + + /** `artifactRoot` holds one directory per check's full output. */ + constructor(store: EvidenceStore, artifactRoot: string) { + this.#store = store; + this.#artifactRoot = artifactRoot; + } + + /** Run `spec` in `cwd` and persist the result. */ + async run(spec: CheckSpec, cwd: string, ctx: EvidenceContext): Promise { + const artifactDir = join(this.#artifactRoot, ctx.ownerKind, safeSegment(ctx.ownerId), randomUUID()); + const outcome = await runCheck(spec, { cwd, artifactDir, signal: ctx.signal }); + const record = this.#store.insert({ + ...outcome, + accountId: ctx.tenant.accountId, + projectId: ctx.tenant.projectId, + ownerKind: ctx.ownerKind, + ownerId: ctx.ownerId, + candidateDigest: ctx.candidateDigest ?? null, + }); + return { outcome, record }; + } + + /** A runner bound to one owner — what a gate is handed. */ + runnerFor(ctx: EvidenceContext): CheckRunner { + return (spec, cwd) => this.run(spec, cwd, ctx); + } + + get(id: string, tenant: EvidenceTenant): EvidenceRecord | null { + return this.#store.get(id, tenant); + } + + listForOwner(ownerKind: EvidenceOwnerKind, ownerId: string, tenant: EvidenceTenant): EvidenceRecord[] { + return this.#store.listForOwner(ownerKind, ownerId, tenant); + } +} + +/** + * Run a check without recording it, for callers with no evidence store (tests, + * a pipeline manager built without one). Output goes to a temporary directory + * that is removed afterwards; the outcome's excerpts survive, its artifact + * paths do not. + */ +export const ephemeralCheck: CheckRunner = async (spec, cwd) => { + const artifactDir = join(tmpdir(), "codeoid-evidence", randomUUID()); + try { + return { outcome: await runCheck(spec, { cwd, artifactDir }) }; + } finally { + rmSync(artifactDir, { recursive: true, force: true }); + } +}; + +/** An owner id as one path segment: ids are daemon-minted, but a directory + * name must never be able to climb out of the artifact root. */ +function safeSegment(id: string): string { + return id.replace(/[^A-Za-z0-9._-]/g, "_").slice(0, 128) || "_"; +} diff --git a/src/daemon/evidence/store.ts b/src/daemon/evidence/store.ts new file mode 100644 index 00000000..6bd9467a --- /dev/null +++ b/src/daemon/evidence/store.ts @@ -0,0 +1,177 @@ +/** + * Durable evidence records. Shares the daemon's SQLite connection (or opens its + * own in tests) and migrates with the same `CREATE TABLE IF NOT EXISTS` + * convention as the routine and pipeline stores. + * + * Records are append-only: evidence describes what happened, so it is never + * edited. Excerpts are bounded by the runner; the full output lives in artifact + * files whose path, size, and hash are recorded here. + */ + +import { Database } from "bun:sqlite"; +import { randomUUID } from "node:crypto"; +import { BUSY_TIMEOUT_MS } from "../store.js"; +import type { + EvidenceArtifact, + EvidenceEnvironment, + EvidenceIsolation, + EvidenceOwnerKind, + EvidenceRecord, + EvidenceResult, +} from "./types.js"; + +interface EvidenceDbRow { + id: string; + account_id: string; + project_id: string; + owner_kind: string; + owner_id: string; + criterion_id: string; + candidate_digest: string | null; + verifier_digest: string; + environment_json: string; + isolation: string; + command: string; + result: string; + reason: string; + exit_code: number | null; + signal: string | null; + timed_out: number; + started_at: number; + ended_at: number; + stdout_excerpt: string; + stderr_excerpt: string; + artifacts_json: string; +} + +export interface EvidenceTenant { + accountId: string; + projectId: string; +} + +export class EvidenceStore { + readonly #db: Database; + + constructor(db: Database | string) { + this.#db = typeof db === "string" ? new Database(db, { create: true }) : db; + this.#db.exec(`PRAGMA busy_timeout = ${BUSY_TIMEOUT_MS}`); + this.#db.exec("PRAGMA journal_mode = WAL"); + this.#migrate(); + } + + #migrate(): void { + this.#db.exec(` + CREATE TABLE IF NOT EXISTS evidence ( + id TEXT PRIMARY KEY, + account_id TEXT NOT NULL, + project_id TEXT NOT NULL, + owner_kind TEXT NOT NULL, + owner_id TEXT NOT NULL, + criterion_id TEXT NOT NULL, + candidate_digest TEXT, + verifier_digest TEXT NOT NULL, + environment_json TEXT NOT NULL, + isolation TEXT NOT NULL, + command TEXT NOT NULL, + result TEXT NOT NULL, + reason TEXT NOT NULL, + exit_code INTEGER, + signal TEXT, + timed_out INTEGER NOT NULL DEFAULT 0, + started_at INTEGER NOT NULL, + ended_at INTEGER NOT NULL, + stdout_excerpt TEXT NOT NULL, + stderr_excerpt TEXT NOT NULL, + artifacts_json TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_evidence_owner + ON evidence(owner_kind, owner_id, started_at); + `); + } + + /** Persist a record and return it with its id. */ + insert(record: Omit): EvidenceRecord { + const id = `ev_${randomUUID()}`; + this.#db + .prepare( + `INSERT INTO evidence ( + id, account_id, project_id, owner_kind, owner_id, criterion_id, + candidate_digest, verifier_digest, environment_json, isolation, command, + result, reason, exit_code, signal, timed_out, started_at, ended_at, + stdout_excerpt, stderr_excerpt, artifacts_json + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ) + .run( + id, + record.accountId, + record.projectId, + record.ownerKind, + record.ownerId, + record.criterionId, + record.candidateDigest, + record.verifierDigest, + JSON.stringify(record.environment), + record.isolation, + record.command, + record.result, + record.reason, + record.exitCode, + record.signal, + record.timedOut ? 1 : 0, + record.startedAt, + record.endedAt, + record.stdoutExcerpt, + record.stderrExcerpt, + JSON.stringify(record.artifacts), + ); + return { ...record, id }; + } + + /** A record, only if it belongs to `tenant`. Unknown and foreign ids are the + * same `null`, so an id from another tenant reveals nothing. */ + get(id: string, tenant: EvidenceTenant): EvidenceRecord | null { + const row = this.#db + .prepare("SELECT * FROM evidence WHERE id = ? AND account_id = ? AND project_id = ?") + .get(id, tenant.accountId, tenant.projectId) as EvidenceDbRow | null; + return row ? toRecord(row) : null; + } + + /** Every record for one owner, oldest first. */ + listForOwner(ownerKind: EvidenceOwnerKind, ownerId: string, tenant: EvidenceTenant): EvidenceRecord[] { + return ( + this.#db + .prepare( + `SELECT * FROM evidence + WHERE owner_kind = ? AND owner_id = ? AND account_id = ? AND project_id = ? + ORDER BY started_at, rowid`, + ) + .all(ownerKind, ownerId, tenant.accountId, tenant.projectId) as EvidenceDbRow[] + ).map(toRecord); + } +} + +function toRecord(row: EvidenceDbRow): EvidenceRecord { + return { + id: row.id, + accountId: row.account_id, + projectId: row.project_id, + ownerKind: row.owner_kind as EvidenceOwnerKind, + ownerId: row.owner_id, + criterionId: row.criterion_id, + candidateDigest: row.candidate_digest, + verifierDigest: row.verifier_digest, + environment: JSON.parse(row.environment_json) as EvidenceEnvironment, + isolation: row.isolation as EvidenceIsolation, + command: row.command, + result: row.result as EvidenceResult, + reason: row.reason, + exitCode: row.exit_code, + signal: row.signal, + timedOut: row.timed_out === 1, + startedAt: row.started_at, + endedAt: row.ended_at, + stdoutExcerpt: row.stdout_excerpt, + stderrExcerpt: row.stderr_excerpt, + artifacts: JSON.parse(row.artifacts_json) as EvidenceArtifact[], + }; +} diff --git a/src/daemon/evidence/types.ts b/src/daemon/evidence/types.ts new file mode 100644 index 00000000..42f18088 --- /dev/null +++ b/src/daemon/evidence/types.ts @@ -0,0 +1,111 @@ +/** + * Evidence: the durable, revision-bound record of one executed check + * (docs/loop-engineering-design.md §7). + * + * A check's verdict is never a model's claim. It is what a command did, on a + * named tree, under a pinned definition, captured by the daemon. Pipelines' + * deterministic gates record evidence today; loops build their acceptance + * decisions on it. + */ + +/** + * What a check established. + * - `pass` — it ran to completion and succeeded. + * - `fail` — it ran and the candidate did not satisfy it, including running + * past its time bound: a hung test is the candidate's problem to fix. + * - `inconclusive` — it exited 0 but established nothing (zero tests selected, + * no test files). Never a pass. + * - `error` — it could not be run or its result could not be read (missing + * binary, spawn failure, unreadable output). Infrastructure, not the candidate. + * - `cancelled` — stopped on request before it finished. + */ +export type EvidenceResult = "pass" | "fail" | "inconclusive" | "error" | "cancelled"; + +/** + * How strongly the check was separated from the code it judged. + * `process` runs it as an ordinary child process in a directory; `sandbox` + * runs it in an isolated namespace with the evaluator mounted read-only. + */ +export type EvidenceIsolation = "process" | "sandbox"; + +/** A check to run. The definition is data: its digest pins it in the record. */ +export interface CheckSpec { + /** Stable id of the criterion this check evaluates (e.g. `exit:test`). */ + id: string; + /** Human label, e.g. `test`. */ + label?: string; + /** Shell command, run as `sh -c `. */ + command: string; + /** Hard bound; past it the process group is killed and the result is `fail`. */ + timeoutMs: number; + /** Extra environment for the check, merged over the daemon's. Values are not + * recorded — only their names, so a credential never lands in evidence. */ + env?: Record; + /** + * The check is a test run: exiting 0 after selecting no tests is + * `inconclusive`, not `pass`. Set for test probes; leave unset for builds + * and linters, which legitimately print nothing. + */ + expectTests?: boolean; +} + +/** Who a piece of evidence belongs to. Reads are authorized through the owner. */ +export type EvidenceOwnerKind = "pipeline" | "loop"; + +/** One captured output stream, stored in full as an artifact (up to a cap). */ +export interface EvidenceArtifact { + name: "stdout" | "stderr"; + path: string; + sha256: string; + bytes: number; + /** The stream exceeded the artifact cap; the file holds its first bytes. */ + truncated: boolean; +} + +/** The environment a check ran in — enough to tell two runs apart. */ +export interface EvidenceEnvironment { + platform: string; + arch: string; + runtime: string; + /** Names (never values) of the variables the spec added. */ + envKeys: string[]; +} + +export interface EvidenceRecord { + id: string; + accountId: string; + projectId: string; + ownerKind: EvidenceOwnerKind; + ownerId: string; + criterionId: string; + /** Git tree (or checkpoint) the check judged; null when the workspace had no + * snapshot — such evidence certifies no particular revision. */ + candidateDigest: string | null; + /** sha256 of the check definition (command, timeout, env names, flags). */ + verifierDigest: string; + environment: EvidenceEnvironment; + isolation: EvidenceIsolation; + command: string; + result: EvidenceResult; + /** One sentence a human or a worker can act on. */ + reason: string; + exitCode: number | null; + signal: string | null; + timedOut: boolean; + startedAt: number; + endedAt: number; + /** Bounded excerpts for display and worker feedback; the full streams are + * in `artifacts`. */ + stdoutExcerpt: string; + stderrExcerpt: string; + artifacts: EvidenceArtifact[]; +} + +/** What a check run produces before it is attributed and stored. */ +export type CheckOutcome = Omit< + EvidenceRecord, + "id" | "accountId" | "projectId" | "ownerKind" | "ownerId" | "candidateDigest" | "artifacts" +> & { + /** Full output, kept by the runner until the caller places the artifacts. */ + artifacts: EvidenceArtifact[]; +}; diff --git a/src/daemon/evidence/wire.ts b/src/daemon/evidence/wire.ts new file mode 100644 index 00000000..350c97fa --- /dev/null +++ b/src/daemon/evidence/wire.ts @@ -0,0 +1,31 @@ +/** + * Evidence on the wire. Artifact paths stay in the daemon: a client sees each + * stream's size and hash, and its excerpt. + */ + +import type { EvidenceWire } from "../../protocol/types.js"; +import type { EvidenceRecord } from "./types.js"; + +export function evidenceToWire(r: EvidenceRecord): EvidenceWire { + return { + id: r.id, + ownerKind: r.ownerKind, + ownerId: r.ownerId, + criterionId: r.criterionId, + candidateDigest: r.candidateDigest, + verifierDigest: r.verifierDigest, + isolation: r.isolation, + command: r.command, + result: r.result, + reason: r.reason, + exitCode: r.exitCode, + signal: r.signal, + timedOut: r.timedOut, + startedAt: new Date(r.startedAt).toISOString(), + endedAt: new Date(r.endedAt).toISOString(), + environment: r.environment, + stdoutExcerpt: r.stdoutExcerpt, + stderrExcerpt: r.stderrExcerpt, + artifacts: r.artifacts.map(({ name, sha256, bytes, truncated }) => ({ name, sha256, bytes, truncated })), + }; +} diff --git a/src/daemon/pipeline/engine.ts b/src/daemon/pipeline/engine.ts index 51e345dd..3aff55f9 100644 --- a/src/daemon/pipeline/engine.ts +++ b/src/daemon/pipeline/engine.ts @@ -15,6 +15,7 @@ * loop persists between legs and resumes after a restart exactly where it was. */ +import type { EvidenceService } from "../evidence/service"; import type { GateVerdict, PhaseCtx, @@ -56,11 +57,21 @@ const MAX_STEPS = 10_000; const clone = (x: T): T => structuredClone(x); const now = (): number => Date.now(); +/** Evidence kept per phase in the run state; older ids stay in the store. */ +const PHASE_EVIDENCE_MAX = 50; + +export interface PipelineEngineOptions { + /** Records every check a gate executes. Absent, checks run unrecorded. */ + evidence?: EvidenceService; +} + export class PipelineEngine { #registries: PipelineRegistries; + #evidence?: EvidenceService; - constructor(registries: PipelineRegistries) { + constructor(registries: PipelineRegistries, options: PipelineEngineOptions = {}) { this.#registries = registries; + this.#evidence = options.evidence; } /** Advance the active phase by one unit of work and return a new state. */ @@ -450,8 +461,19 @@ export class PipelineEngine { // (`always` / `manual`) or directly registered gate — see scoped.ts. const g = resolveScoped(this.#registries.gates, pipeline.packId, id); if (!g) return { pass: false, reason: `unknown ${at} gate "${id}"` }; + const runCheck = this.#evidence?.runnerFor({ + tenant: { accountId: pipeline.accountId, projectId: pipeline.projectId }, + ownerKind: "pipeline", + ownerId: pipeline.id, + }); try { - return await g.evaluate({ pipeline: clone(pipeline), phase }); + const verdict = await g.evaluate({ pipeline: clone(pipeline), phase, ...(runCheck ? { runCheck } : {}) }); + // Every gate runs for the active phase, so its evidence belongs there. + const active = pipeline.phases[pipeline.cursor]; + if (active && verdict.evidenceIds?.length) { + active.evidence = [...(active.evidence ?? []), ...verdict.evidenceIds].slice(-PHASE_EVIDENCE_MAX); + } + return verdict; } catch (err) { // A throwing gate is a failing verdict, not a crash — same reasoning as // the phase kind above: never leave the pipeline stuck mid-advance. diff --git a/src/daemon/pipeline/gate-probes.test.ts b/src/daemon/pipeline/gate-probes.test.ts index 5030eef0..690c0938 100644 --- a/src/daemon/pipeline/gate-probes.test.ts +++ b/src/daemon/pipeline/gate-probes.test.ts @@ -142,3 +142,26 @@ describe("workdir guard", () => { expect(v.reason).toContain("needs a workdir"); }); }); + +describe("probe evidence", () => { + test("a test probe that ran no tests fails as inconclusive, and its check is recorded", async () => { + const d = tmp(); + writeFileSync(join(d, "package.json"), JSON.stringify({ scripts: { test: "echo 'Ran 0 tests across 0 files.'" } })); + const seen: string[] = []; + const c: GateCtx = { + ...ctx(d), + runCheck: async (spec, cwd) => { + seen.push(`${spec.id}@${cwd}:${spec.expectTests}`); + const { ephemeralCheck } = await import("../evidence/service"); + const run = await ephemeralCheck(spec, cwd); + return { ...run, record: { ...run.outcome, id: "ev_1" } as never }; + }, + }; + const v = await buildProbeGate("g", { type: "test" }, "exit", true).evaluate(c); + expect(v.pass).toBe(false); + expect(v.reason).toContain("inconclusive"); + expect(v.evidenceIds).toEqual(["ev_1"]); + expect(seen).toEqual([`x:g:test@${d}:true`]); + }); +}); + diff --git a/src/daemon/pipeline/gate-probes.ts b/src/daemon/pipeline/gate-probes.ts index 029999fb..a53140c0 100644 --- a/src/daemon/pipeline/gate-probes.ts +++ b/src/daemon/pipeline/gate-probes.ts @@ -21,6 +21,8 @@ */ import { existsSync, readFileSync } from "node:fs"; +import { ephemeralCheck } from "../evidence/service"; +import type { CheckOutcome } from "../evidence/types"; import type { GateCtx, GatePlugin, GateVerdict } from "./interface"; /** A probe path/glob must stay WITHIN the run's workdir: it is relative and has @@ -59,33 +61,29 @@ export interface ProbeSpec { const errText = (err: unknown): string => (err instanceof Error ? err.message : String(err)); -/** Run a shell command in `cwd`, bounded by a timeout; resolve its exit code (or - * a synthetic non-zero on timeout/spawn failure). Mirrors the command gate's - * spawn shape (stdout/stderr ignored — a probe's verdict is the exit code). */ -async function runCommand(cmd: string, cwd: string): Promise<{ code: number; timedOut: boolean }> { - let proc: ReturnType; - try { - proc = Bun.spawn(["sh", "-c", cmd], { cwd, stdout: "ignore", stderr: "ignore", stdin: "ignore" }); - } catch { - return { code: 127, timedOut: false }; - } - const TIMED_OUT = Symbol("timeout"); - let timer: ReturnType | undefined; - try { - const result = await Promise.race([ - proc.exited, - new Promise((res) => { - timer = setTimeout(() => res(TIMED_OUT), GATE_PROBE_TIMEOUT_MS); - }), - ]); - if (result === TIMED_OUT) { - proc.kill(); - return { code: 124, timedOut: true }; - } - return { code: result, timedOut: false }; - } finally { - clearTimeout(timer); - } +/** Run one probe command through the gate's check runner (recording evidence + * when the run has an evidence store), bounded by GATE_PROBE_TIMEOUT_MS. */ +async function runProbe( + ctx: GateCtx, + id: string, + label: string, + cmd: string, + cwd: string, +): Promise<{ outcome: CheckOutcome; evidenceId?: string }> { + const run = ctx.runCheck ?? ephemeralCheck; + const { outcome, record } = await run( + { id: `${ctx.phase.id}:${id}:${label}`, label, command: cmd, timeoutMs: GATE_PROBE_TIMEOUT_MS, expectTests: label === "test" }, + cwd, + ); + return { outcome, ...(record ? { evidenceId: record.id } : {}) }; +} + +/** The diagnostic tail a failed check contributes to its gate's reason — what + * the phase's next attempt is shown. Bounded: the full output is evidence. */ +export function diagnosticTail(outcome: CheckOutcome, max = 1_500): string { + const text = [outcome.stderrExcerpt.trim(), outcome.stdoutExcerpt.trim()].filter(Boolean).join("\n"); + if (!text) return ""; + return text.length > max ? `…${text.slice(text.length - max)}` : text; } /** Detected build/test/lint commands for a workspace, keyed by the manifest we @@ -213,10 +211,11 @@ export function buildProbeGate( } } if (spec.type === "git-diff-nonempty") { - const { code } = await runCommand("test -n \"$(git status --porcelain)\"", workdir); - return code === 0 - ? { pass: true } - : { pass: false, reason: `probe "${id}": the working tree has no changes` }; + const { outcome, evidenceId } = await runProbe(ctx, id, "git-diff", "test -n \"$(git status --porcelain)\"", workdir); + const evidenceIds = evidenceId ? [evidenceId] : []; + return outcome.result === "pass" + ? { pass: true, evidenceIds } + : { pass: false, reason: `probe "${id}": the working tree has no changes`, evidenceIds }; } // ── Execution probes (trust-gated) ──────────────────────────────────── @@ -234,18 +233,20 @@ export function buildProbeGate( reason: `probe "${id}" (${spec.type}): no ${spec.type} command auto-derived for this workspace (unrecognized ecosystem)`, }; } + const evidenceIds: string[] = []; for (const { label, cmd } of cmds) { - const { code, timedOut } = await runCommand(cmd, workdir); - if (code !== 0) { + const { outcome, evidenceId } = await runProbe(ctx, id, label, cmd, workdir); + if (evidenceId) evidenceIds.push(evidenceId); + if (outcome.result !== "pass") { + const tail = diagnosticTail(outcome); return { pass: false, - reason: timedOut - ? `probe "${id}": ${label} timed out after ${GATE_PROBE_TIMEOUT_MS}ms (${cmd})` - : `probe "${id}": ${label} failed (exit ${code}): ${cmd}`, + reason: `probe "${id}": ${label} ${outcome.result} — ${outcome.reason}: ${cmd}${tail ? `\n${tail}` : ""}`, + evidenceIds, }; } } - return { pass: true }; + return { pass: true, evidenceIds }; }, }; } diff --git a/src/daemon/pipeline/interface.ts b/src/daemon/pipeline/interface.ts index ac34c69e..a20e0f34 100644 --- a/src/daemon/pipeline/interface.ts +++ b/src/daemon/pipeline/interface.ts @@ -10,6 +10,7 @@ import type { ResolvedFrom, RoleModelSource } from "./binding"; import type { FindingsLoopState, FindingsSpec } from "./findings"; +import type { CheckRunner } from "../evidence/service"; // ── Phase definition (the static plan) ──────────────────────────────────── @@ -123,6 +124,8 @@ export interface PipelinePhase { /** The findings loop's persisted state (rounds, dispositions, what runs * next) — present only for a phase that declares `def.findings`. */ findings?: FindingsLoopState; + /** Evidence ids recorded by this phase's gates, oldest first (bounded). */ + evidence?: string[]; } /** The full, daemon-owned pipeline state — the source of truth persisted per @@ -205,12 +208,17 @@ export interface PhaseKind { export interface GateCtx { pipeline: PipelineState; phase: PhaseDef; + /** Runs a check and records its evidence, bound to this run. Gates that + * execute commands go through it; absent, they run unrecorded. */ + runCheck?: CheckRunner; } export interface GateVerdict { pass: boolean; reason?: string; questions?: string[]; + /** Evidence the gate recorded for this verdict, in the order it ran. */ + evidenceIds?: string[]; } /** A gate: a pass/fail predicate positioned at phase entry (grounding) or exit diff --git a/src/daemon/pipeline/manager.ts b/src/daemon/pipeline/manager.ts index 99ce03be..b8b9321c 100644 --- a/src/daemon/pipeline/manager.ts +++ b/src/daemon/pipeline/manager.ts @@ -15,6 +15,7 @@ import { randomUUID } from "node:crypto"; // explicit --model on session create (docs/role-model-binding.md §5). No // Session dependency: models.ts is a pure catalog module. import { resolveModelIdForProvider } from "../models.js"; +import type { EvidenceService } from "../evidence/service"; import { type ModelBinding, type ModelBindingConfig, resolveBinding } from "./binding"; import { registerBuiltins } from "./builtin"; import { PipelineEngine } from "./engine"; @@ -139,6 +140,8 @@ export interface PipelineManagerOptions { /** Backend seam for prompt/slash skills — enables the "skill" phase kind to * drive a real session turn. Omit for pure `noop`/`fn`-skill pipelines. */ runner?: PhaseRunner; + /** Records the evidence of every check a gate executes. */ + evidence?: EvidenceService; } export class PipelineManager { @@ -152,7 +155,7 @@ export class PipelineManager { constructor(store: PipelineStore, options: PipelineManagerOptions = {}) { this.#store = store; this.#registries = options.registries ?? defaultRegistries(options.runner); - this.#engine = new PipelineEngine(this.#registries); + this.#engine = new PipelineEngine(this.#registries, { evidence: options.evidence }); this.resume(); } diff --git a/src/daemon/pipeline/pack.test.ts b/src/daemon/pipeline/pack.test.ts index e103830a..3a876c30 100644 --- a/src/daemon/pipeline/pack.test.ts +++ b/src/daemon/pipeline/pack.test.ts @@ -11,6 +11,8 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import { LIMITS } from "../../protocol/types.js"; import { ARTIFACT_KIND_MAX } from "../blackboard/types.js"; +import { EvidenceService } from "../evidence/service"; +import { EvidenceStore } from "../evidence/store"; import { createRegistries } from "./registry"; import type { PhaseDef } from "./interface"; import { PipelineManager } from "./manager"; @@ -373,6 +375,34 @@ phases: } }); + test("a trusted command gate records revision-free evidence on the phase, with diagnostics in the reason", async () => { + const db = new Database(":memory:"); + const root = mkdtempSync(join(tmpdir(), "pack-evidence-")); + const evidence = new EvidenceService(new EvidenceStore(db), root); + const mgr = new PipelineManager(new PipelineStore(db), { evidence }); + mgr.installPack( + loadPack(noopPack(' - id: red\n kind: command\n run: "echo assertion failed: want 2 got 3 >&2; exit 1"', "red"), { + trusted: true, + }), + ); + const p = mgr.create({ name: "run", pack: "noop-pack", ...tenant }); + const halted = await mgr.advance(p.id); + const phase = halted.phases[0]; + expect(phase.evidence).toHaveLength(1); + const rec = evidence.get(phase.evidence![0]!, { accountId: tenant.accountId, projectId: tenant.projectId })!; + expect(rec.result).toBe("fail"); + expect(rec.ownerKind).toBe("pipeline"); + expect(rec.ownerId).toBe(p.id); + expect(rec.criterionId).toBe("a:red"); + expect(rec.candidateDigest).toBeNull(); + if (phase.state.status !== "halted") throw new Error("expected halt"); + expect(phase.state.reason).toContain("assertion failed: want 2 got 3"); + // The evidence survives a manager rebuilt over the same database. + const again = new PipelineManager(new PipelineStore(db), { evidence }); + expect(again.get(p.id)?.phases[0].evidence).toEqual(phase.evidence); + rmSync(root, { recursive: true, force: true }); + }); + test("an UNTRUSTED command gate never executes — fails closed (registry default)", async () => { const mgr = new PipelineManager(new PipelineStore(new Database(":memory:"))); // No { trusted } → default false. Even a "true" command must not run. diff --git a/src/daemon/pipeline/pack.ts b/src/daemon/pipeline/pack.ts index 92e41f90..67aaad66 100644 --- a/src/daemon/pipeline/pack.ts +++ b/src/daemon/pipeline/pack.ts @@ -20,7 +20,8 @@ import { renderLedger, SEVERITIES, } from "./findings"; -import { buildProbeGate, probePathEscapes, type ProbeSpec } from "./gate-probes"; +import { ephemeralCheck } from "../evidence/service"; +import { buildProbeGate, diagnosticTail, probePathEscapes, type ProbeSpec } from "./gate-probes"; import type { GatePlugin, Pack, @@ -493,26 +494,19 @@ function buildGate(g: PackManifest["gates"][number], dir: string, trusted: boole at, async evaluate(ctx) { const cwd = ctx.pipeline.workdir ?? dir; - const proc = Bun.spawn(["sh", "-c", run], { cwd, stdout: "ignore", stderr: "ignore" }); - const TIMED_OUT = Symbol("timeout"); - let timer: ReturnType | undefined; - try { - const result = await Promise.race([ - proc.exited, - new Promise((res) => { - timer = setTimeout(() => res(TIMED_OUT), GATE_COMMAND_TIMEOUT_MS); - }), - ]); - if (result === TIMED_OUT) { - proc.kill(); - return { pass: false, reason: `command gate "${g.id}" timed out after ${GATE_COMMAND_TIMEOUT_MS}ms: ${run}` }; - } - return result === 0 - ? { pass: true } - : { pass: false, reason: `command gate "${g.id}" failed (exit ${result}): ${run}` }; - } finally { - clearTimeout(timer); - } + const runner = ctx.runCheck ?? ephemeralCheck; + const { outcome, record } = await runner( + { id: `${ctx.phase.id}:${g.id}`, label: g.id, command: run, timeoutMs: GATE_COMMAND_TIMEOUT_MS }, + cwd, + ); + const evidenceIds = record ? [record.id] : []; + if (outcome.result === "pass") return { pass: true, evidenceIds }; + const tail = diagnosticTail(outcome); + return { + pass: false, + reason: `command gate "${g.id}" ${outcome.result} — ${outcome.reason}: ${run}${tail ? `\n${tail}` : ""}`, + evidenceIds, + }; }, }; } diff --git a/src/daemon/pipeline/wiring.ts b/src/daemon/pipeline/wiring.ts index 5d30c57c..e0b6155e 100644 --- a/src/daemon/pipeline/wiring.ts +++ b/src/daemon/pipeline/wiring.ts @@ -7,6 +7,7 @@ */ import type { Database } from "bun:sqlite"; +import type { EvidenceService } from "../evidence/service"; import { PipelineManager } from "./manager"; import { loadPack } from "./pack"; import type { PhaseRunner } from "./runner"; @@ -32,6 +33,8 @@ export interface PipelineWiringOptions { /** Share this DB handle (the daemon Store's) instead of opening a second * connection to `config.dbPath`. */ db?: Database; + /** Records the evidence of every check a gate executes. */ + evidence?: EvidenceService; } /** @@ -45,7 +48,7 @@ export function createPipelineManagerFromConfig( ): PipelineManager | undefined { if (!config?.pipeline?.enabled) return undefined; const store = opts.db ? new PipelineStore(opts.db) : new PipelineStore(config.dbPath); - const manager = new PipelineManager(store, { runner: opts.runner }); + const manager = new PipelineManager(store, { runner: opts.runner, evidence: opts.evidence }); // Install configured packs. Fail SOFT per pack: a malformed / missing pack // logs and is skipped rather than taking down the daemon boot. for (const p of config.pipeline.packs ?? []) { diff --git a/src/daemon/session-manager.ts b/src/daemon/session-manager.ts index a78b0bb8..ef9e42db 100644 --- a/src/daemon/session-manager.ts +++ b/src/daemon/session-manager.ts @@ -100,6 +100,7 @@ import { type DispatcherHost, } from "./dispatch.js"; import { createPipelineManagerFromConfig } from "./pipeline/wiring.js"; +import { EvidenceService, EvidenceStore, evidenceToWire } from "./evidence/index.js"; import type { PipelineManager } from "./pipeline/manager.js"; import { resolveBinding, type ModelBindingConfig } from "./pipeline/binding.js"; import type { RoleDef } from "./pipeline/pack.js"; @@ -448,6 +449,7 @@ export class SessionManager { #pushService: PushService; /** SDLC pipeline manager — undefined when the pipeline is disabled (default). */ #pipelines?: PipelineManager; + #evidence?: EvidenceService; /** Routines (docs/routines.md); undefined when `routines.enabled` is off. */ #routines?: RoutineService; /** Pack curation surface (registries + install/trust/select) — always present, @@ -597,9 +599,18 @@ export class SessionManager { // pipelines on construction (resume). The runner drives prompt/slash phases // on worker turns; the `() => this` thunk is only dereferenced at run time, // after construction. Undefined when disabled ⇒ the daemon stays dark. + // Evidence of every executed check (docs/loop-engineering-design.md §7), + // in the daemon DB with full output beside it. + if (opts?.config) { + this.#evidence = new EvidenceService( + new EvidenceStore(store.database), + join(dirname(opts.config.dbPath), "evidence"), + ); + } this.#pipelines = createPipelineManagerFromConfig(opts?.config, { runner: new SessionPhaseRunner(() => this), db: store.database, + ...(this.#evidence ? { evidence: this.#evidence } : {}), }); // Routines share the daemon DB. Built only when enabled, so the kill // switch leaves every routine in place but nothing fires. @@ -1356,6 +1367,8 @@ mcpHub: this.#mcpHub, case "routine.token": case "routine.preview": return this.#routineHandle(msg, auth); + case "evidence.get": + return this.#evidenceGet(msg, auth); case "pipeline.create": return this.#pipelineCreate(msg, auth); case "pipeline.list": @@ -5361,6 +5374,7 @@ mcpHub: this.#mcpHub, // Surface the phase's produced output at the boundary halt so the UI // can show "here's what this phase did" next to Approve/Revise/Reject. if (p.lastSummary) w.summary = p.lastSummary; + if (p.evidence?.length) w.evidence = [...p.evidence]; } if (p.feedback && p.feedback.length > 0) w.feedback = p.feedback; // The findings loop (findings.ts): counts + the ledger, so a client can @@ -5485,6 +5499,27 @@ mcpHub: this.#mcpHub, return { type: "pipeline.list.result", requestId: msg.id, pipelines }; } + /** + * One evidence record. Authorized through its owner: a pipeline's evidence + * needs `pipeline:read` and a pipeline the caller can see. Anything the caller + * may not read is `not_found`, so an id reveals nothing across tenants. + */ + #evidenceGet(msg: Extract, auth: AuthContext): DaemonMessage { + const notFound: DaemonMessage = { type: "response.error", requestId: msg.id, error: "Evidence not found", code: "not_found" }; + const record = this.#evidence?.get(msg.evidenceId, { accountId: auth.accountId, projectId: auth.projectId }); + if (!record) return notFound; + if (record.ownerKind === "pipeline") { + const g = this.#pipelineGuard(msg.id, auth, SCOPES.PIPELINE_READ); + if ("error" in g) return g.error; + if (!this.#ownedPipeline(g.pm, record.ownerId, auth)) return notFound; + } else { + // Loop evidence is authorized by the loop service, which doesn't exist + // yet; until it does, nothing else may read it. + return notFound; + } + return { type: "evidence.record", requestId: msg.id, evidence: evidenceToWire(record) }; + } + #pipelineGet(msg: Extract, auth: AuthContext): DaemonMessage { const g = this.#pipelineGuard(msg.id, auth, SCOPES.PIPELINE_READ); if ("error" in g) return g.error; diff --git a/src/terminal/client.ts b/src/terminal/client.ts index 8c998076..44033073 100644 --- a/src/terminal/client.ts +++ b/src/terminal/client.ts @@ -27,6 +27,7 @@ import { acquireToken, TokenError } from "./token.js"; import { sanitizeTerminalOutput } from "../tui/ansi/codes.js"; import { formatPackList, formatPackShow } from "./pack-format.js"; import { formatPipeline, haltedRequestId } from "./pipeline-format.js"; +import { formatEvidence } from "./evidence-format.js"; import { type PendingDialog, parseDialogAnswer } from "./dialog.js"; import { formatRewind, parseUndoArgs } from "./rewind.js"; import { awaitingApproval, compareSettled, formatCompare } from "./compare.js"; @@ -406,6 +407,15 @@ export class TerminalClient { this.#renderPacks(await this.#request({ type: "pipeline.pack.select", id: randomUUID(), packId: id })); } + async evidenceShow(evidenceId: string, opts: { full?: boolean } = {}): Promise { + const resp = await this.#request({ type: "evidence.get", id: randomUUID(), evidenceId }); + if (resp.type !== "evidence.record") { + this.#printError(resp); + return; + } + for (const line of formatEvidence(resp.evidence, opts)) console.log(line); + } + async packShow(id: string, opts: { resolve?: boolean } = {}): Promise { const resp = await this.#request({ type: "pipeline.pack.list", id: randomUUID() }); if (resp.type !== "pipeline.pack.list.result") { diff --git a/src/terminal/evidence-format.test.ts b/src/terminal/evidence-format.test.ts new file mode 100644 index 00000000..6d716d56 --- /dev/null +++ b/src/terminal/evidence-format.test.ts @@ -0,0 +1,43 @@ +import { describe, expect, test } from "bun:test"; +import type { EvidenceWire } from "../protocol/types.js"; +import { formatEvidence } from "./evidence-format"; + +const e: EvidenceWire = { + id: "ev_1", + ownerKind: "pipeline", + ownerId: "p1", + criterionId: "impl:test", + candidateDigest: null, + verifierDigest: "0123456789abcdef0123", + isolation: "process", + command: "bun test", + result: "fail", + reason: "did not finish within 1000ms and was stopped", + exitCode: null, + signal: "SIGTERM", + timedOut: true, + startedAt: "2026-10-25T10:00:00.000Z", + endedAt: "2026-10-25T10:00:01.000Z", + environment: { platform: "linux", arch: "x64", runtime: "bun 1.3.13", envKeys: [] }, + stdoutExcerpt: Array.from({ length: 60 }, (_, i) => `out ${i}`).join("\n"), + stderrExcerpt: "", + artifacts: [{ name: "stdout", sha256: "f".repeat(64), bytes: 420, truncated: true }], +}; + +describe("formatEvidence", () => { + test("states the result, why, and that no tree was pinned", () => { + const text = formatEvidence(e).join("\n"); + expect(text).toContain("ev_1 FAIL did not finish within 1000ms"); + expect(text).toContain("signal SIGTERM, timed out, 1.0s"); + expect(text).toContain("certifies no particular revision"); + expect(text).toContain("420 bytes (truncated at cap)"); + }); + + test("shows the last 40 output lines unless asked for all", () => { + const short = formatEvidence(e).join("\n"); + expect(short).toContain("out 59"); + expect(short).not.toContain("out 0\n"); + expect(short).toContain("--full"); + expect(formatEvidence(e, { full: true }).join("\n")).toContain("out 0\n"); + }); +}); diff --git a/src/terminal/evidence-format.ts b/src/terminal/evidence-format.ts new file mode 100644 index 00000000..0899b099 --- /dev/null +++ b/src/terminal/evidence-format.ts @@ -0,0 +1,34 @@ +/** + * Render one evidence record for the terminal (`codeoid evidence `). + * Pure: returns lines, so it is tested without a daemon. + */ + +import type { EvidenceWire } from "../protocol/types.js"; + +export function formatEvidence(e: EvidenceWire, opts: { full?: boolean } = {}): string[] { + const secs = ((Date.parse(e.endedAt) - Date.parse(e.startedAt)) / 1000).toFixed(1); + const exit = e.signal ? `signal ${e.signal}` : e.exitCode === null ? "no exit code" : `exit ${e.exitCode}`; + const lines = [ + `${e.id} ${e.result.toUpperCase()} ${e.reason}`, + ` check ${e.criterionId}`, + ` command ${e.command}`, + ` outcome ${exit}${e.timedOut ? ", timed out" : ""}, ${secs}s`, + ` candidate ${e.candidateDigest ?? "(no pinned tree — certifies no particular revision)"}`, + ` verifier ${e.verifierDigest.slice(0, 16)} isolation=${e.isolation}`, + ` owner ${e.ownerKind} ${e.ownerId}`, + ` ran ${e.startedAt} on ${e.environment.platform}/${e.environment.arch}, ${e.environment.runtime}`, + ]; + for (const a of e.artifacts) { + lines.push(` ${a.name.padEnd(9)} ${a.bytes} bytes${a.truncated ? " (truncated at cap)" : ""} sha256 ${a.sha256.slice(0, 16)}`); + } + const block = (title: string, text: string): void => { + const body = text.trimEnd(); + if (!body) return; + const shown = opts.full ? body.split("\n") : body.split("\n").slice(-40); + lines.push("", `── ${title}${!opts.full && body.split("\n").length > 40 ? " (last 40 lines; --full for the excerpt)" : ""}`); + lines.push(...shown); + }; + block("stderr", e.stderrExcerpt); + block("stdout", e.stdoutExcerpt); + return lines; +} diff --git a/src/terminal/pipeline-format.ts b/src/terminal/pipeline-format.ts index 8a7bf845..a93f6f0e 100644 --- a/src/terminal/pipeline-format.ts +++ b/src/terminal/pipeline-format.ts @@ -49,6 +49,10 @@ export function formatPipeline(p: PipelineWire): string[] { if (ph.feedback && ph.feedback.length > 0) { out.push(` revisions: ${ph.feedback.length}`); } + if (ph.evidence && ph.evidence.length > 0) { + const last = ph.evidence[ph.evidence.length - 1]; + out.push(` evidence: ${ph.evidence.length} check(s), latest ${last} (codeoid evidence )`); + } // The findings loop (review → fix → re-review): where it stands, and what // the engine runs next while the phase is still in motion. if (ph.findings) { diff --git a/src/tests/pipeline-handlers.test.ts b/src/tests/pipeline-handlers.test.ts index f85d8960..9e56ac17 100644 --- a/src/tests/pipeline-handlers.test.ts +++ b/src/tests/pipeline-handlers.test.ts @@ -356,3 +356,80 @@ describe("pipeline.* handlers", () => { if (r.type === "response.error") expect(r.code).toBe("invalid_request"); }); }); + +describe("evidence.get", () => { + async function createRun(): Promise { + return snapshot( + await manager.handle( + { + type: "pipeline.create", + id: "1", + name: "EV-1", + phases: [{ id: "one", kind: "noop", gate: "always" }], + workdir: join(tmp, "repo"), + }, + AUTH, + CLIENT, + ), + ); + } + + async function record(ownerKind: "pipeline" | "loop", ownerId: string, tenant = AUTH): Promise { + const { EvidenceService } = await import("../daemon/evidence/service.js"); + const { EvidenceStore } = await import("../daemon/evidence/store.js"); + const svc = new EvidenceService(new EvidenceStore(store.database), join(tmp, "evidence")); + const { record } = await svc.run({ id: "one:gate", command: "echo nope >&2; exit 1", timeoutMs: 5_000 }, tmp, { + tenant: { accountId: tenant.accountId, projectId: tenant.projectId }, + ownerKind, + ownerId, + }); + return record!.id; + } + + const get = (evidenceId: string, auth: AuthContext = AUTH) => + manager.handle({ type: "evidence.get", id: "e", evidenceId }, auth, { ...CLIENT, auth }); + + test("the pipeline's tenant reads its evidence, without artifact paths", async () => { + const run = await createRun(); + const id = await record("pipeline", run.id); + const res = await get(id); + if (res.type !== "evidence.record") throw new Error(`expected evidence.record, got ${JSON.stringify(res)}`); + expect(res.evidence.result).toBe("fail"); + expect(res.evidence.ownerId).toBe(run.id); + expect(res.evidence.stderrExcerpt.trim()).toBe("nope"); + expect(JSON.stringify(res.evidence)).not.toContain(tmp); + }); + + test("another tenant gets not_found, the same as an unknown id", async () => { + const run = await createRun(); + const id = await record("pipeline", run.id); + const other = { ...AUTH, accountId: "other" }; + const res = await get(id, other); + expect(res.type).toBe("response.error"); + if (res.type === "response.error") expect(res.code).toBe("not_found"); + const unknown = await get("ev_nope"); + if (unknown.type === "response.error") expect(unknown.code).toBe("not_found"); + }); + + test("reading pipeline evidence needs pipeline:read", async () => { + const run = await createRun(); + const id = await record("pipeline", run.id); + const res = await get(id, { ...AUTH, scopes: [SCOPES.SESSION_LIST] as AuthContext["scopes"] }); + expect(res.type).toBe("response.error"); + if (res.type === "response.error") expect(res.code).toBe("forbidden"); + }); + + test("evidence whose pipeline is gone is not_found", async () => { + const id = await record("pipeline", "pipeline-that-never-existed"); + const res = await get(id); + if (res.type === "response.error") expect(res.code).toBe("not_found"); + else throw new Error("expected not_found"); + }); + + test("loop evidence is not readable here until loops authorize it", async () => { + const id = await record("loop", "run-1"); + const res = await get(id); + if (res.type === "response.error") expect(res.code).toBe("not_found"); + else throw new Error("expected not_found"); + }); +}); diff --git a/src/tests/protocol.test.ts b/src/tests/protocol.test.ts index b5f95f98..0271bac8 100644 --- a/src/tests/protocol.test.ts +++ b/src/tests/protocol.test.ts @@ -496,6 +496,8 @@ describe("DaemonMessage routing", () => { return `routine.token:${msg.routineId}`; case "routine.preview.result": return `routine.preview:${msg.fires.length}`; + case "evidence.record": + return `evidence:${msg.evidence.result}`; case "mcp.oauth.begin.result": return `mcp.oauth.begin:${msg.status}`; case "mcp.oauth.complete.result": diff --git a/web/src/components/EvidenceView.test.tsx b/web/src/components/EvidenceView.test.tsx new file mode 100644 index 00000000..c42e25da --- /dev/null +++ b/web/src/components/EvidenceView.test.tsx @@ -0,0 +1,59 @@ +// @vitest-environment jsdom +import { afterEach, describe, expect, it } from "vitest"; +import { cleanup, render } from "@solidjs/testing-library"; + +import { EvidenceView } from "./EvidenceView"; +import type { EvidenceWire } from "../protocol/types"; + +const base: EvidenceWire = { + id: "ev_1", + ownerKind: "pipeline", + ownerId: "p1", + criterionId: "impl:test", + candidateDigest: null, + verifierDigest: "abc", + isolation: "process", + command: "bun test", + result: "fail", + reason: "exited with code 1", + exitCode: 1, + signal: null, + timedOut: false, + startedAt: "2026-10-25T10:00:00.000Z", + endedAt: "2026-10-25T10:00:02.500Z", + environment: { platform: "linux", arch: "x64", runtime: "bun 1.3", envKeys: [] }, + stdoutExcerpt: "", + stderrExcerpt: "expect(received).toBe(expected)\nExpected: 2\nReceived: 3", + artifacts: [], +}; + +afterEach(cleanup); + +function mount(e: EvidenceWire): HTMLElement { + return render(() => ).container; +} + +describe("EvidenceView", () => { + it("shows the result, the command, the duration and the failing output", () => { + const text = mount(base).textContent ?? ""; + expect(text).toContain("fail"); + expect(text).toContain("bun test"); + expect(text).toContain("2.5s"); + expect(text).toContain("Received: 3"); + expect(text).toContain("no pinned tree"); + }); + + it("names the pinned tree when there is one", () => { + expect(mount({ ...base, result: "pass", candidateDigest: "0123456789abcdef" }).textContent).toContain( + "tree 0123456789ab", + ); + }); + + it("keeps only the end of long output", () => { + const long = Array.from({ length: 100 }, (_, i) => `line ${i}`).join("\n"); + const text = mount({ ...base, stderrExcerpt: long }).textContent ?? ""; + expect(text).toContain("line 99"); + expect(text).not.toContain("line 10\n"); + expect(text).not.toMatch(/line 5(?!\d)/); + }); +}); diff --git a/web/src/components/EvidenceView.tsx b/web/src/components/EvidenceView.tsx new file mode 100644 index 00000000..1f0dc15a --- /dev/null +++ b/web/src/components/EvidenceView.tsx @@ -0,0 +1,86 @@ +/** + * One evidence record (docs/loop-engineering-design.md §7): what a check ran, + * on which tree, and what it found — the daemon's record, not a model's + * account. `EvidenceView` is presentational; `EvidenceLink` fetches on demand. + */ + +import { Component, Show, createSignal } from "solid-js"; + +import type { EvidenceWire } from "../protocol/types"; +import { fetchEvidence } from "../state/evidence"; + +const RESULT_CLASS: Record = { + pass: "border-success/40 bg-success/10 text-success", + fail: "border-danger/40 bg-danger/10 text-danger", + inconclusive: "border-warn/40 bg-warn/10 text-warn", + error: "border-warn/40 bg-warn/10 text-warn", + cancelled: "border-border bg-bg/40 text-fg-muted", +}; + +/** Last `n` lines of an excerpt — the end of a test run is where its summary is. */ +function tailLines(text: string, n: number): string { + const lines = text.trimEnd().split("\n"); + return lines.length > n ? lines.slice(-n).join("\n") : lines.join("\n"); +} + +export const EvidenceView: Component<{ evidence: EvidenceWire }> = (props) => { + const e = () => props.evidence; + const seconds = () => ((Date.parse(e().endedAt) - Date.parse(e().startedAt)) / 1000).toFixed(1); + const output = () => tailLines([e().stderrExcerpt, e().stdoutExcerpt].filter((s) => s.trim()).join("\n"), 30); + return ( +
+
+ + {e().result} + + {e().reason} + · {seconds()}s +
+
{e().command}
+
+ {e().candidateDigest ? `tree ${e().candidateDigest!.slice(0, 12)}` : "no pinned tree"} · {e().isolation} ·{" "} + {e().id} +
+ + 
+          {output()}
+
+ +
+ ); +}; + +/** "N checks" with the latest record shown on request. */ +export const EvidenceLink: Component<{ ids: string[] }> = (props) => { + const [record, setRecord] = createSignal(null); + const [error, setError] = createSignal(null); + const [loading, setLoading] = createSignal(false); + const latest = () => props.ids[props.ids.length - 1]!; + const load = async () => { + setLoading(true); + setError(null); + try { + setRecord(await fetchEvidence(latest())); + } catch (err) { + setError(err instanceof Error ? err.message : String(err)); + } finally { + setLoading(false); + } + }; + return ( +
+ + +

{error()}

+ + {(r) => } +
+ ); +}; diff --git a/web/src/components/PipelineRunner.tsx b/web/src/components/PipelineRunner.tsx index 4a152d90..9304baa1 100644 --- a/web/src/components/PipelineRunner.tsx +++ b/web/src/components/PipelineRunner.tsx @@ -20,6 +20,7 @@ import { Component, For, Show, createSignal, onCleanup, onMount } from "solid-js"; +import { EvidenceLink } from "./EvidenceView"; import { openPipelineModal } from "./NewSessionModal"; import { abort, approve, pipelinesState, reject, revise } from "../state/pipelines"; import { focusSession, focusedSessionId, sessionList } from "../state/sessions"; @@ -395,6 +396,10 @@ const HaltCard: Component<{

{props.phase.reason}

+ 0}> + + + {(f) => (

diff --git a/web/src/state/evidence.ts b/web/src/state/evidence.ts new file mode 100644 index 00000000..d0979362 --- /dev/null +++ b/web/src/state/evidence.ts @@ -0,0 +1,30 @@ +/** + * Evidence reads (docs/loop-engineering-design.md §7): fetch one record of an + * executed check by id. Records are immutable, so a fetched record is cached + * for the life of the page. + */ + +import type { EvidenceRecordMsg, EvidenceWire } from "../protocol/types"; +import { getClient, newRequestId } from "./connection"; + +const READ_TIMEOUT_MS = 15_000; +const cache = new Map(); + +export async function fetchEvidence(evidenceId: string): Promise { + const hit = cache.get(evidenceId); + if (hit) return hit; + const id = newRequestId(); + const msg = await getClient().request( + { type: "evidence.get", id, evidenceId }, + { + waitForResult: (m) => (m.type === "evidence.record" && m.requestId === id ? m : undefined), + timeoutMs: READ_TIMEOUT_MS, + }, + ); + cache.set(evidenceId, msg.evidence); + return msg.evidence; +} + +export function _resetEvidenceForTest(): void { + cache.clear(); +} From 771899837102e57e3efc686c17e96966812a10db Mon Sep 17 00:00:00 2001 From: Yash Datta Date: Sun, 25 Oct 2026 08:28:31 +0800 Subject: [PATCH 2/3] =?UTF-8?q?fix:=20apply=20round-1=20audit=20to=20evide?= =?UTF-8?q?nce=20=E2=80=94=20stripped=20env,=20redaction,=20private=20file?= =?UTF-8?q?s,=20retention,=20cancellation?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Checks run with the agent subprocess environment (no CODEOID_/ZEROID_/ TELEGRAM_), since their output now flows back to the worker. - Stored excerpts and gate diagnostics are redacted; artifacts are 0600 in 0700 directories; unrecorded checks use a private mkdtemp directory. - Zero-test detection scans every output line (go/cargo counts across the whole run), not the excerpt; excerpt cuts keep whole UTF-8 characters. - EvidenceService tracks live checks: pipeline.abort and daemon drain stop them. A retention sweep (30 days / 2 GiB, orphan dirs) runs off the boot path and daily; the per-stream artifact cap is 8 MiB. - evidence.get authorizes before any lookup; pipeline wire carries a phase's evidence in every status; stderr gets priority in gate diagnostics; the web view refetches when a newer check lands; the delayed SIGKILL is cleared once the group exits; a missing workdir is named as such. Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/pipeline-run.md | 4 +- src/daemon/evidence/parsers.ts | 97 +++++++++++++----- src/daemon/evidence/redact.ts | 38 +++++++ src/daemon/evidence/runner.test.ts | 54 +++++++++- src/daemon/evidence/runner.ts | 108 ++++++++++++------- src/daemon/evidence/service.test.ts | 43 +++++++- src/daemon/evidence/service.ts | 131 +++++++++++++++++++++--- src/daemon/evidence/store.ts | 51 ++++++++- src/daemon/pipeline/gate-probes.test.ts | 17 ++- src/daemon/pipeline/gate-probes.ts | 14 ++- src/daemon/session-manager.ts | 30 +++++- web/src/components/EvidenceView.tsx | 5 +- 12 files changed, 496 insertions(+), 96 deletions(-) create mode 100644 src/daemon/evidence/redact.ts diff --git a/docs/pipeline-run.md b/docs/pipeline-run.md index a1e9c795..3f6e6f92 100644 --- a/docs/pipeline-run.md +++ b/docs/pipeline-run.md @@ -76,7 +76,9 @@ The only thing it saves is one auto-attach, which the client does for free. Until implemented they are simply absent, not fake. - **Every check a gate executes leaves evidence** (docs/loop-engineering-design.md §7). `command` gates and execution probes record what ran, its result, its exit status or signal, bounded output excerpts, and the full output as files beside the daemon database (`/evidence/`). - A check that exits 0 after running no tests is `inconclusive`, not a pass; a missing command is `error`; a check past its time bound is stopped with everything it spawned and recorded as a timed-out `fail`. + An execution probe of type `test` (or `verify`) that exits 0 after running no tests is `inconclusive`, not a pass; a `command` gate is judged on its exit status alone. + A missing command is `error`; a check past its time bound is stopped with everything it spawned and recorded as a timed-out `fail`; aborting the pipeline or stopping the daemon stops its running checks. + Checks run with the same stripped environment as an agent's own commands (no daemon credentials), and stored excerpts are redacted; artifacts are readable only by the daemon's user and are pruned after 30 days or past 2 GiB in total. The halted phase shows the latest check's output in the cockpit, `codeoid pipeline show` lists each phase's evidence ids, and `codeoid evidence ` prints one record. Pipeline evidence names no pinned tree yet; it records what ran in the workdir, not which revision it certified. diff --git a/src/daemon/evidence/parsers.ts b/src/daemon/evidence/parsers.ts index f6cd350c..32c40862 100644 --- a/src/daemon/evidence/parsers.ts +++ b/src/daemon/evidence/parsers.ts @@ -6,11 +6,14 @@ * package. §7 of the loop design says an empty test selection is not a pass, so * a check marked `expectTests` that prints one of these is `inconclusive`. * - * Patterns are each runner's own summary line for "nothing ran", matched on - * the combined output. They are deliberately specific: a false match turns a - * real pass into an inconclusive result, which costs an attempt but never - * certifies anything, so the bias is toward missing an exotic runner rather - * than misreading a common one. + * Patterns are each runner's own summary line for "nothing ran". They are + * deliberately specific: a false match turns a real pass into an inconclusive + * result, which costs an attempt but never certifies anything, so the bias is + * toward missing an exotic runner rather than misreading a common one. + * + * The scan sees every line of the output, not an excerpt: go and cargo decide + * from counts across the whole run, and the line that shows tests DID run can + * sit in the middle of a long log. */ interface ZeroTestPattern { @@ -19,7 +22,7 @@ interface ZeroTestPattern { } const ZERO_TEST_PATTERNS: readonly ZeroTestPattern[] = [ - // bun test: "Ran 0 tests across 0 files." / "0 pass\n 0 fail" with no tests. + // bun test: "Ran 0 tests across 0 files." / "No tests found!". { runner: "bun", pattern: /\bRan 0 tests\b/ }, { runner: "bun", pattern: /\bNo tests found!/ }, // jest: "No tests found, exiting with code 0" (with --passWithNoTests). @@ -28,26 +31,71 @@ const ZERO_TEST_PATTERNS: readonly ZeroTestPattern[] = [ { runner: "vitest", pattern: /\bNo test files found, exiting with code 0\b/ }, // pytest: "no tests ran in 0.01s" (exit 5 normally, 0 with some plugins). { runner: "pytest", pattern: /\bno tests ran\b/i }, - // cargo: every "running N tests" line says 0 — checked separately below. // node --test: "# tests 0". - { runner: "node", pattern: /^# tests 0$/m }, + { runner: "node", pattern: /^# tests 0$/ }, // mocha: "0 passing". - { runner: "mocha", pattern: /^\s*0 passing\b/m }, + { runner: "mocha", pattern: /^\s*0 passing\b/ }, ]; -/** go test prints "? pkg [no test files]" per package and "ok pkg" for each - * package whose tests ran. Nothing ran when there are no-test-file lines and - * no ok/FAIL lines. */ -function goRanNothing(output: string): boolean { - if (!/\[no test files\]/.test(output)) return false; - return !/^(ok|FAIL|---)\s/m.test(output); +/** Line-at-a-time scan of a check's output (both streams). */ +export class ZeroTestScan { + #matched: string | null = null; + // go: "? pkg [no test files]" per package; "ok"/"FAIL"/"---" when tests ran. + #goNoTestFiles = false; + #goRan = false; + // cargo: "running N tests" per test binary. + #cargoRunningLines = 0; + #cargoNonZero = false; + + line(text: string): void { + if (this.#matched === null) { + for (const { runner, pattern } of ZERO_TEST_PATTERNS) { + if (pattern.test(text)) { + this.#matched = runner; + break; + } + } + } + if (/\[no test files\]/.test(text)) this.#goNoTestFiles = true; + if (/^(ok|FAIL|---)\s/.test(text)) this.#goRan = true; + const cargo = /^running (\d+) tests?$/.exec(text); + if (cargo) { + this.#cargoRunningLines++; + if (Number(cargo[1]) > 0) this.#cargoNonZero = true; + } + } + + /** The runner whose "nothing ran" summary was seen, or null. */ + result(): string | null { + if (this.#matched) return this.#matched; + if (this.#goNoTestFiles && !this.#goRan) return "go"; + if (this.#cargoRunningLines > 0 && !this.#cargoNonZero) return "cargo"; + return null; + } } -/** cargo test prints "running N tests" per test binary; nothing ran when there - * is at least one such line and every one says 0. */ -function cargoRanNothing(output: string): boolean { - const counts = [...output.matchAll(/^running (\d+) tests?$/gm)].map((m) => Number(m[1])); - return counts.length > 0 && counts.every((n) => n === 0); +/** Splits a byte stream into lines for a ZeroTestScan, carrying partial lines + * across chunks. Lines past a length bound are scanned truncated. */ +export class LineSplitter { + static readonly MAX_LINE = 4_096; + #partial = ""; + readonly #onLine: (line: string) => void; + + constructor(onLine: (line: string) => void) { + this.#onLine = onLine; + } + + push(text: string): void { + const parts = (this.#partial + text).split("\n"); + this.#partial = parts.pop() ?? ""; + if (this.#partial.length > LineSplitter.MAX_LINE) this.#partial = this.#partial.slice(0, LineSplitter.MAX_LINE); + for (const p of parts) this.#onLine(p.replace(/\r$/, "").slice(0, LineSplitter.MAX_LINE)); + } + + end(): void { + if (this.#partial) this.#onLine(this.#partial.replace(/\r$/, "")); + this.#partial = ""; + } } /** @@ -56,10 +104,7 @@ function cargoRanNothing(output: string): boolean { * summary is not evidence of an empty selection). */ export function detectZeroTests(output: string): string | null { - for (const { runner, pattern } of ZERO_TEST_PATTERNS) { - if (pattern.test(output)) return runner; - } - if (goRanNothing(output)) return "go"; - if (cargoRanNothing(output)) return "cargo"; - return null; + const scan = new ZeroTestScan(); + for (const line of output.split("\n")) scan.line(line.replace(/\r$/, "")); + return scan.result(); } diff --git a/src/daemon/evidence/redact.ts b/src/daemon/evidence/redact.ts new file mode 100644 index 00000000..b1a91e22 --- /dev/null +++ b/src/daemon/evidence/redact.ts @@ -0,0 +1,38 @@ +/** + * Redact credentials from a check's captured output before it is stored as an + * excerpt or shown to a model. Test output echoes tokens more often than one + * would like: `set -x`, a failing HTTP fixture, a connection string in an error. + * + * Uses the daemon's redaction rules (mcp/redact.ts), which scan a bounded + * window per call, so long text is redacted a group of lines at a time. A token + * never spans a line break in practice, so splitting on lines loses nothing. + */ + +import { redactSecretText } from "../mcp/redact.js"; + +/** Kept under redactSecretText's scan window so every byte is scanned. */ +const CHUNK_CHARS = 1_800; + +export function redactOutput(text: string): string { + if (!text) return text; + const out: string[] = []; + let chunk = ""; + const flush = (): void => { + if (chunk) out.push(redactSecretText(chunk)); + chunk = ""; + }; + for (const line of text.split("\n")) { + // A single line longer than the window is redacted in slices. + if (line.length > CHUNK_CHARS) { + flush(); + const slices: string[] = []; + for (let i = 0; i < line.length; i += CHUNK_CHARS) slices.push(redactSecretText(line.slice(i, i + CHUNK_CHARS))); + out.push(slices.join("")); + continue; + } + if (chunk.length + line.length + 1 > CHUNK_CHARS) flush(); + chunk = chunk ? `${chunk}\n${line}` : line; + } + flush(); + return out.join("\n"); +} diff --git a/src/daemon/evidence/runner.test.ts b/src/daemon/evidence/runner.test.ts index f89d4112..9e735b6c 100644 --- a/src/daemon/evidence/runner.test.ts +++ b/src/daemon/evidence/runner.test.ts @@ -1,5 +1,5 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { existsSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { detectZeroTests } from "./parsers"; @@ -166,3 +166,55 @@ describe("detectZeroTests", () => { expect(detectZeroTests(output)).toBeNull(); }); }); + +describe("audit round 1", () => { + test("a check never sees the daemon's own credentials", async () => { + process.env.CODEOID_TEST_ROOT_KEY = "zid_sk_should_not_leak"; + try { + const out = await run(spec("env")); + expect(out.stdoutExcerpt).not.toContain("zid_sk_should_not_leak"); + expect(out.stdoutExcerpt).toContain("PATH="); + } finally { + delete process.env.CODEOID_TEST_ROOT_KEY; + } + }); + + test("excerpts are redacted; the private artifact keeps the raw bytes", async () => { + const out = await run(spec("echo 'Authorization: Bearer abc.def-ghi'; echo 'token=s3cr3t-value'")); + expect(out.stdoutExcerpt).not.toContain("abc.def-ghi"); + expect(out.stdoutExcerpt).not.toContain("s3cr3t-value"); + const raw = readFileSync(out.artifacts.find((a) => a.name === "stdout")!.path, "utf8"); + expect(raw).toContain("s3cr3t-value"); + }); + + test("artifacts are readable only by the daemon's user", async () => { + const out = await run(spec("echo hi")); + const file = out.artifacts[0]!.path; + expect(statSync(file).mode & 0o777).toBe(0o600); + expect(statSync(join(dir, "art")).mode & 0o777).toBe(0o700); + }); + + test("zero-test detection sees the whole stream, not just the excerpt", async () => { + // 40 empty test binaries, real tests in the omitted middle, empty doc-tests. + const cmd = [ + "i=0; while [ $i -lt 400 ]; do echo 'running 0 tests'; echo; i=$((i+1)); done", + "echo 'running 2000 tests'", + "i=0; while [ $i -lt 3000 ]; do echo \"test t$i ... ok\"; i=$((i+1)); done", + "echo 'running 0 tests'", + ].join("; "); + const out = await run(spec(cmd, { expectTests: true })); + expect(out.result).toBe("pass"); + }); + + test("an excerpt cut never splits a UTF-8 character", async () => { + // 4095 ASCII bytes then a 3-byte character straddling the 4 KiB head cut. + const out = await run(spec("printf '%4095s' '' | tr ' ' a; printf '€'; i=0; while [ $i -lt 2000 ]; do echo line$i; i=$((i+1)); done")); + expect(out.stdoutExcerpt).not.toContain("\uFFFD"); + }); + + test("a missing working directory is named as such", async () => { + const out = await runCheck(spec("true"), { cwd: join(dir, "nope"), artifactDir: join(dir, "art") }); + expect(out.result).toBe("error"); + expect(out.reason).toContain("working directory does not exist"); + }); +}); diff --git a/src/daemon/evidence/runner.ts b/src/daemon/evidence/runner.ts index e073d6b4..66aa0ab9 100644 --- a/src/daemon/evidence/runner.ts +++ b/src/daemon/evidence/runner.ts @@ -4,8 +4,14 @@ * The runner owns the process it starts: the check runs as the leader of its * own process group, so a timeout, a cancel, or the check's own exit can kill * everything it spawned — a test server left behind by `sh -c` included. Output - * is kept twice: bounded excerpts for display and worker feedback, and the full - * streams as artifact files (up to a cap) so nothing a reviewer needs is lost. + * is kept twice: bounded, redacted excerpts for display and worker feedback, + * and the full streams as owner-only artifact files (up to a cap) so nothing a + * reviewer needs is lost. + * + * A check gets the same stripped environment as an agent's own subprocesses + * (providers/env.ts): its command and output can be shaped by the agent being + * judged, and its output is shown back to that agent, so the daemon's own + * credentials must never be in reach. * * Classification is the runner's job, not the caller's: exit 0 is `pass` only * when it actually established something (see parsers.ts), a timeout is @@ -14,16 +20,18 @@ import { spawn } from "node:child_process"; import { createHash } from "node:crypto"; -import { createWriteStream, mkdirSync, type WriteStream } from "node:fs"; +import { createWriteStream, existsSync, mkdirSync, statSync, type WriteStream } from "node:fs"; import { join } from "node:path"; -import { detectZeroTests } from "./parsers"; +import { buildSubprocessEnv } from "../providers/env.js"; +import { LineSplitter, ZeroTestScan } from "./parsers"; +import { redactOutput } from "./redact"; import type { CheckOutcome, CheckSpec, EvidenceArtifact, EvidenceEnvironment, EvidenceResult } from "./types"; /** Excerpt kept from each end of a stream for display and worker feedback. */ export const EXCERPT_HEAD_BYTES = 4 * 1024; export const EXCERPT_TAIL_BYTES = 12 * 1024; /** Default cap on one stream's artifact file. */ -export const DEFAULT_MAX_ARTIFACT_BYTES = 64 * 1024 * 1024; +export const DEFAULT_MAX_ARTIFACT_BYTES = 8 * 1024 * 1024; /** Between SIGTERM and SIGKILL when the runner stops a check. */ export const DEFAULT_KILL_GRACE_MS = 2_000; /** How long to wait for output pipes to drain after the check exits, before @@ -33,7 +41,7 @@ const DRAIN_WAIT_MS = 2_000; export interface RunCheckOptions { /** Directory the check runs in. */ cwd: string; - /** Directory the full output streams are written to (created if missing). */ + /** Directory the full output streams are written to (created if missing, 0700). */ artifactDir: string; /** Aborting stops the check and records `cancelled`. */ signal?: AbortSignal; @@ -63,14 +71,32 @@ function environmentOf(spec: CheckSpec): EvidenceEnvironment { }; } -/** One output stream: written to its artifact file up to a cap, hashed, and - * kept as a head + rolling tail for the excerpt. */ +/** Bytes [start, end) of `buf` adjusted inward to whole UTF-8 characters, so a + * cut never leaves a replacement character at either edge. */ +function utf8Whole(buf: Buffer): Buffer { + let start = 0; + while (start < buf.length && start < 4 && (buf[start]! & 0xc0) === 0x80) start++; + let end = buf.length; + // Find the last lead byte; drop it if its sequence is incomplete. + for (let i = end - 1; i >= Math.max(start, end - 4); i--) { + const b = buf[i]!; + if ((b & 0xc0) === 0x80) continue; + const need = b >= 0xf0 ? 4 : b >= 0xe0 ? 3 : b >= 0xc0 ? 2 : 1; + if (end - i < need) end = i; + break; + } + return buf.subarray(start, end); +} + +/** One output stream: written to its artifact file up to a cap, hashed, fed + * line by line to the zero-test scan, and kept as a head + rolling tail. */ class CapturedStream { readonly #name: "stdout" | "stderr"; readonly #path: string; readonly #file: WriteStream; readonly #hash = createHash("sha256"); readonly #max: number; + readonly #lines: LineSplitter; #bytes = 0; #written = 0; #head: Buffer[] = []; @@ -78,15 +104,17 @@ class CapturedStream { #tail: Buffer[] = []; #tailBytes = 0; - constructor(name: "stdout" | "stderr", dir: string, max: number) { + constructor(name: "stdout" | "stderr", dir: string, max: number, scan: ZeroTestScan) { this.#name = name; this.#path = join(dir, `${name}.log`); - this.#file = createWriteStream(this.#path); + this.#file = createWriteStream(this.#path, { mode: 0o600 }); this.#max = max; + this.#lines = new LineSplitter((l) => scan.line(l)); } push(chunk: Buffer): void { this.#bytes += chunk.length; + this.#lines.push(chunk.toString("utf8")); if (this.#written < this.#max) { const room = this.#max - this.#written; const part = chunk.length <= room ? chunk : chunk.subarray(0, room); @@ -109,10 +137,11 @@ class CapturedStream { } } - /** The excerpt: the head, then — if output was longer — a marker and the tail. */ + /** The excerpt, redacted: the head, then — if output was longer — a marker + * and the tail. */ excerpt(): string { - const head = Buffer.concat(this.#head).toString("utf8"); - if (this.#tailBytes === 0) return head; + const head = utf8Whole(Buffer.concat(this.#head)).toString("utf8"); + if (this.#tailBytes === 0) return redactOutput(head); let tail = Buffer.concat(this.#tail); let skipped = this.#bytes - this.#headBytes - tail.length; if (tail.length > EXCERPT_TAIL_BYTES) { @@ -120,11 +149,12 @@ class CapturedStream { tail = tail.subarray(tail.length - EXCERPT_TAIL_BYTES); } const marker = skipped > 0 ? `\n… [${skipped} bytes omitted — full output in the ${this.#name} artifact] …\n` : ""; - return head + marker + tail.toString("utf8"); + return redactOutput(head + marker + utf8Whole(tail).toString("utf8")); } /** Flush the artifact file and describe it. */ async close(): Promise { + this.#lines.end(); await new Promise((res) => this.#file.end(res)); return { name: this.#name, @@ -171,31 +201,28 @@ export async function runCheck(spec: CheckSpec, opts: RunCheckOptions): Promise< endedAt: Date.now(), ...fields, }); + const notRun = (result: EvidenceResult, reason: string): CheckOutcome => + finish({ result, reason, exitCode: null, signal: null, timedOut: false }); - if (opts.signal?.aborted) { - return finish({ result: "cancelled", reason: "cancelled before it started", exitCode: null, signal: null, timedOut: false }); + if (opts.signal?.aborted) return notRun("cancelled", "cancelled before it started"); + if (!existsSync(opts.cwd) || !statSync(opts.cwd).isDirectory()) { + return notRun("error", `the working directory does not exist: ${opts.cwd}`); } - try { - mkdirSync(opts.artifactDir, { recursive: true }); + mkdirSync(opts.artifactDir, { recursive: true, mode: 0o700 }); } catch (err) { - return finish({ - result: "error", - reason: `could not create the artifact directory: ${errText(err)}`, - exitCode: null, - signal: null, - timedOut: false, - }); + return notRun("error", `could not create the artifact directory: ${errText(err)}`); } const max = opts.maxArtifactBytes ?? DEFAULT_MAX_ARTIFACT_BYTES; - const out = new CapturedStream("stdout", opts.artifactDir, max); - const err = new CapturedStream("stderr", opts.artifactDir, max); + const scan = new ZeroTestScan(); + const out = new CapturedStream("stdout", opts.artifactDir, max, scan); + const err = new CapturedStream("stderr", opts.artifactDir, max, scan); const grace = opts.killGraceMs ?? DEFAULT_KILL_GRACE_MS; const child = spawn("sh", ["-c", spec.command], { cwd: opts.cwd, - env: { ...process.env, ...(spec.env ?? {}) }, + env: { ...buildSubprocessEnv({}), ...(spec.env ?? {}) }, stdio: ["ignore", "pipe", "pipe"], // Its own process group, so killGroup reaches everything it spawns. detached: true, @@ -206,11 +233,13 @@ export async function runCheck(spec: CheckSpec, opts: RunCheckOptions): Promise< const closed = new Promise((res) => child.once("close", () => res())); let stoppedBy: "timeout" | "cancel" | null = null; + let killTimer: ReturnType | undefined; const stop = (why: "timeout" | "cancel"): void => { if (stoppedBy || child.pid === undefined) return; stoppedBy = why; - killGroup(child.pid, "SIGTERM"); - setTimeout(() => killGroup(child.pid!, "SIGKILL"), grace).unref(); + const pid = child.pid; + killGroup(pid, "SIGTERM"); + killTimer = setTimeout(() => killGroup(pid, "SIGKILL"), grace); }; const timer = setTimeout(() => stop("timeout"), spec.timeoutMs); const onAbort = (): void => stop("cancel"); @@ -225,8 +254,12 @@ export async function runCheck(spec: CheckSpec, opts: RunCheckOptions): Promise< clearTimeout(timer); opts.signal?.removeEventListener("abort", onAbort); - // The check is over; anything it left running in its group goes too. + // The check is over; anything it left running in its group goes too. The + // group id is still held by those stragglers, so this cannot reach another + // process group. The delayed SIGKILL is no longer needed — and once every + // member is gone the id may be reused, so it must not fire. if (child.pid !== undefined) killGroup(child.pid, "SIGKILL"); + clearTimeout(killTimer); let drainTimer: ReturnType | undefined; const drained = new Promise((res) => { drainTimer = setTimeout(res, DRAIN_WAIT_MS); @@ -237,9 +270,7 @@ export async function runCheck(spec: CheckSpec, opts: RunCheckOptions): Promise< child.stderr?.destroy(); const artifacts = [await out.close(), await err.close()]; - const stdoutExcerpt = out.excerpt(); - const stderrExcerpt = err.excerpt(); - const captured = { stdoutExcerpt, stderrExcerpt, artifacts }; + const captured = { stdoutExcerpt: out.excerpt(), stderrExcerpt: err.excerpt(), artifacts }; const signal = exit.signal ?? null; if (exit.spawnError) { @@ -265,14 +296,14 @@ export async function runCheck(spec: CheckSpec, opts: RunCheckOptions): Promise< ...captured, }); } - return finish({ ...classifyExit(spec, exit.code, signal, `${stdoutExcerpt}\n${stderrExcerpt}`), ...captured }); + return finish({ ...classifyExit(spec, exit.code, signal, scan.result()), ...captured }); } function classifyExit( spec: CheckSpec, code: number | null, signal: NodeJS.Signals | null, - output: string, + zeroTestsRunner: string | null, ): Pick { const at = { exitCode: code, signal, timedOut: false }; const result = (r: EvidenceResult, reason: string) => ({ result: r, reason, ...at }); @@ -281,9 +312,8 @@ function classifyExit( return result("error", code === 127 ? "command not found (exit 127)" : "command not executable (exit 126)"); } if (code !== 0) return result("fail", `exited with code ${code}`); - if (spec.expectTests) { - const runner = detectZeroTests(output); - if (runner) return result("inconclusive", `exited 0 but ran no tests (${runner} reported an empty selection)`); + if (spec.expectTests && zeroTestsRunner) { + return result("inconclusive", `exited 0 but ran no tests (${zeroTestsRunner} reported an empty selection)`); } return result("pass", "exited 0"); } diff --git a/src/daemon/evidence/service.test.ts b/src/daemon/evidence/service.test.ts index 469508b6..6429b560 100644 --- a/src/daemon/evidence/service.test.ts +++ b/src/daemon/evidence/service.test.ts @@ -1,6 +1,6 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import { Database } from "bun:sqlite"; -import { existsSync, mkdtempSync, readFileSync, rmSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, utimesSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { EvidenceService, ephemeralCheck } from "./service"; @@ -66,6 +66,47 @@ describe("EvidenceService", () => { for (const a of record!.artifacts) expect(a.path.startsWith(`${root}/`)).toBe(true); }); + test("aborting an owner stops its running checks and records them cancelled", async () => { + const svc = new EvidenceService(new EvidenceStore(":memory:"), join(dir, "evidence")); + const running = svc.run({ id: "c", command: "sleep 30", timeoutMs: 60_000 }, dir, ctx); + const other = svc.run({ id: "c", command: "sleep 0.3", timeoutMs: 60_000 }, dir, { ...ctx, ownerId: "pipe-2" }); + await Bun.sleep(150); + svc.abortOwner("pipeline", "pipe-1"); + expect((await running).record?.result).toBe("cancelled"); + expect((await other).record?.result).toBe("pass"); + }); + + test("retention drops old records, then the oldest past the size cap, with their files", async () => { + const svc = new EvidenceService(new EvidenceStore(":memory:"), join(dir, "evidence")); + const a = (await svc.run({ id: "a", command: "head -c 3000 /dev/zero", timeoutMs: 5_000 }, dir, ctx)).record!; + await Bun.sleep(5); + const b = (await svc.run({ id: "b", command: "head -c 3000 /dev/zero", timeoutMs: 5_000 }, dir, ctx)).record!; + await Bun.sleep(5); + const c = (await svc.run({ id: "c", command: "head -c 3000 /dev/zero", timeoutMs: 5_000 }, dir, ctx)).record!; + // Size cap fits two records: the oldest goes. + expect(svc.sweep({ maxAgeMs: 1e12, maxTotalBytes: 6_500 })).toBe(1); + expect(svc.get(a.id, TENANT)).toBeNull(); + expect(existsSync(a.artifacts[0]!.path)).toBe(false); + expect(svc.get(b.id, TENANT)).not.toBeNull(); + // Age: everything started before now+1 goes. + expect(svc.sweep({ maxAgeMs: -1, maxTotalBytes: 1e12 })).toBe(2); + expect(existsSync(c.artifacts[0]!.path)).toBe(false); + }); + + test("retention removes artifact directories no record points at, once they are old", async () => { + const root = join(dir, "evidence"); + const svc = new EvidenceService(new EvidenceStore(":memory:"), root); + const orphan = join(root, "pipeline", "pipe-x", "stale"); + const fresh = join(root, "pipeline", "pipe-x", "fresh"); + mkdirSync(orphan, { recursive: true }); + mkdirSync(fresh, { recursive: true }); + const old = (Date.now() - 2 * 60 * 60 * 1000) / 1000; + utimesSync(orphan, old, old); + svc.sweep(); + expect(existsSync(orphan)).toBe(false); + expect(existsSync(fresh)).toBe(true); + }); + test("an unrecorded check leaves no files behind", async () => { const { outcome, record } = await ephemeralCheck({ id: "c", command: "echo hi", timeoutMs: 5_000 }, dir); expect(record).toBeUndefined(); diff --git a/src/daemon/evidence/service.ts b/src/daemon/evidence/service.ts index b2b40643..d643b6b0 100644 --- a/src/daemon/evidence/service.ts +++ b/src/daemon/evidence/service.ts @@ -1,10 +1,14 @@ /** * Runs checks and records what they did. The one entry point pipelines' gates * and (later) loops use, so every executed check leaves the same evidence. + * + * The service also owns the checks it starts: an owner (a pipeline being + * aborted) or the daemon (shutting down) can stop them, so a test run never + * outlives the thing that asked for it. */ import { randomUUID } from "node:crypto"; -import { rmSync } from "node:fs"; +import { mkdtempSync, readdirSync, rmSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { runCheck } from "./runner.js"; @@ -31,9 +35,25 @@ export interface CheckRun { /** The narrow surface a gate needs: run a check in a directory. */ export type CheckRunner = (spec: CheckSpec, cwd: string) => Promise; +export interface EvidenceRetention { + /** Records (and their artifacts) older than this are deleted. */ + maxAgeMs: number; + /** Past this many artifact bytes in total, the oldest records go first. */ + maxTotalBytes: number; +} + +export const DEFAULT_EVIDENCE_RETENTION: EvidenceRetention = { + maxAgeMs: 30 * 24 * 60 * 60 * 1000, + maxTotalBytes: 2 * 1024 * 1024 * 1024, +}; + +const ownerKey = (kind: EvidenceOwnerKind, id: string): string => `${kind}\u0000${id}`; + export class EvidenceService { readonly #store: EvidenceStore; readonly #artifactRoot: string; + /** Live checks by owner, so an owner or the daemon can stop them. */ + readonly #live = new Map>(); /** `artifactRoot` holds one directory per check's full output. */ constructor(store: EvidenceStore, artifactRoot: string) { @@ -44,16 +64,30 @@ export class EvidenceService { /** Run `spec` in `cwd` and persist the result. */ async run(spec: CheckSpec, cwd: string, ctx: EvidenceContext): Promise { const artifactDir = join(this.#artifactRoot, ctx.ownerKind, safeSegment(ctx.ownerId), randomUUID()); - const outcome = await runCheck(spec, { cwd, artifactDir, signal: ctx.signal }); - const record = this.#store.insert({ - ...outcome, - accountId: ctx.tenant.accountId, - projectId: ctx.tenant.projectId, - ownerKind: ctx.ownerKind, - ownerId: ctx.ownerId, - candidateDigest: ctx.candidateDigest ?? null, - }); - return { outcome, record }; + const key = ownerKey(ctx.ownerKind, ctx.ownerId); + const ac = new AbortController(); + const onAbort = (): void => ac.abort(); + ctx.signal?.addEventListener("abort", onAbort, { once: true }); + if (ctx.signal?.aborted) ac.abort(); + const live = this.#live.get(key) ?? new Set(); + live.add(ac); + this.#live.set(key, live); + try { + const outcome = await runCheck(spec, { cwd, artifactDir, signal: ac.signal }); + const record = this.#store.insert({ + ...outcome, + accountId: ctx.tenant.accountId, + projectId: ctx.tenant.projectId, + ownerKind: ctx.ownerKind, + ownerId: ctx.ownerId, + candidateDigest: ctx.candidateDigest ?? null, + }); + return { outcome, record }; + } finally { + ctx.signal?.removeEventListener("abort", onAbort); + live.delete(ac); + if (live.size === 0) this.#live.delete(key); + } } /** A runner bound to one owner — what a gate is handed. */ @@ -61,6 +95,16 @@ export class EvidenceService { return (spec, cwd) => this.run(spec, cwd, ctx); } + /** Stop every check running for one owner (e.g. a pipeline being aborted). */ + abortOwner(ownerKind: EvidenceOwnerKind, ownerId: string): void { + for (const ac of this.#live.get(ownerKey(ownerKind, ownerId)) ?? []) ac.abort(); + } + + /** Stop every running check (daemon shutdown). */ + abortAll(): void { + for (const set of this.#live.values()) for (const ac of set) ac.abort(); + } + get(id: string, tenant: EvidenceTenant): EvidenceRecord | null { return this.#store.get(id, tenant); } @@ -68,16 +112,73 @@ export class EvidenceService { listForOwner(ownerKind: EvidenceOwnerKind, ownerId: string, tenant: EvidenceTenant): EvidenceRecord[] { return this.#store.listForOwner(ownerKind, ownerId, tenant); } + + /** + * Delete records and artifacts past the retention policy: anything older than + * `maxAgeMs`, then the oldest until the artifacts fit in `maxTotalBytes`. + * Artifact directories with no record (a crash between writing output and + * storing the record) are removed too. Returns how many records went. + */ + sweep(policy: EvidenceRetention = DEFAULT_EVIDENCE_RETENTION, now = Date.now()): number { + const doomed = new Set(this.#store.idsStartedBefore(now - policy.maxAgeMs)); + let total = 0; + for (const r of this.#store.artifactSizesNewestFirst()) { + if (doomed.has(r.id)) continue; + total += r.bytes; + if (total > policy.maxTotalBytes) doomed.add(r.id); + } + const dirs = this.#store.artifactDirsFor([...doomed]); + this.#store.deleteIds([...doomed]); + for (const d of dirs) rmSync(d, { recursive: true, force: true }); + this.#removeOrphanDirs(new Set(this.#store.allArtifactDirs()), now); + return doomed.size; + } + + /** Remove check directories under the root that no record points at, once + * they are old enough not to belong to a check still running. */ + #removeOrphanDirs(known: Set, now: number): void { + const minAgeMs = 60 * 60 * 1000; + let kinds: string[]; + try { + kinds = readdirSync(this.#artifactRoot); + } catch { + return; + } + for (const kind of kinds) { + for (const owner of safeReaddir(join(this.#artifactRoot, kind))) { + const ownerDir = join(this.#artifactRoot, kind, owner); + for (const check of safeReaddir(ownerDir)) { + const dir = join(ownerDir, check); + if (known.has(dir)) continue; + try { + if (now - statSync(dir).mtimeMs < minAgeMs) continue; + } catch { + continue; + } + rmSync(dir, { recursive: true, force: true }); + } + if (safeReaddir(ownerDir).length === 0) rmSync(ownerDir, { recursive: true, force: true }); + } + } + } +} + +function safeReaddir(dir: string): string[] { + try { + return readdirSync(dir); + } catch { + return []; + } } /** * Run a check without recording it, for callers with no evidence store (tests, - * a pipeline manager built without one). Output goes to a temporary directory - * that is removed afterwards; the outcome's excerpts survive, its artifact - * paths do not. + * a pipeline manager built without one). Output goes to a private temporary + * directory that is removed afterwards; the outcome's excerpts survive, its + * artifact paths do not. */ export const ephemeralCheck: CheckRunner = async (spec, cwd) => { - const artifactDir = join(tmpdir(), "codeoid-evidence", randomUUID()); + const artifactDir = mkdtempSync(join(tmpdir(), "codeoid-evidence-")); try { return { outcome: await runCheck(spec, { cwd, artifactDir }) }; } finally { diff --git a/src/daemon/evidence/store.ts b/src/daemon/evidence/store.ts index 6bd9467a..4d4ee5c9 100644 --- a/src/daemon/evidence/store.ts +++ b/src/daemon/evidence/store.ts @@ -10,6 +10,7 @@ import { Database } from "bun:sqlite"; import { randomUUID } from "node:crypto"; +import { dirname } from "node:path"; import { BUSY_TIMEOUT_MS } from "../store.js"; import type { EvidenceArtifact, @@ -82,10 +83,13 @@ export class EvidenceStore { ended_at INTEGER NOT NULL, stdout_excerpt TEXT NOT NULL, stderr_excerpt TEXT NOT NULL, - artifacts_json TEXT NOT NULL + artifacts_json TEXT NOT NULL, + artifact_dir TEXT, + artifact_bytes INTEGER NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS idx_evidence_owner ON evidence(owner_kind, owner_id, started_at); + CREATE INDEX IF NOT EXISTS idx_evidence_started ON evidence(started_at); `); } @@ -98,8 +102,8 @@ export class EvidenceStore { id, account_id, project_id, owner_kind, owner_id, criterion_id, candidate_digest, verifier_digest, environment_json, isolation, command, result, reason, exit_code, signal, timed_out, started_at, ended_at, - stdout_excerpt, stderr_excerpt, artifacts_json - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + stdout_excerpt, stderr_excerpt, artifacts_json, artifact_dir, artifact_bytes + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, ) .run( id, @@ -123,6 +127,8 @@ export class EvidenceStore { record.stdoutExcerpt, record.stderrExcerpt, JSON.stringify(record.artifacts), + record.artifacts[0] ? dirname(record.artifacts[0].path) : null, + record.artifacts.reduce((n, a) => n + a.bytes, 0), ); return { ...record, id }; } @@ -136,6 +142,45 @@ export class EvidenceStore { return row ? toRecord(row) : null; } + // ── Retention (EvidenceService.sweep) ───────────────────────────────── + + idsStartedBefore(cutoff: number): string[] { + return (this.#db.prepare("SELECT id FROM evidence WHERE started_at < ?").all(cutoff) as Array<{ id: string }>).map( + (r) => r.id, + ); + } + + artifactSizesNewestFirst(): Array<{ id: string; bytes: number }> { + return this.#db + .prepare("SELECT id, artifact_bytes AS bytes FROM evidence ORDER BY started_at DESC, rowid DESC") + .all() as Array<{ id: string; bytes: number }>; + } + + artifactDirsFor(ids: string[]): string[] { + const out: string[] = []; + for (let i = 0; i < ids.length; i += 500) { + const chunk = ids.slice(i, i + 500); + const rows = this.#db + .prepare(`SELECT artifact_dir AS d FROM evidence WHERE id IN (${chunk.map(() => "?").join(",")}) AND artifact_dir IS NOT NULL`) + .all(...chunk) as Array<{ d: string }>; + out.push(...rows.map((r) => r.d)); + } + return out; + } + + allArtifactDirs(): string[] { + return ( + this.#db.prepare("SELECT artifact_dir AS d FROM evidence WHERE artifact_dir IS NOT NULL").all() as Array<{ d: string }> + ).map((r) => r.d); + } + + deleteIds(ids: string[]): void { + for (let i = 0; i < ids.length; i += 500) { + const chunk = ids.slice(i, i + 500); + this.#db.prepare(`DELETE FROM evidence WHERE id IN (${chunk.map(() => "?").join(",")})`).run(...chunk); + } + } + /** Every record for one owner, oldest first. */ listForOwner(ownerKind: EvidenceOwnerKind, ownerId: string, tenant: EvidenceTenant): EvidenceRecord[] { return ( diff --git a/src/daemon/pipeline/gate-probes.test.ts b/src/daemon/pipeline/gate-probes.test.ts index 690c0938..60afc058 100644 --- a/src/daemon/pipeline/gate-probes.test.ts +++ b/src/daemon/pipeline/gate-probes.test.ts @@ -2,7 +2,7 @@ import { describe, expect, test } from "bun:test"; import { mkdtempSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { buildProbeGate, detectEcosystem, type ProbeSpec } from "./gate-probes"; +import { buildProbeGate, detectEcosystem, diagnosticTail, type ProbeSpec } from "./gate-probes"; import type { GateCtx, PipelineState } from "./interface"; function tmp(): string { @@ -165,3 +165,18 @@ describe("probe evidence", () => { }); }); + +describe("diagnosticTail", () => { + const outcome = (stdout: string, stderr: string) => + ({ stdoutExcerpt: stdout, stderrExcerpt: stderr }) as Parameters[0]; + + test("long console noise on stdout cannot push the failure on stderr out", () => { + const tail = diagnosticTail(outcome("noise ".repeat(1_000), "expected 2, received 3"), 600); + expect(tail).toContain("expected 2, received 3"); + expect(tail.length).toBeLessThanOrEqual(600 + 5); + }); + + test("uses the whole budget for the only stream there is", () => { + expect(diagnosticTail(outcome("x".repeat(2_000), ""), 500).length).toBe(501); + }); +}); diff --git a/src/daemon/pipeline/gate-probes.ts b/src/daemon/pipeline/gate-probes.ts index a53140c0..94c86b6b 100644 --- a/src/daemon/pipeline/gate-probes.ts +++ b/src/daemon/pipeline/gate-probes.ts @@ -79,11 +79,17 @@ async function runProbe( } /** The diagnostic tail a failed check contributes to its gate's reason — what - * the phase's next attempt is shown. Bounded: the full output is evidence. */ + * the phase's next attempt is shown. stderr gets most of the room, because + * test runners report failures there and stdout is often console noise. Both + * excerpts are already redacted; the full output is evidence. */ export function diagnosticTail(outcome: CheckOutcome, max = 1_500): string { - const text = [outcome.stderrExcerpt.trim(), outcome.stdoutExcerpt.trim()].filter(Boolean).join("\n"); - if (!text) return ""; - return text.length > max ? `…${text.slice(text.length - max)}` : text; + const tail = (s: string, n: number): string => (s.length > n ? `…${s.slice(s.length - n)}` : s); + const err = outcome.stderrExcerpt.trim(); + const out = outcome.stdoutExcerpt.trim(); + if (!err) return tail(out, max); + if (!out) return tail(err, max); + const outRoom = Math.min(out.length, Math.floor(max / 3)); + return `${tail(err, max - outRoom)}\n${tail(out, outRoom)}`; } /** Detected build/test/lint commands for a workspace, keyed by the manifest we diff --git a/src/daemon/session-manager.ts b/src/daemon/session-manager.ts index ef9e42db..305ceaac 100644 --- a/src/daemon/session-manager.ts +++ b/src/daemon/session-manager.ts @@ -450,6 +450,7 @@ export class SessionManager { /** SDLC pipeline manager — undefined when the pipeline is disabled (default). */ #pipelines?: PipelineManager; #evidence?: EvidenceService; + #evidenceTimers: ReturnType[] = []; /** Routines (docs/routines.md); undefined when `routines.enabled` is off. */ #routines?: RoutineService; /** Pack curation surface (registries + install/trust/select) — always present, @@ -602,10 +603,20 @@ export class SessionManager { // Evidence of every executed check (docs/loop-engineering-design.md §7), // in the daemon DB with full output beside it. if (opts?.config) { - this.#evidence = new EvidenceService( + const evidence = new EvidenceService( new EvidenceStore(store.database), join(dirname(opts.config.dbPath), "evidence"), ); + this.#evidence = evidence; + // Retention: off the boot path, then daily. + const sweep = (): void => { + try { + evidence.sweep(); + } catch (err) { + console.error(`[evidence] retention sweep failed: ${err instanceof Error ? err.message : String(err)}`); + } + }; + this.#evidenceTimers.push(setTimeout(sweep, 60_000).unref(), setInterval(sweep, 24 * 60 * 60 * 1000).unref()); } this.#pipelines = createPipelineManagerFromConfig(opts?.config, { runner: new SessionPhaseRunner(() => this), @@ -3646,6 +3657,11 @@ mcpHub: this.#mcpHub, // An abandoned login is a live pty holding an open OAuth attempt. Kill // those first, and unconditionally — they are not work worth draining. this.#backendLogin.dispose(); + // Running checks are detached process groups; without this they outlive + // the daemon and race the same gate when the pipeline resumes. + this.#evidence?.abortAll(); + for (const t of this.#evidenceTimers) clearTimeout(t); + this.#evidenceTimers = []; const deadline = Date.now() + timeoutMs; const systemAuth: AuthContext = { sub: "system:shutdown", @@ -5374,8 +5390,9 @@ mcpHub: this.#mcpHub, // Surface the phase's produced output at the boundary halt so the UI // can show "here's what this phase did" next to Approve/Revise/Reject. if (p.lastSummary) w.summary = p.lastSummary; - if (p.evidence?.length) w.evidence = [...p.evidence]; } + // Evidence whatever the status — a failed phase is where it matters most. + if (p.evidence?.length) w.evidence = [...p.evidence]; if (p.feedback && p.feedback.length > 0) w.feedback = p.feedback; // The findings loop (findings.ts): counts + the ledger, so a client can // render "2 open (1 blocking), 1 fix leg" and the per-round table. @@ -5506,11 +5523,13 @@ mcpHub: this.#mcpHub, */ #evidenceGet(msg: Extract, auth: AuthContext): DaemonMessage { const notFound: DaemonMessage = { type: "response.error", requestId: msg.id, error: "Evidence not found", code: "not_found" }; + // Authorize before looking anything up, so whether an id exists is never + // the difference between two answers. + const g = this.#pipelineGuard(msg.id, auth, SCOPES.PIPELINE_READ); + if ("error" in g) return g.error; const record = this.#evidence?.get(msg.evidenceId, { accountId: auth.accountId, projectId: auth.projectId }); if (!record) return notFound; if (record.ownerKind === "pipeline") { - const g = this.#pipelineGuard(msg.id, auth, SCOPES.PIPELINE_READ); - if ("error" in g) return g.error; if (!this.#ownedPipeline(g.pm, record.ownerId, auth)) return notFound; } else { // Loop evidence is authorized by the loop service, which doesn't exist @@ -5593,6 +5612,9 @@ mcpHub: this.#mcpHub, if ("error" in g) return g.error; const s = this.#ownedPipeline(g.pm, msg.pipelineId, auth); if (!s) return { type: "response.error", requestId: msg.id, error: "Pipeline not found", code: "not_found" }; + // Stop its running checks first: abort waits behind the in-flight step, + // and that step may be a long test run. + this.#evidence?.abortOwner("pipeline", msg.pipelineId); const next = (await g.pm.abort(msg.pipelineId)) ?? s; return { type: "pipeline.snapshot", requestId: msg.id, pipeline: this.#pipelineToWire(next) }; } diff --git a/web/src/components/EvidenceView.tsx b/web/src/components/EvidenceView.tsx index 1f0dc15a..337e6d5c 100644 --- a/web/src/components/EvidenceView.tsx +++ b/web/src/components/EvidenceView.tsx @@ -4,7 +4,7 @@ * account. `EvidenceView` is presentational; `EvidenceLink` fetches on demand. */ -import { Component, Show, createSignal } from "solid-js"; +import { Component, Show, createEffect, createSignal, on } from "solid-js"; import type { EvidenceWire } from "../protocol/types"; import { fetchEvidence } from "../state/evidence"; @@ -56,6 +56,9 @@ export const EvidenceLink: Component<{ ids: string[] }> = (props) => { const [error, setError] = createSignal(null); const [loading, setLoading] = createSignal(false); const latest = () => props.ids[props.ids.length - 1]!; + // A newer check replaces what is shown: drop the old record so the next + // open fetches the latest. + createEffect(on(latest, () => setRecord(null), { defer: true })); const load = async () => { setLoading(true); setError(null); From 156ac868f26ad2be7e8ab96a8e8c2bfa24889a1a Mon Sep 17 00:00:00 2001 From: Yash Datta Date: Sun, 25 Oct 2026 08:43:55 +0800 Subject: [PATCH 3/3] =?UTF-8?q?fix:=20apply=20round-2=20audit=20to=20evide?= =?UTF-8?q?nce=20=E2=80=94=20cancelled=20checks=20drop=20the=20step,=20log?= =?UTF-8?q?-safe=20redaction,=20resilient=20artifacts?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - A gate check stopped by an abort or shutdown no longer fails the gate: the engine drops the step unsaved, so the gate re-runs on resume instead of halting, burning a retry, or failing the pipeline. - An owner's abort also cancels checks it starts afterwards; shutdown refuses new checks. - Redaction uses log-specific rules over the whole excerpt (PEM blocks, JWTs, URL userinfo, auth headers, known token formats, secret-named keys) and keeps test names, paths, hashes and exit codes readable; excerpt edges are cut at line boundaries. - An artifact write error (disk full, EMFILE) no longer crashes the daemon; the artifact is marked truncated. - Artifact directories are stored relative to the evidence root and the sweep never removes a running check's directory. - Checks keep common toolchain variables (LC_*, XDG_*, CI, GOPATH, …) plus CODEOID_CHECK_ENV_ALLOW, never the daemon's credentials. - The web evidence cache is per connection and bounded; a git-diff probe that cannot run says so. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/daemon/evidence/redact.test.ts | 47 ++++++++++++++++++ src/daemon/evidence/redact.ts | 65 +++++++++++++++---------- src/daemon/evidence/runner.test.ts | 41 ++++++++++++++-- src/daemon/evidence/runner.ts | 75 +++++++++++++++++++++++++---- src/daemon/evidence/service.test.ts | 22 +++++++++ src/daemon/evidence/service.ts | 43 ++++++++++++++--- src/daemon/evidence/store.ts | 9 ++-- src/daemon/pipeline/engine.ts | 33 ++++++++++--- src/daemon/pipeline/gate-probes.ts | 14 ++++-- src/daemon/pipeline/interface.ts | 4 ++ src/daemon/pipeline/pack.test.ts | 22 +++++++++ src/daemon/pipeline/pack.ts | 1 + web/src/state/evidence.ts | 21 +++++--- 13 files changed, 329 insertions(+), 68 deletions(-) create mode 100644 src/daemon/evidence/redact.test.ts diff --git a/src/daemon/evidence/redact.test.ts b/src/daemon/evidence/redact.test.ts new file mode 100644 index 00000000..43ce40b6 --- /dev/null +++ b/src/daemon/evidence/redact.test.ts @@ -0,0 +1,47 @@ +import { describe, expect, test } from "bun:test"; +import { redactOutput } from "./redact"; + +describe("redactOutput", () => { + test("keeps what a worker needs to repair a failure", () => { + const log = [ + "exit code: 1", + "--- PASS: TestParseHeader (0.00s)", + "--- FAIL: TestTokenizerHandlesUnicode (0.01s)", + "FAIL src/daemon/pipeline/session-manager.routines.test.ts", + " at Object. (/home/u/repo/src/very_long_component_name_for_testing.ts:42:7)", + "commit 3f9c2a1b8e7d6c5b4a3928170f6e5d4c3b2a1908", + "expected 2 received 3", + ].join("\n"); + expect(redactOutput(log)).toBe(log); + }); + + test("masks known credential shapes and secret-named values", () => { + const out = redactOutput( + [ + "Authorization: Bearer abcdefgh.ijklmnop-qrstuv", + "export GITHUB_TOKEN=ghp_0123456789abcdefghijABCDEFGHIJ", + "using key sk-ant-api03-0123456789abcdefghijklmn", + "DATABASE_PASSWORD='hunter2-is-secret'", + "connecting to postgres://app:s3cretpass@db.internal:5432/app", + "aws AKIAABCDEFGHIJKLMNOP", + ].join("\n"), + ); + for (const secret of ["ijklmnop", "ghp_0123", "sk-ant-api03", "hunter2", "s3cretpass", "AKIAABCDEF"]) { + expect(out).not.toContain(secret); + } + expect(out).toContain("db.internal:5432"); + }); + + test("a PEM block longer than any scan window is masked whole", () => { + const body = Array.from({ length: 60 }, (_, i) => `MIIBVwIBADANBgkqhkiG9w0BAQEFAASCAT8wggE7AgEAAkEA${String(i).padStart(4, "0")}`).join("\n"); + const out = redactOutput(`before\n-----BEGIN RSA PRIVATE KEY-----\n${body}\n-----END RSA PRIVATE KEY-----\nafter`); + expect(out).not.toContain("MIIBVwIBADAN"); + expect(out).toContain("before"); + expect(out).toContain("after"); + }); + + test("a secret deep inside a very long line is still masked", () => { + const line = `${"x".repeat(1_790)} password=hunter2Secret!x ${"y".repeat(500)}`; + expect(redactOutput(line)).not.toContain("hunter2Secret"); + }); +}); diff --git a/src/daemon/evidence/redact.ts b/src/daemon/evidence/redact.ts index b1a91e22..667ae385 100644 --- a/src/daemon/evidence/redact.ts +++ b/src/daemon/evidence/redact.ts @@ -3,36 +3,51 @@ * excerpt or shown to a model. Test output echoes tokens more often than one * would like: `set -x`, a failing HTTP fixture, a connection string in an error. * - * Uses the daemon's redaction rules (mcp/redact.ts), which scan a bounded - * window per call, so long text is redacted a group of lines at a time. A token - * never spans a line break in practice, so splitting on lines loses nothing. + * These rules are written for logs, not configuration. Test output is mostly + * file paths, test names, hashes, and exit codes, and a worker repairing a + * failure needs all of them, so nothing is masked for merely looking random: + * only known credential shapes and values bound to keys that name a secret. + * The whole text is scanned at once (every pattern is linear), so a PEM block + * or a long line is never split between two scans. */ -import { redactSecretText } from "../mcp/redact.js"; +const MASK = "…[redacted]"; -/** Kept under redactSecretText's scan window so every byte is scanned. */ -const CHUNK_CHARS = 1_800; +/** Key names whose values are secrets, in `key=value` / `key: value` form. */ +const SECRET_KEY = + "(?:[A-Za-z0-9_.-]{0,40}[_.-])?(?:password|passwd|pwd|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key|client[_-]?secret|auth[_-]?token|credentials?|session[_-]?key)"; + +const RULES: ReadonlyArray<[RegExp, string | ((...m: string[]) => string)]> = [ + // PEM blocks, terminated or cut off by the excerpt. + [/-----BEGIN [A-Z0-9 ]{1,40}-----[\s\S]*?(?:-----END [A-Z0-9 ]{1,40}-----|$)/g, MASK], + // JWTs. + [/\beyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}(?:\.[A-Za-z0-9_-]*)?/g, MASK], + // scheme://user:password@host — keep the scheme and host. + [/\b([a-z][a-z0-9+.-]{0,20}:\/\/)[^/\s:@]{0,200}:[^/\s@]{1,500}@/gi, `$1${MASK}@`], + // Authorization headers and schemes. + [/\b(authorization|proxy-authorization)(\s*[:=]\s*)(?:(bearer|basic|token|digest)\s+)?[^\s"',;]{4,}/gi, (_m, k, sep, scheme) => `${k}${sep}${scheme ? `${scheme} ` : ""}${MASK}`], + [/\b(bearer|basic)\s+[A-Za-z0-9._~+/=-]{8,}/gi, `$1 ${MASK}`], + // Well-known token formats. + [/\b(?:sk|pk|rk)-(?:ant-|proj-|live-|test-)?[A-Za-z0-9_-]{16,}/g, MASK], + [/\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{20,}/g, MASK], + [/\bgithub_pat_[A-Za-z0-9_]{20,}/g, MASK], + [/\bglpat-[A-Za-z0-9_-]{16,}/g, MASK], + [/\bxox[abposr]-[A-Za-z0-9-]{10,}/g, MASK], + [/\b(?:AKIA|ASIA)[0-9A-Z]{16}\b/g, MASK], + [/\bAIza[0-9A-Za-z_-]{30,}/g, MASK], + [/\b(?:zid_sk|hf_sk|npm)_[A-Za-z0-9_-]{16,}/g, MASK], + // key=value, key: value, "key": "value" — only for keys that name a secret. + [ + new RegExp(`(^|[^A-Za-z0-9])(${SECRET_KEY})(["']?\\s*[:=]\\s*)("[^"\\n]{1,500}"|'[^'\\n]{1,500}'|[^\\s"',;)}\\]]{1,500})`, "gim"), + (_m, pre, key, sep) => `${pre}${key}${sep}${MASK}`, + ], +]; export function redactOutput(text: string): string { if (!text) return text; - const out: string[] = []; - let chunk = ""; - const flush = (): void => { - if (chunk) out.push(redactSecretText(chunk)); - chunk = ""; - }; - for (const line of text.split("\n")) { - // A single line longer than the window is redacted in slices. - if (line.length > CHUNK_CHARS) { - flush(); - const slices: string[] = []; - for (let i = 0; i < line.length; i += CHUNK_CHARS) slices.push(redactSecretText(line.slice(i, i + CHUNK_CHARS))); - out.push(slices.join("")); - continue; - } - if (chunk.length + line.length + 1 > CHUNK_CHARS) flush(); - chunk = chunk ? `${chunk}\n${line}` : line; + let out = text; + for (const [pattern, replacement] of RULES) { + out = out.replace(pattern, replacement as string); } - flush(); - return out.join("\n"); + return out; } diff --git a/src/daemon/evidence/runner.test.ts b/src/daemon/evidence/runner.test.ts index 9e735b6c..d74a8861 100644 --- a/src/daemon/evidence/runner.test.ts +++ b/src/daemon/evidence/runner.test.ts @@ -1,9 +1,9 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import { existsSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, statSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { detectZeroTests } from "./parsers"; -import { EXCERPT_HEAD_BYTES, EXCERPT_TAIL_BYTES, runCheck, verifierDigest } from "./runner"; +import { EXCERPT_HEAD_BYTES, EXCERPT_TAIL_BYTES, checkEnv, runCheck, verifierDigest } from "./runner"; import type { CheckSpec } from "./types"; let dir: string; @@ -99,7 +99,7 @@ describe("output", () => { const out = await run(spec("i=0; while [ $i -lt 4000 ]; do echo line-$i; i=$((i+1)); done")); expect(out.stdoutExcerpt.startsWith("line-0\n")).toBe(true); expect(out.stdoutExcerpt).toContain("line-3999"); - expect(out.stdoutExcerpt).toContain("bytes omitted"); + expect(out.stdoutExcerpt).toContain("omitted — full output in the stdout artifact"); expect(out.stdoutExcerpt.length).toBeLessThan(EXCERPT_HEAD_BYTES + EXCERPT_TAIL_BYTES + 200); const stdout = out.artifacts.find((a) => a.name === "stdout")!; const full = readFileSync(stdout.path, "utf8"); @@ -218,3 +218,38 @@ describe("audit round 1", () => { expect(out.reason).toContain("working directory does not exist"); }); }); + +describe("audit round 2", () => { + test("an artifact file that cannot be written does not crash, and says it is truncated", async () => { + const art = join(dir, "art"); + mkdirSync(join(art, "stdout.log"), { recursive: true }); + const out = await runCheck(spec("echo still-classified; exit 3"), { cwd: dir, artifactDir: art }); + expect(out.result).toBe("fail"); + expect(out.stdoutExcerpt).toContain("still-classified"); + expect(out.artifacts.find((a) => a.name === "stdout")!.truncated).toBe(true); + }); + + test("checks keep the toolchain variables builds need, never the daemon's", () => { + const env = checkEnv({ + PATH: "/bin", + LC_ALL: "C.UTF-8", + CI: "1", + GOPATH: "/go", + CODEOID_API_KEY: "zid_sk_x", + CODEOID_CHECK_ENV_ALLOW: "MY_FLAG,CODEOID_API_KEY", + MY_FLAG: "on", + RANDOM_SECRET: "nope", + }); + expect(env.LC_ALL).toBe("C.UTF-8"); + expect(env.CI).toBe("1"); + expect(env.GOPATH).toBe("/go"); + expect(env.MY_FLAG).toBe("on"); + expect(env.CODEOID_API_KEY).toBeUndefined(); + expect(env.RANDOM_SECRET).toBeUndefined(); + }); + + test("output between the excerpt sizes loses no character at the seam", async () => { + const out = await run(spec("printf '%4095s' '' | tr ' ' a; printf 'é'; printf '%2000s' '' | tr ' ' b")); + expect(out.stdoutExcerpt).toContain("aé"); + }); +}); diff --git a/src/daemon/evidence/runner.ts b/src/daemon/evidence/runner.ts index 66aa0ab9..c1d142e5 100644 --- a/src/daemon/evidence/runner.ts +++ b/src/daemon/evidence/runner.ts @@ -71,6 +71,39 @@ function environmentOf(spec: CheckSpec): EvidenceEnvironment { }; } +/** + * The environment a check runs with: the agent subprocess basics plus the + * toolchain variables builds and test runners commonly need, and anything the + * operator lists in CODEOID_CHECK_ENV_ALLOW (comma-separated). Never the + * daemon's own credentials (CODEOID_ / ZEROID_ / TELEGRAM_ stay denied). + */ +const CHECK_ENV_EXACT = [ + "CI", + "NODE_ENV", + "TERM", + "GOPATH", + "GOFLAGS", + "GOPRIVATE", + "GOPROXY", + "GOCACHE", + "GOMODCACHE", + "CARGO_HOME", + "RUSTUP_HOME", + "JAVA_HOME", + "VIRTUAL_ENV", + "CONDA_PREFIX", + "PYTHONPATH", + "DOCKER_HOST", +]; + +export function checkEnv(base: Record = process.env): Record { + const extra = (base.CODEOID_CHECK_ENV_ALLOW ?? "") + .split(",") + .map((n) => n.trim()) + .filter((n) => n.length > 0 && !/^(CODEOID_|ZEROID_|TELEGRAM_)/.test(n)); + return buildSubprocessEnv({ exact: [...CHECK_ENV_EXACT, ...extra], prefixes: ["LC_", "XDG_"] }, base); +} + /** Bytes [start, end) of `buf` adjusted inward to whole UTF-8 characters, so a * cut never leaves a replacement character at either edge. */ function utf8Whole(buf: Buffer): Buffer { @@ -103,11 +136,18 @@ class CapturedStream { #headBytes = 0; #tail: Buffer[] = []; #tailBytes = 0; + /** The artifact file failed (disk full, too many open files): stop writing + * it, keep building excerpts. An unhandled stream error would take the + * whole daemon down. */ + #writeFailed = false; constructor(name: "stdout" | "stderr", dir: string, max: number, scan: ZeroTestScan) { this.#name = name; this.#path = join(dir, `${name}.log`); this.#file = createWriteStream(this.#path, { mode: 0o600 }); + this.#file.on("error", () => { + this.#writeFailed = true; + }); this.#max = max; this.#lines = new LineSplitter((l) => scan.line(l)); } @@ -115,7 +155,7 @@ class CapturedStream { push(chunk: Buffer): void { this.#bytes += chunk.length; this.#lines.push(chunk.toString("utf8")); - if (this.#written < this.#max) { + if (!this.#writeFailed && this.#written < this.#max) { const room = this.#max - this.#written; const part = chunk.length <= room ? chunk : chunk.subarray(0, room); this.#file.write(part); @@ -140,28 +180,45 @@ class CapturedStream { /** The excerpt, redacted: the head, then — if output was longer — a marker * and the tail. */ excerpt(): string { - const head = utf8Whole(Buffer.concat(this.#head)).toString("utf8"); - if (this.#tailBytes === 0) return redactOutput(head); let tail = Buffer.concat(this.#tail); let skipped = this.#bytes - this.#headBytes - tail.length; if (tail.length > EXCERPT_TAIL_BYTES) { skipped += tail.length - EXCERPT_TAIL_BYTES; tail = tail.subarray(tail.length - EXCERPT_TAIL_BYTES); } - const marker = skipped > 0 ? `\n… [${skipped} bytes omitted — full output in the ${this.#name} artifact] …\n` : ""; - return redactOutput(head + marker + utf8Whole(tail).toString("utf8")); + const headBuf = Buffer.concat(this.#head); + // Nothing omitted: one contiguous text, decoded as one so no character is + // lost at the head/tail seam. + if (skipped <= 0) return redactOutput(Buffer.concat([headBuf, tail]).toString("utf8")); + // Cut at line boundaries, so redaction never sees half a token at an edge. + let head = utf8Whole(headBuf).toString("utf8"); + const headCut = head.lastIndexOf("\n"); + if (headCut > 0) head = head.slice(0, headCut + 1); + let rest = utf8Whole(tail).toString("utf8"); + const tailCut = rest.indexOf("\n"); + if (tailCut >= 0 && tailCut < rest.length - 1) rest = rest.slice(tailCut + 1); + const marker = `\n… [${skipped} bytes or more omitted — full output in the ${this.#name} artifact] …\n`; + return redactOutput(head + marker + rest); } - /** Flush the artifact file and describe it. */ + /** Flush the artifact file and describe it. Resolves even when the file + * failed; the artifact then says it is truncated. */ async close(): Promise { this.#lines.end(); - await new Promise((res) => this.#file.end(res)); + if (!this.#writeFailed) { + await new Promise((res) => { + this.#file.once("error", () => res()); + this.#file.end(() => res()); + }); + } else { + this.#file.destroy(); + } return { name: this.#name, path: this.#path, sha256: this.#hash.digest("hex"), bytes: this.#written, - truncated: this.#bytes > this.#written, + truncated: this.#writeFailed || this.#bytes > this.#written, }; } } @@ -222,7 +279,7 @@ export async function runCheck(spec: CheckSpec, opts: RunCheckOptions): Promise< const child = spawn("sh", ["-c", spec.command], { cwd: opts.cwd, - env: { ...buildSubprocessEnv({}), ...(spec.env ?? {}) }, + env: { ...checkEnv(), ...(spec.env ?? {}) }, stdio: ["ignore", "pipe", "pipe"], // Its own process group, so killGroup reaches everything it spawns. detached: true, diff --git a/src/daemon/evidence/service.test.ts b/src/daemon/evidence/service.test.ts index 6429b560..f932696f 100644 --- a/src/daemon/evidence/service.test.ts +++ b/src/daemon/evidence/service.test.ts @@ -76,6 +76,28 @@ describe("EvidenceService", () => { expect((await other).record?.result).toBe("pass"); }); + test("an aborted owner's next check is cancelled before it starts", async () => { + const svc = new EvidenceService(new EvidenceStore(":memory:"), join(dir, "evidence")); + svc.abortOwner("pipeline", "pipe-1"); + const marker = join(dir, "ran"); + const { record } = await svc.run({ id: "c", command: `touch ${marker}`, timeoutMs: 5_000 }, dir, ctx); + expect(record?.result).toBe("cancelled"); + expect(existsSync(marker)).toBe(false); + svc.clearAbort("pipeline", "pipe-1"); + expect((await svc.run({ id: "c", command: "true", timeoutMs: 5_000 }, dir, ctx)).record?.result).toBe("pass"); + }); + + test("retention never sweeps the directory of a check still running", async () => { + const root = join(dir, "evidence"); + const svc = new EvidenceService(new EvidenceStore(":memory:"), root); + const running = svc.run({ id: "c", command: "sleep 0.5", timeoutMs: 5_000 }, dir, ctx); + await Bun.sleep(100); + // Age every directory past the orphan threshold, then sweep. + svc.sweep({ maxAgeMs: 1e12, maxTotalBytes: 1e12 }, Date.now() + 2 * 60 * 60 * 1000); + const { record } = await running; + expect(existsSync(record!.artifacts[0]!.path)).toBe(true); + }); + test("retention drops old records, then the oldest past the size cap, with their files", async () => { const svc = new EvidenceService(new EvidenceStore(":memory:"), join(dir, "evidence")); const a = (await svc.run({ id: "a", command: "head -c 3000 /dev/zero", timeoutMs: 5_000 }, dir, ctx)).record!; diff --git a/src/daemon/evidence/service.ts b/src/daemon/evidence/service.ts index d643b6b0..3b226a30 100644 --- a/src/daemon/evidence/service.ts +++ b/src/daemon/evidence/service.ts @@ -54,6 +54,13 @@ export class EvidenceService { readonly #artifactRoot: string; /** Live checks by owner, so an owner or the daemon can stop them. */ readonly #live = new Map>(); + /** Artifact directories of checks still running — never swept. */ + readonly #liveDirs = new Set(); + /** Owners whose checks were stopped: anything they start afterwards is + * cancelled before it runs, so an abort can't miss a check that was about + * to start. Cleared with `clearAbort` when the owner can run again. */ + readonly #aborted = new Set(); + #stopping = false; /** `artifactRoot` holds one directory per check's full output. */ constructor(store: EvidenceStore, artifactRoot: string) { @@ -63,18 +70,21 @@ export class EvidenceService { /** Run `spec` in `cwd` and persist the result. */ async run(spec: CheckSpec, cwd: string, ctx: EvidenceContext): Promise { - const artifactDir = join(this.#artifactRoot, ctx.ownerKind, safeSegment(ctx.ownerId), randomUUID()); + const relDir = join(ctx.ownerKind, safeSegment(ctx.ownerId), randomUUID()); + const artifactDir = join(this.#artifactRoot, relDir); const key = ownerKey(ctx.ownerKind, ctx.ownerId); const ac = new AbortController(); + if (this.#stopping || this.#aborted.has(key)) ac.abort(); const onAbort = (): void => ac.abort(); ctx.signal?.addEventListener("abort", onAbort, { once: true }); if (ctx.signal?.aborted) ac.abort(); const live = this.#live.get(key) ?? new Set(); live.add(ac); this.#live.set(key, live); + this.#liveDirs.add(relDir); try { const outcome = await runCheck(spec, { cwd, artifactDir, signal: ac.signal }); - const record = this.#store.insert({ + const record = this.#store.insert(relDir, { ...outcome, accountId: ctx.tenant.accountId, projectId: ctx.tenant.projectId, @@ -85,6 +95,7 @@ export class EvidenceService { return { outcome, record }; } finally { ctx.signal?.removeEventListener("abort", onAbort); + this.#liveDirs.delete(relDir); live.delete(ac); if (live.size === 0) this.#live.delete(key); } @@ -95,13 +106,22 @@ export class EvidenceService { return (spec, cwd) => this.run(spec, cwd, ctx); } - /** Stop every check running for one owner (e.g. a pipeline being aborted). */ + /** Stop every check running for one owner (e.g. a pipeline being aborted), + * and any it starts until `clearAbort`. */ abortOwner(ownerKind: EvidenceOwnerKind, ownerId: string): void { - for (const ac of this.#live.get(ownerKey(ownerKind, ownerId)) ?? []) ac.abort(); + const key = ownerKey(ownerKind, ownerId); + this.#aborted.add(key); + for (const ac of this.#live.get(key) ?? []) ac.abort(); } - /** Stop every running check (daemon shutdown). */ + /** Let an owner run checks again after `abortOwner`. */ + clearAbort(ownerKind: EvidenceOwnerKind, ownerId: string): void { + this.#aborted.delete(ownerKey(ownerKind, ownerId)); + } + + /** Stop every running check, and refuse new ones (daemon shutdown). */ abortAll(): void { + this.#stopping = true; for (const set of this.#live.values()) for (const ac of set) ac.abort(); } @@ -129,11 +149,18 @@ export class EvidenceService { } const dirs = this.#store.artifactDirsFor([...doomed]); this.#store.deleteIds([...doomed]); - for (const d of dirs) rmSync(d, { recursive: true, force: true }); - this.#removeOrphanDirs(new Set(this.#store.allArtifactDirs()), now); + for (const d of dirs) rmSync(this.#inRoot(d), { recursive: true, force: true }); + this.#removeOrphanDirs(new Set([...this.#store.allArtifactDirs(), ...this.#liveDirs]), now); return doomed.size; } + /** A stored artifact directory (relative to the root) as an absolute path, + * refusing anything that would resolve outside the root. */ + #inRoot(rel: string): string { + const abs = join(this.#artifactRoot, rel); + return abs.startsWith(`${this.#artifactRoot}/`) ? abs : join(this.#artifactRoot, "_invalid_"); + } + /** Remove check directories under the root that no record points at, once * they are old enough not to belong to a check still running. */ #removeOrphanDirs(known: Set, now: number): void { @@ -149,7 +176,7 @@ export class EvidenceService { const ownerDir = join(this.#artifactRoot, kind, owner); for (const check of safeReaddir(ownerDir)) { const dir = join(ownerDir, check); - if (known.has(dir)) continue; + if (known.has(join(kind, owner, check))) continue; try { if (now - statSync(dir).mtimeMs < minAgeMs) continue; } catch { diff --git a/src/daemon/evidence/store.ts b/src/daemon/evidence/store.ts index 4d4ee5c9..b897b25a 100644 --- a/src/daemon/evidence/store.ts +++ b/src/daemon/evidence/store.ts @@ -10,7 +10,6 @@ import { Database } from "bun:sqlite"; import { randomUUID } from "node:crypto"; -import { dirname } from "node:path"; import { BUSY_TIMEOUT_MS } from "../store.js"; import type { EvidenceArtifact, @@ -93,8 +92,10 @@ export class EvidenceStore { `); } - /** Persist a record and return it with its id. */ - insert(record: Omit): EvidenceRecord { + /** Persist a record and return it with its id. `artifactDir` is the + * record's directory relative to the artifact root, so moving or + * re-pathing the data directory never orphans it. */ + insert(artifactDir: string, record: Omit): EvidenceRecord { const id = `ev_${randomUUID()}`; this.#db .prepare( @@ -127,7 +128,7 @@ export class EvidenceStore { record.stdoutExcerpt, record.stderrExcerpt, JSON.stringify(record.artifacts), - record.artifacts[0] ? dirname(record.artifacts[0].path) : null, + artifactDir, record.artifacts.reduce((n, a) => n + a.bytes, 0), ); return { ...record, id }; diff --git a/src/daemon/pipeline/engine.ts b/src/daemon/pipeline/engine.ts index 3aff55f9..9dfc0362 100644 --- a/src/daemon/pipeline/engine.ts +++ b/src/daemon/pipeline/engine.ts @@ -57,6 +57,13 @@ const MAX_STEPS = 10_000; const clone = (x: T): T => structuredClone(x); const now = (): number => Date.now(); +/** A gate's check was stopped before its verdict — the step is dropped. */ +export class GateCancelled extends Error { + constructor(readonly gateId: string) { + super(`gate "${gateId}" was cancelled before it reached a verdict`); + } +} + /** Evidence kept per phase in the run state; older ids stay in the store. */ const PHASE_EVIDENCE_MAX = 50; @@ -275,7 +282,15 @@ export class PipelineEngine { let s = state; let guard = 0; while ((s.status === "draft" || s.status === "running") && guard++ < MAX_STEPS) { - s = await this.step(s); + try { + s = await this.step(s); + } catch (err) { + // A gate's check was stopped (abort, shutdown): drop the step unsaved + // and stop driving. The last persisted state re-runs the gate when the + // pipeline resumes; it is never recorded as a failed check. + if (err instanceof GateCancelled) return s; + throw err; + } if (onProgress) await onProgress(s); } // Guard tripped (a mis-authored retry loop) — fail terminally rather than @@ -466,19 +481,21 @@ export class PipelineEngine { ownerKind: "pipeline", ownerId: pipeline.id, }); + let verdict: GateVerdict; try { - const verdict = await g.evaluate({ pipeline: clone(pipeline), phase, ...(runCheck ? { runCheck } : {}) }); - // Every gate runs for the active phase, so its evidence belongs there. - const active = pipeline.phases[pipeline.cursor]; - if (active && verdict.evidenceIds?.length) { - active.evidence = [...(active.evidence ?? []), ...verdict.evidenceIds].slice(-PHASE_EVIDENCE_MAX); - } - return verdict; + verdict = await g.evaluate({ pipeline: clone(pipeline), phase, ...(runCheck ? { runCheck } : {}) }); } catch (err) { // A throwing gate is a failing verdict, not a crash — same reasoning as // the phase kind above: never leave the pipeline stuck mid-advance. return { pass: false, reason: `${at} gate "${id}" threw: ${errMessage(err)}` }; } + if (verdict.cancelled) throw new GateCancelled(id); + // Every gate runs for the active phase, so its evidence belongs there. + const active = pipeline.phases[pipeline.cursor]; + if (active && verdict.evidenceIds?.length) { + active.evidence = [...(active.evidence ?? []), ...verdict.evidenceIds].slice(-PHASE_EVIDENCE_MAX); + } + return verdict; } } diff --git a/src/daemon/pipeline/gate-probes.ts b/src/daemon/pipeline/gate-probes.ts index 94c86b6b..d0d89153 100644 --- a/src/daemon/pipeline/gate-probes.ts +++ b/src/daemon/pipeline/gate-probes.ts @@ -219,9 +219,16 @@ export function buildProbeGate( if (spec.type === "git-diff-nonempty") { const { outcome, evidenceId } = await runProbe(ctx, id, "git-diff", "test -n \"$(git status --porcelain)\"", workdir); const evidenceIds = evidenceId ? [evidenceId] : []; - return outcome.result === "pass" - ? { pass: true, evidenceIds } - : { pass: false, reason: `probe "${id}": the working tree has no changes`, evidenceIds }; + if (outcome.result === "cancelled") return { pass: false, cancelled: true, reason: "check cancelled", evidenceIds }; + if (outcome.result === "pass") return { pass: true, evidenceIds }; + return { + pass: false, + reason: + outcome.result === "fail" + ? `probe "${id}": the working tree has no changes` + : `probe "${id}": could not inspect the working tree — ${outcome.reason}`, + evidenceIds, + }; } // ── Execution probes (trust-gated) ──────────────────────────────────── @@ -243,6 +250,7 @@ export function buildProbeGate( for (const { label, cmd } of cmds) { const { outcome, evidenceId } = await runProbe(ctx, id, label, cmd, workdir); if (evidenceId) evidenceIds.push(evidenceId); + if (outcome.result === "cancelled") return { pass: false, cancelled: true, reason: "check cancelled", evidenceIds }; if (outcome.result !== "pass") { const tail = diagnosticTail(outcome); return { diff --git a/src/daemon/pipeline/interface.ts b/src/daemon/pipeline/interface.ts index a20e0f34..bbd3aa5e 100644 --- a/src/daemon/pipeline/interface.ts +++ b/src/daemon/pipeline/interface.ts @@ -219,6 +219,10 @@ export interface GateVerdict { questions?: string[]; /** Evidence the gate recorded for this verdict, in the order it ran. */ evidenceIds?: string[]; + /** The gate's check was stopped (pipeline abort, daemon shutdown) before it + * reached a verdict. Not a failure: the step is dropped, unsaved, so the + * gate runs again when the pipeline resumes. */ + cancelled?: boolean; } /** A gate: a pass/fail predicate positioned at phase entry (grounding) or exit diff --git a/src/daemon/pipeline/pack.test.ts b/src/daemon/pipeline/pack.test.ts index 3a876c30..d13e72a6 100644 --- a/src/daemon/pipeline/pack.test.ts +++ b/src/daemon/pipeline/pack.test.ts @@ -403,6 +403,28 @@ phases: rmSync(root, { recursive: true, force: true }); }); + test("a gate check stopped by an abort is dropped, never recorded as a failure", async () => { + const db = new Database(":memory:"); + const root = mkdtempSync(join(tmpdir(), "pack-evidence-")); + const evidence = new EvidenceService(new EvidenceStore(db), root); + const mgr = new PipelineManager(new PipelineStore(db), { evidence }); + mgr.installPack(loadPack(noopPack(' - id: slow\n kind: command\n run: "sleep 30"', "slow"), { trusted: true })); + const p = mgr.create({ name: "run", pack: "noop-pack", ...tenant }); + const advancing = mgr.advance(p.id); + await Bun.sleep(300); + evidence.abortOwner("pipeline", p.id); + const after = await advancing; + // Not halted, not failed, not retried: the step was dropped, so the run + // stays at its last saved state and re-runs the gate when resumed. + expect(["draft", "running"]).toContain(after.status); + const stored = mgr.get(p.id)!; + expect(["draft", "running"]).toContain(stored.status); + expect(stored.phases[0].state.status).not.toBe("halted"); + expect(stored.phases[0].state.status).not.toBe("failed"); + expect(stored.phases[0].feedback ?? []).toEqual([]); + rmSync(root, { recursive: true, force: true }); + }); + test("an UNTRUSTED command gate never executes — fails closed (registry default)", async () => { const mgr = new PipelineManager(new PipelineStore(new Database(":memory:"))); // No { trusted } → default false. Even a "true" command must not run. diff --git a/src/daemon/pipeline/pack.ts b/src/daemon/pipeline/pack.ts index 67aaad66..cf69be4e 100644 --- a/src/daemon/pipeline/pack.ts +++ b/src/daemon/pipeline/pack.ts @@ -500,6 +500,7 @@ function buildGate(g: PackManifest["gates"][number], dir: string, trusted: boole cwd, ); const evidenceIds = record ? [record.id] : []; + if (outcome.result === "cancelled") return { pass: false, cancelled: true, reason: "check cancelled", evidenceIds }; if (outcome.result === "pass") return { pass: true, evidenceIds }; const tail = diagnosticTail(outcome); return { diff --git a/web/src/state/evidence.ts b/web/src/state/evidence.ts index d0979362..2a76f058 100644 --- a/web/src/state/evidence.ts +++ b/web/src/state/evidence.ts @@ -1,20 +1,28 @@ /** * Evidence reads (docs/loop-engineering-design.md §7): fetch one record of an - * executed check by id. Records are immutable, so a fetched record is cached - * for the life of the page. + * executed check by id. Records are immutable, so a fetched record is cached — + * per connection (a new token or project starts empty, so nothing is served + * that the current identity hasn't been authorized for) and bounded. */ import type { EvidenceRecordMsg, EvidenceWire } from "../protocol/types"; import { getClient, newRequestId } from "./connection"; const READ_TIMEOUT_MS = 15_000; -const cache = new Map(); +const CACHE_MAX = 200; +const caches = new WeakMap>(); export async function fetchEvidence(evidenceId: string): Promise { + const client = getClient(); + let cache = caches.get(client); + if (!cache) { + cache = new Map(); + caches.set(client, cache); + } const hit = cache.get(evidenceId); if (hit) return hit; const id = newRequestId(); - const msg = await getClient().request( + const msg = await client.request( { type: "evidence.get", id, evidenceId }, { waitForResult: (m) => (m.type === "evidence.record" && m.requestId === id ? m : undefined), @@ -22,9 +30,6 @@ export async function fetchEvidence(evidenceId: string): Promise { }, ); cache.set(evidenceId, msg.evidence); + if (cache.size > CACHE_MAX) cache.delete(cache.keys().next().value!); return msg.evidence; } - -export function _resetEvidenceForTest(): void { - cache.clear(); -}