diff --git a/packages/protocol/src/types.ts b/packages/protocol/src/types.ts index db8b51ad..13e0538c 100644 --- a/packages/protocol/src/types.ts +++ b/packages/protocol/src/types.ts @@ -440,6 +440,12 @@ export interface SessionInfo { turnQueued?: true; /** Remaining turns budget for autonomous mode (undefined = unbounded, 0 = exhausted). */ turnsRemaining?: number; + /** + * An unattended worker (onBudgetExhausted "end_attempt") spent its + * autonomous tool budget: its tool calls are refused until a person sets a + * mode or sends a message, or its controller sets a new budget. Absent = not. + */ + budgetExhausted?: true; /** Files pinned to the session — prepended to every turn's prompt. */ pinnedFiles?: string[]; /** SPIFFE/WIMSE URI of the primary session agent (falls back to anonymous:session:). */ diff --git a/src/daemon/process-tree.ts b/src/daemon/process-tree.ts new file mode 100644 index 00000000..db4ca61c --- /dev/null +++ b/src/daemon/process-tree.ts @@ -0,0 +1,179 @@ +/** + * Killing what a backend left running (Session.terminateBackend, supervisor.ts). + * + * Tearing a backend down kills ITS process; what that process had started — + * the shell a tool runs in, and whatever that shell runs — does not always go + * with it. Measured against the Claude Code CLI (SDK 0.3.281, a fake Messages + * API driving a real `Bash: sleep` call): + * + * - abort (SIGTERM via the SDK) and Query.interrupt(): the CLI kills the + * tool's shell. Nothing survives. + * - the CLI SIGKILLed (the SDK's own escalation 5s after a SIGTERM the CLI + * didn't act on): the tool's `sleep` SURVIVES, re-parented. Each Bash + * call runs in its OWN process group (pgid = the shell's pid), so killing + * the CLI's process group would not reach it either. + * - a command that detaches itself (`setsid …`, `nohup … &`): survives every + * path — it left the tree before anything here looks at it. + * + * codex and gemini-cli are stopped with a bare SIGKILL (jsonrpc-stdio.ts), so + * their running commands are in the second case on every teardown. + * + * So the stop is a TREE kill, not a group kill: snapshot every descendant of + * the backend's own process(es) before the teardown, and after it SIGKILL any + * of them still alive — with everything under them, including children they + * spawned after the snapshot. A survivor is matched by pid, command line AND + * start time, so a pid the OS reused in between is never touched. What this cannot reach, and does + * not claim to: anything that detached itself before the snapshot (only a + * cgroup or sandbox boundary bounds that), and a backend whose own process + * ids are unknown (qwen-code: @qwen-code/sdk spawns its CLI internally). + * + * Uses `ps` (Linux, macOS). Unavailable `ps` or Windows: empty snapshots, so + * terminateBackend degrades to the plain teardown. + */ + +import { execFile } from "node:child_process"; +import { readFileSync } from "node:fs"; + +export interface ProcessEntry { + pid: number; + ppid: number; + args: string; + /** + * When the process started, as an opaque comparable token — Linux: + * /proc//stat field 22 (clock ticks since boot); macOS: ps `lstart`. + * With pid and command line, the identity a survivor must still match. + * Absent when it could not be read (the process is gone, or no /proc). + */ + start?: string; +} + +const PS_TIMEOUT_MS = 3_000; +const DARWIN = process.platform === "darwin"; + +/** `ps` with no COLUMNS (it would truncate `args` even with -ww on some systems). */ +function psEnv(): Record { + const env: Record = {}; + for (const [k, v] of Object.entries(process.env)) if (v !== undefined && k !== "COLUMNS") env[k] = v; + return env; +} + +/** Every process visible to this user: pid, parent pid, full command line (and start on macOS). */ +export function listProcesses(): Promise { + if (process.platform === "win32") return Promise.resolve([]); + const columns = DARWIN ? "pid=,ppid=,lstart=,args=" : "pid=,ppid=,args="; + return new Promise((resolve) => { + execFile( + "ps", + ["-A", "-ww", "-o", columns], + { timeout: PS_TIMEOUT_MS, maxBuffer: 32 * 1024 * 1024, env: psEnv() }, + (err, stdout) => { + if (err) { + resolve([]); + return; + } + resolve(parsePs(stdout, { lstart: DARWIN })); + }, + ); + }); +} + +/** + * Parse `ps -o pid=,ppid=[,lstart=],args=` output. `lstart` is five fields + * ("Sat Oct 24 10:08:43 2026") kept verbatim as the start token. Exported for tests. + */ +export function parsePs(stdout: string, opts: { lstart?: boolean } = {}): ProcessEntry[] { + const out: ProcessEntry[] = []; + const line = opts.lstart ? /^\s*(\d+)\s+(\d+)\s+(\S+\s+\S+\s+\S+\s+\S+\s+\S+)\s+(.*)$/ : /^\s*(\d+)\s+(\d+)\s+(.*)$/; + for (const text of stdout.split("\n")) { + const m = line.exec(text); + if (!m) continue; + if (opts.lstart) out.push({ pid: Number(m[1]), ppid: Number(m[2]), start: m[3]!.replace(/\s+/g, " "), args: m[4]!.trim() }); + else out.push({ pid: Number(m[1]), ppid: Number(m[2]), args: m[3]!.trim() }); + } + return out; +} + +/** Linux: /proc//stat field 22 (starttime). The comm field may hold spaces and parens, so count from the last ")". */ +export function procStartTime(pid: number): string | undefined { + try { + const stat = readFileSync(`/proc/${pid}/stat`, "utf8"); + const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" "); + // fields[0] is field 3 (state); field 22 is fields[19]. + return fields[19]; + } catch { + return undefined; + } +} + +/** Fill `start` where ps didn't (Linux) — only for the entries that matter. */ +function withStart(entries: ProcessEntry[]): ProcessEntry[] { + if (process.platform !== "linux") return entries; + return entries.map((p) => (p.start !== undefined ? p : { ...p, ...optional(procStartTime(p.pid)) })); +} +function optional(start: string | undefined): { start?: string } { + return start === undefined ? {} : { start }; +} + +/** `roots` (those present) and all their descendants in `table`. Never this process. */ +export function treeOf(table: readonly ProcessEntry[], roots: readonly number[]): ProcessEntry[] { + const children = new Map(); + const byPid = new Map(); + for (const p of table) { + byPid.set(p.pid, p); + const list = children.get(p.ppid); + if (list) list.push(p); + else children.set(p.ppid, [p]); + } + const out: ProcessEntry[] = []; + const seen = new Set([process.pid]); + const queue = roots.filter((pid) => byPid.has(pid) && !seen.has(pid)); + for (const pid of queue) seen.add(pid); + while (queue.length > 0) { + const pid = queue.shift()!; + out.push(byPid.get(pid)!); + for (const child of children.get(pid) ?? []) { + if (seen.has(child.pid)) continue; + seen.add(child.pid); + queue.push(child.pid); + } + } + return out; +} + +/** The process tree under `roots` right now, each entry with its start time. */ +export async function snapshotTree(roots: readonly number[]): Promise { + if (roots.length === 0) return []; + return withStart(treeOf(await listProcesses(), roots)); +} + +/** Same process: pid, command line, and (when both are known) start time. */ +function sameProcess(then: ProcessEntry, now: ProcessEntry): boolean { + if (then.pid !== now.pid || then.args !== now.args) return false; + return then.start === undefined || now.start === undefined || then.start === now.start; +} + +/** + * SIGKILL what is left of `snapshot`: every entry still running as the SAME + * process (pid + command line + start time — a reused pid is spared), and + * every descendant those have now, including children spawned after the + * snapshot was taken. One `ps` call. Returns the pids signalled. + */ +export async function killSurvivors(snapshot: readonly ProcessEntry[]): Promise { + if (snapshot.length === 0) return []; + const table = await listProcesses(); + const now = new Map(table.map((p) => [p.pid, p])); + const confirmed = withStart(snapshot.map((p) => now.get(p.pid)).filter((p): p is ProcessEntry => p !== undefined)) + .filter((live) => sameProcess(snapshot.find((p) => p.pid === live.pid)!, live)) + .map((p) => p.pid); + const killed: number[] = []; + for (const p of treeOf(table, confirmed)) { + if (p.pid === process.pid) continue; + try { + process.kill(p.pid, "SIGKILL"); + killed.push(p.pid); + } catch { + // already gone, or not ours to signal + } + } + return killed; +} diff --git a/src/daemon/providers/acp/index.ts b/src/daemon/providers/acp/index.ts index 1afae808..611ded93 100644 --- a/src/daemon/providers/acp/index.ts +++ b/src/daemon/providers/acp/index.ts @@ -38,7 +38,7 @@ import type { CatalogEntry, } from "../interface.js"; import { renderHistorySeed, type CanonicalTurn, type HistorySeedResult } from "../canonical.js"; -import { buildGeminiCliEnv } from "../env.js"; +import { buildGeminiCliEnv, withExtraEnv } from "../env.js"; import { StdioJsonRpcProcess } from "../jsonrpc-stdio.js"; import { MEMORY_MCP_SERVER_NAME, type MemoryMcpMount } from "../../memory/mcp-http.js"; import { BLACKBOARD_MCP_SERVER_NAME } from "../../blackboard/mcp-http.js"; @@ -89,6 +89,8 @@ export interface GeminiAcpProviderInit { */ resolveAccount?: () => ResolvedAccount | null; onModels?: (models: ReadonlyArray) => void; + /** Caller env for the child process (ProviderSessionInit.extraEnv), merged last via withExtraEnv. */ + extraEnv?: Readonly>; } export class GeminiAcpProvider implements SessionProvider { @@ -109,6 +111,7 @@ export class GeminiAcpProvider implements SessionProvider { #command: string; #argsPrefix: string[]; #resolveAccount: (() => ResolvedAccount | null) | null; + readonly #extraEnv: Readonly> | undefined; #workspaceId: string; #memoryMcp: MemoryMcpMount | null; /** Role-scoped blackboard mount; token minted+revoked by the SessionManager. */ @@ -144,6 +147,7 @@ export class GeminiAcpProvider implements SessionProvider { this.#backingSessionId = init.initialBackingId; this.#command = init.command; this.#resolveAccount = init.resolveAccount ?? null; + this.#extraEnv = init.extraEnv; this.#argsPrefix = init.argsPrefix ?? []; this.#workspaceId = init.workspaceId ?? init.sessionId; this.#memoryMcp = init.memoryMcp ?? null; @@ -248,6 +252,11 @@ export class GeminiAcpProvider implements SessionProvider { await this.teardown(); } + async backendProcessIds(): Promise { + const pid = this.#proc?.pid; + return pid ? [pid] : []; + } + async teardown(): Promise { this.#turnQueue?.close(); this.#turnQueue = null; @@ -315,7 +324,11 @@ export class GeminiAcpProvider implements SessionProvider { cwd: opts.workdir, // The account's own GEMINI_CLI_HOME (an overlay selecting its auth type). // ...and, for a thinking effort, an overlay of that whose settings set it (#417). - env: applyGeminiCliEffort(opts.effort, applyGeminiCliAccount(this.#resolveAccount?.() ?? null, buildGeminiCliEnv())), + // Then the caller's extra env, which can't override either (withExtraEnv). + env: withExtraEnv( + applyGeminiCliEffort(opts.effort, applyGeminiCliAccount(this.#resolveAccount?.() ?? null, buildGeminiCliEnv())), + this.#extraEnv, + ), onNotification: (method, params) => this.#onNotification(method, params), onServerRequest: (method, params) => this.#onServerRequest(method, params), onExit: ({ code, signal, stderrTail }) => { diff --git a/src/daemon/providers/claude/index.ts b/src/daemon/providers/claude/index.ts index f7366ca5..1f7b7158 100644 --- a/src/daemon/providers/claude/index.ts +++ b/src/daemon/providers/claude/index.ts @@ -66,7 +66,8 @@ import { removeClaudeSession, toClaudeSessionEntries, writeClaudeSession } from import { BUDGET_STOP_MESSAGE, isBackgroundLifecycleEvent } from "../interface.js"; import { isUngrantableSkillCommand, redactCommand } from "../../skill-command.js"; import { renderHistorySeed, type CanonicalTurn, type HistorySeedResult } from "../canonical.js"; -import { buildSubprocessEnv, withGatewayCredential } from "../env.js"; +import { buildSubprocessEnv, withExtraEnv, withGatewayCredential } from "../env.js"; +import { listProcesses } from "../../process-tree.js"; import type { LLMCallUsage } from "../../context-math.js"; import type { PackSubagent } from "../../pipeline/subagents.js"; @@ -132,6 +133,8 @@ export interface ClaudeProviderInit { * switch takes effect on the next turn and RESUMES the same backing session. */ resolveAccount?: () => ResolvedClaudeAccount | null; + /** Caller env for the CLI child process (ProviderSessionInit.extraEnv), merged last via withExtraEnv. */ + extraEnv?: Readonly>; /** codeoid session id — used as the initial backing id and for audit/logging. */ sessionId: string; /** Test seam for native continuation (#387); defaults to the real SDK. */ @@ -235,6 +238,8 @@ export class ClaudeProvider implements SessionProvider { // Claude-specific backing-session state #claudeCodeSessionId: string; + /** The backing id the live query's CLI was spawned with (backendProcessIds). */ + #spawnedWithId: string | null = null; #hasQueried = false; #backingRecoveryAttempted = false; #lastPushedContent: string | null = null; @@ -644,6 +649,19 @@ export class ClaudeProvider implements SessionProvider { await this.teardown(); } + /** + * The Claude Code CLI this provider's query runs in. The SDK spawns it + * (spawnClaudeCodeProcess is not used: replacing the SDK's spawn would drop + * its stderr diagnostics), so it is found as a direct child of this process + * whose argv names this provider's backing session as an exact token + * (argvNamesSession — ids are UUIDs, unique per session). + */ + async backendProcessIds(): Promise { + if (!this.#query) return []; + const id = this.#spawnedWithId ?? this.#claudeCodeSessionId; + return (await listProcesses()).filter((p) => p.ppid === process.pid && argvNamesSession(p.args, id)).map((p) => p.pid); + } + async teardown(): Promise { this.#inputQueue?.close(); this.#abortController?.abort(); @@ -829,6 +847,8 @@ export class ClaudeProvider implements SessionProvider { const sessionOpts = this.#hasQueried ? { resume: this.#claudeCodeSessionId } : { sessionId: this.#claudeCodeSessionId }; + // What this query's CLI was started with — its argv names it (backendProcessIds). + this.#spawnedWithId = this.#claudeCodeSessionId; const merged = this.#mcpServerSet(opts, oauthMcp); const mcpServers = Object.keys(merged).length > 0 ? merged : undefined; @@ -859,11 +879,18 @@ export class ClaudeProvider implements SessionProvider { // applyAccountEnv). The forge gateway re-point applies only to the // daemon's own environment: a named account carries the user's own // gateway token, which must never be swapped for an agent identity. - env: account - ? account.apply(buildAgentEnv(process.env)) - : buildAgentEnv(process.env, { - gatewayCredential: init.identityManager?.getGatewayCredential(init.sessionId), - }), + // + // The caller's extra env (a loop worker's) goes on last; withExtraEnv + // refuses every ANTHROPIC_/CLAUDE_ and credential-shaped key, so it + // can never undo the account or the gateway re-point. + env: withExtraEnv( + account + ? account.apply(buildAgentEnv(process.env)) + : buildAgentEnv(process.env, { + gatewayCredential: init.identityManager?.getGatewayCredential(init.sessionId), + }), + init.extraEnv, + ), allowedTools: [ // Every EXACT TOOL NAME here is auto-approved by the SDK BEFORE // canUseTool is consulted, so it never reaches our gate — the SDK @@ -1564,6 +1591,22 @@ export interface TranslateState { cwd?: string; } +/** + * Whether a Claude CLI command line names backing session `id` as an EXACT + * argv token — the forms the SDK emits (`--resume=`, `--session-id=`, + * SDK 0.3.281) and their two-token spelling. A substring match would also hit + * a process that merely mentions the id (a prompt, a log path). Exported for tests. + */ +export function argvNamesSession(args: string, id: string): boolean { + const argv = args.split(/\s+/); + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === `--resume=${id}` || a === `--session-id=${id}`) return true; + if ((a === "--resume" || a === "--session-id") && argv[i + 1] === id) return true; + } + return false; +} + /** * Translate one SDKMessage from the Claude Agent SDK into zero or more * ProviderEvents. Pure function — no I/O, no side-effects beyond calling @@ -1742,6 +1785,11 @@ export function translateSDKMessage( // Only a turn that completed is a point to fork at: an errored one // (a context overflow) would carry the failure into the fork, and an // interrupted one can end mid tool call. + // A turn the CLI ends "successfully" but flagged as an error carries + // its notice (a usage limit, "API Error: …") as the result text. + ...(r.is_error && r.subtype === "success" && typeof r.result === "string" && r.result + ? { resultText: r.result } + : {}), ...(sessionId && chainEnd && !zeroTurn && r.subtype === "success" && !r.is_error ? { nativeRef: { backingId: sessionId, ref: chainEnd, ...(state.cwd ? { dir: state.cwd } : {}) } } : {}), @@ -1877,6 +1925,38 @@ export function translateSDKMessage( emit({ type: "tool_progress", toolName: p.tool_name, elapsedSeconds: p.elapsed_time_seconds }); break; } + + case "rate_limit_event": { + // SDKRateLimitInfo.resetsAt is epoch SECONDS (the CLI renders it with + // `new Date(resetsAt * 1000)`); ProviderEvent carries milliseconds. + const info = (msg as { + rate_limit_info?: { + status?: string; + resetsAt?: number; + rateLimitType?: string; + isUsingOverage?: boolean; + overageInUse?: boolean; + overageStatus?: string; + }; + }).rate_limit_info; + const status = info?.status; + if (status !== "allowed" && status !== "allowed_warning" && status !== "rejected") break; + // The plan window is spent but extra usage (overage) is carrying the + // requests: not a limit that stops the turn. + const overage = + info?.isUsingOverage === true || + info?.overageInUse === true || + info?.overageStatus === "allowed" || + info?.overageStatus === "allowed_warning"; + emit({ + type: "rate_limit", + status, + ...(overage ? { overage: true } : {}), + ...(typeof info?.resetsAt === "number" && info.resetsAt > 0 ? { resetsAt: info.resetsAt * 1000 } : {}), + ...(info?.rateLimitType ? { limitType: info.rateLimitType } : {}), + }); + break; + } } } diff --git a/src/daemon/providers/codex/index.ts b/src/daemon/providers/codex/index.ts index 576040ac..7c350fcd 100644 --- a/src/daemon/providers/codex/index.ts +++ b/src/daemon/providers/codex/index.ts @@ -65,7 +65,7 @@ import type { } from "../interface.js"; import { renderHistorySeed, type CanonicalTurn, type HistorySeedResult } from "../canonical.js"; import { type ResponsesItem, toResponsesItems } from "./native-history.js"; -import { buildCodexEnv } from "../env.js"; +import { buildCodexEnv, withExtraEnv } from "../env.js"; import { CodexRpcProcess } from "./rpc.js"; import { EFFORT_LEVELS, type EffortLevel, type SessionMode } from "../../../protocol/types.js"; import { MEMORY_MCP_SERVER_NAME, MEMORY_MCP_TOKEN_ENV, type MemoryMcpMount } from "../../memory/mcp-http.js"; @@ -130,6 +130,8 @@ export interface CodexProviderInit { */ resolveAccount?: () => ResolvedAccount | null; onModels?: (models: ReadonlyArray) => void; + /** Caller env for the child process (ProviderSessionInit.extraEnv), merged last via withExtraEnv. */ + extraEnv?: Readonly>; } /** Item types that surface as tool records (vs text/reasoning streams). */ @@ -265,12 +267,52 @@ export function describeCodexError(params: Record): string { } } +/** + * The structured half of a codex `error` notification, for the usage-limit + * classifier: the v2 `TurnError.codexErrorInfo` variant (`usageLimitExceeded`, + * `rateLimitExceeded`, `httpConnectionFailed { httpStatusCode }`, … — codex-rs + * 0.154.0 app-server protocol) and the limit's reset instant when it carries + * one (`resets_at` / `resetsAt`, epoch seconds or ms). The info is a bare + * string for a unit variant and `{ variant: { …fields } }` otherwise. + * Undefined when the notification has no error info. describeCodexError keeps + * producing the display text; this never reaches the user. + */ +export function codexErrorDetail(params: Record): Record | undefined { + const err = params.error && typeof params.error === "object" ? (params.error as Record) : undefined; + const info = err?.codexErrorInfo ?? params.codexErrorInfo; + let variant: string | undefined; + let fields: Record = {}; + if (typeof info === "string") variant = info; + else if (info && typeof info === "object") { + const [name, value] = Object.entries(info as Record)[0] ?? []; + variant = name; + if (value && typeof value === "object") fields = value as Record; + } + if (!variant) return undefined; + const data = err?.data && typeof err.data === "object" ? (err.data as Record) : {}; + const reset = [fields.resetsAt, fields.resets_at, err?.resetsAt, err?.resets_at, data.resetsAt, data.resets_at].find( + (v): v is number => typeof v === "number" && v > 0, + ); + const status = fields.httpStatusCode; + return { + codexErrorInfo: variant, + ...(reset !== undefined ? { resetsAt: reset } : {}), + ...(typeof status === "number" ? { status } : {}), + }; +} + export class CodexProvider implements SessionProvider { /** * codex's approval replies carry only a decision (`approved` / `accept`) — * codex runs the input IT proposed, so a rewrite can't apply (#367). */ readonly appliesToolInputRewrite = false; + /** + * Autonomous mode is `approvalPolicy: "never"` (approvalPolicyForMode): codex + * runs commands and patches without an approval request, so codeoid's tool + * budget never sees them. + */ + readonly enforcesToolBudget = false; readonly id = "codex"; readonly displayName = "Codex (OpenAI)"; @@ -282,6 +324,7 @@ export class CodexProvider implements SessionProvider { #command: string; #argsPrefix: string[]; #resolveAccount: (() => ResolvedAccount | null) | null; + readonly #extraEnv: Readonly> | undefined; #onModels?: CodexProviderInit["onModels"]; #workspaceId: string; #memoryMcp: MemoryMcpMount | null; @@ -374,6 +417,7 @@ export class CodexProvider implements SessionProvider { this.#backingSessionId = init.initialBackingId; this.#command = init.command; this.#resolveAccount = init.resolveAccount ?? null; + this.#extraEnv = init.extraEnv; this.#argsPrefix = init.argsPrefix ?? []; this.#onModels = init.onModels; this.#workspaceId = init.workspaceId ?? init.sessionId; @@ -705,6 +749,11 @@ export class CodexProvider implements SessionProvider { await this.teardown(); } + async backendProcessIds(): Promise { + const pid = this.#proc?.pid; + return pid ? [pid] : []; + } + async teardown(): Promise { this.#pendingMcpCalls.clear(); this.#turnQueue?.close(); @@ -821,10 +870,15 @@ export class CodexProvider implements SessionProvider { cwd: opts.workdir, // The account's own CODEX_HOME (an overlay sharing everything but its // auth.json), applied last so nothing re-adds a daemon key. - env: await applyCodexAccount( - this.#resolveAccount?.() ?? null, - { ...buildCodexEnv(), ...mcp.env, ...bb.env, ...reg.env, ...oauth.env }, - { command: this.#command, argsPrefix: this.#argsPrefix }, + // The caller's extra env goes on last (withExtraEnv refuses anything + // credential- or home-shaped, so the account stays the account). + env: withExtraEnv( + await applyCodexAccount( + this.#resolveAccount?.() ?? null, + { ...buildCodexEnv(), ...mcp.env, ...bb.env, ...reg.env, ...oauth.env }, + { command: this.#command, argsPrefix: this.#argsPrefix }, + ), + this.#extraEnv, ), onNotification: (method, params) => this.#onNotification(method, params), onServerRequest: (method, params) => this.#onServerRequest(method, params), @@ -1167,7 +1221,8 @@ export class CodexProvider implements SessionProvider { // the literal "[object Object]" — the cause destroyed at the one place // it was supposed to be reported. Observed live: a role-child failed // twice with `Error: [object Object]` and nothing anywhere said why. - this.#push({ type: "error", message: `codex: ${describeCodexError(params)}` }); + const detail = codexErrorDetail(params); + this.#push({ type: "error", message: `codex: ${describeCodexError(params)}`, ...(detail ? { detail } : {}) }); break; } default: diff --git a/src/daemon/providers/env.ts b/src/daemon/providers/env.ts index b0d44a23..7ddd13fa 100644 --- a/src/daemon/providers/env.ts +++ b/src/daemon/providers/env.ts @@ -80,6 +80,107 @@ export function buildSubprocessEnv( return out; } +// ── Caller-supplied extra env (SessionCreateOptions.extraEnv) ──────────────── + +/** + * What a caller's extra env may NEVER set on a backend's child process: the + * variables codeoid owns for credential selection and account isolation, and + * the ones that change what the backend process loads or where its requests + * go. A loop controller passes extra env for the work (a test DB URL, a build + * flag); none of it may swap the account a session runs on, re-point a vendor + * home at another login, inject code into the backend (NODE_OPTIONS, + * LD_PRELOAD), or route its bearer token through a proxy. + * + * Matched by exact name, namespace prefix, or credential suffix, on + * upper-case names (anything else is refused outright). Deliberately broad: a + * controller that needs one of these has to get it through the account system + * or a trusted setup recipe, not an env side door. + */ +const EXTRA_ENV_DENY_EXACT = new Set([ + // Identity and the vendor homes every account overlay lives under. + "HOME", "USER", "LOGNAME", "USERPROFILE", + "XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_STATE_HOME", "XDG_CACHE_HOME", "XDG_RUNTIME_DIR", + // What runs: the binary lookup, the shell tools run in, Node preloads. + "PATH", "SHELL", "NODE_OPTIONS", "NODE_PATH", "BUN_OPTIONS", + // Where requests go, and which TLS roots let something read them in flight + // (the REDIRECTING_ENV sets in claude-accounts.ts / account-apply.ts). + "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", + "NODE_EXTRA_CA_CERTS", "NODE_TLS_REJECT_UNAUTHORIZED", "SSL_CERT_FILE", "SSL_CERT_DIR", + "CODE_ASSIST_ENDPOINT", + // Variables a tool the backend runs reads as configuration or as a command + // to execute: shell startup files, pagers and editors, ripgrep's config, + // glibc's charset-module path. + "RIPGREP_CONFIG_PATH", "BASH_ENV", "ENV", "PROMPT_COMMAND", + "PAGER", "GIT_PAGER", "LESSOPEN", "LESSCLOSE", "EDITOR", "VISUAL", "GCONV_PATH", + // Backend runtime knobs: logging that can echo requests, and where temp + // files (prompts, patches) land. + "RUST_LOG", "RUST_BACKTRACE", "TMPDIR", "TEMP", "TMP", "TZ", + // Build tools that run a wrapper or take flags (a rustc wrapper is a + // program), CA bundles that let a proxy read requests, the user's SSH + // agent, and bash options / xtrace prompt (PS4 is expanded, so it runs code). + "RUSTC_WRAPPER", "RUSTFLAGS", "GOFLAGS", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE", "SSH_AUTH_SOCK", + "SHELLOPTS", "BASHOPTS", "PS4", +]); +const EXTRA_ENV_DENY_PREFIXES = [ + // codeoid's own namespaces (DENY_PREFIXES above) — tokens, the escape hatch. + "CODEOID_", "ZEROID_", "TELEGRAM_", + // Every backend's credential / home / endpoint namespace the account + // appliers set or scrub (claude-accounts.ts, account-apply.ts). + "ANTHROPIC_", "CLAUDE_", "CODEX_", "OPENAI_", "GEMINI_", "GOOGLE_", "QWEN_", "DASHSCOPE_", "BAILIAN_", "PI_", + // Cloud credentials (Bedrock, Vertex, Azure) and where telemetry goes. + "AWS_", "AZURE_", "VERTEX_", "OTEL_", + // Dynamic-loader injection. + "LD_", "DYLD_", + // Toolchains whose env is configuration that can run code or swap + // credentials: git (hooks via GIT_CONFIG_*, GIT_SSH_COMMAND), npm, Python, + // Ruby, Perl, the JVM, gcloud. Locale categories (LC_*) too. + "GIT_", "NPM_CONFIG_", "PYTHON", "RUBY", "PERL5", "PERL_", "JAVA_", "_JAVA_", "JDK_", "CLOUDSDK_", "LC_", "CARGO_", +] as const; +const EXTRA_ENV_DENY_SUFFIXES = ["_API_KEY", "_API_TOKEN", "_AUTH_TOKEN", "_ACCESS_TOKEN", "_REFRESH_TOKEN", "_OAUTH_TOKEN"] as const; +/** + * Upper-case names only: lower-case twins (`http_proxy`, `npm_config_*`) are + * read by tools as the same setting, and refusing the shape is simpler and + * safer than enumerating every twin. + */ +const ENV_NAME = /^[A-Z][A-Z0-9_]{0,63}$/; + +function extraEnvRefused(name: string, value: unknown): boolean { + if (!ENV_NAME.test(name) || typeof value !== "string" || value.includes("\0")) return true; + return ( + EXTRA_ENV_DENY_EXACT.has(name) || + EXTRA_ENV_DENY_PREFIXES.some((p) => name.startsWith(p)) || + EXTRA_ENV_DENY_SUFFIXES.some((s) => name.endsWith(s)) + ); +} + +/** + * The keys of `extra` codeoid refuses to pass to a backend (denylisted, not a + * valid variable name, or a non-string / NUL-bearing value). Empty = all allowed. + */ +export function extraEnvViolations(extra: Readonly>): string[] { + return Object.entries(extra) + .filter(([k, v]) => extraEnvRefused(k, v)) + .map(([k]) => k); +} + +/** + * `env` (what codeoid built for the backend, account applied) with the + * caller's extra env merged over it — minus every refused key, which keeps + * codeoid's value. Pure. Session refuses such keys up front; filtering here + * again means a provider built by some other path can't be talked into it. + */ +export function withExtraEnv( + env: Record, + extra: Readonly> | undefined, +): Record { + if (!extra) return env; + const out: Record = { ...env }; + for (const [k, v] of Object.entries(extra)) { + if (!extraEnvRefused(k, v)) out[k] = v; + } + return out; +} + /** * The header Highflame's AI gateway (firehog) authenticates a sandboxed agent * on. Forge injects it at launch carrying the SANDBOX BADGE; see diff --git a/src/daemon/providers/interface.ts b/src/daemon/providers/interface.ts index 36ce95e1..027ae450 100644 --- a/src/daemon/providers/interface.ts +++ b/src/daemon/providers/interface.ts @@ -203,6 +203,13 @@ export interface NormalizedTurnResult { stopReason?: string; isError?: boolean; errorMessage?: string; + /** + * The backend's own result text for a turn it flagged as an error without + * an error subtype (Claude: `subtype: "success"`, `is_error: true`) — where + * the CLI puts a usage-limit or "API Error: …" notice. Text the CLI wrote, + * possibly the model's: read only with CLI-shaped rules (classifyCliNotice). + */ + resultText?: string; /** * Where this turn ends in the backend's OWN record of the conversation, for * a backend that can later fork or resume from that point natively (#387). @@ -336,6 +343,20 @@ export type ProviderEvent = /** Per-LLM-call usage, split by primary vs subagent (null parent_tool_use_id = primary). */ | { type: "llm_call"; usage: LLMCallUsage; isPrimary: boolean } | { type: "api_retry"; attempt?: number; retryDelayMs?: number; errorStatus?: number | null } + /** + * The backend's plan rate-limit state changed (Claude's `rate_limit_event`, + * subscription accounts only). `rejected` means requests are refused until + * `resetsAt` (epoch ms) — the exact reset a usage-limit error's text often + * gives only as a wall-clock time. Observational: nothing retries on it. + */ + | { + type: "rate_limit"; + status: "allowed" | "allowed_warning" | "rejected"; + resetsAt?: number; + limitType?: string; + /** Extra usage (overage) is carrying requests past the spent window — nothing is refused. */ + overage?: boolean; + } | { type: "tool_progress"; toolName?: string; elapsedSeconds?: number } /** * Provider-authored standalone message — extension output, status cards, @@ -361,7 +382,12 @@ export type ProviderEvent = * denied. Distinct from a tool approval, whose SDK turn is still live. */ | { type: "approval_pending"; command: string } - | { type: "error"; message: string }; + /** + * The turn failed. `detail` is the backend's STRUCTURED error, when it gave + * one (codex's `codexErrorInfo` + reset instant — codexErrorDetail): read by + * the usage-limit classifier, never shown. `message` stays the display text. + */ + | { type: "error"; message: string; detail?: Record }; /** * True when a text/thinking ProviderEvent was produced by a subagent @@ -656,6 +682,22 @@ export interface AgentProvider { * applies the new input to what it actually executes. */ readonly appliesToolInputRewrite?: boolean; + /** + * False when, in autonomous mode, the backend runs tool calls WITHOUT + * asking canUseTool — so codeoid's autonomous tool budget (and + * `onBudgetExhausted`) cannot see, count or stop them. Codex: autonomous + * maps to `approvalPolicy: "never"`. Absent = true (every call is gated). + * A loop controller falls back to wall-time limits (supervisor.ts) there. + */ + readonly enforcesToolBudget?: boolean; + /** + * Pids of the processes this backend spawned itself (its CLI / app-server) + * that are alive now. Session.terminateBackend snapshots their whole tree + * before the teardown and kills what outlives it (process-tree.ts). Absent + * or empty = unknown (in-process backends; qwen-code, whose SDK spawns + * internally): only the teardown runs. + */ + backendProcessIds?(): Promise; dispose(): Promise; } diff --git a/src/daemon/providers/jsonrpc-stdio.ts b/src/daemon/providers/jsonrpc-stdio.ts index 7162e0d5..94613453 100644 --- a/src/daemon/providers/jsonrpc-stdio.ts +++ b/src/daemon/providers/jsonrpc-stdio.ts @@ -84,6 +84,11 @@ export class StdioJsonRpcProcess { return !this.#exited; } + /** The child's pid while it runs (process-tree kill on terminateBackend). */ + get pid(): number | undefined { + return this.#exited ? undefined : this.#proc.pid; + } + /** Client→server request; resolves with the JSON-RPC result. */ request( method: string, diff --git a/src/daemon/providers/pi/index.ts b/src/daemon/providers/pi/index.ts index f3b34de6..e64dc8a0 100644 --- a/src/daemon/providers/pi/index.ts +++ b/src/daemon/providers/pi/index.ts @@ -36,7 +36,7 @@ import type { } from "../interface.js"; import { EFFORT_LEVELS, type EffortLevel, type ProviderCommand } from "../../../protocol/types.js"; import { renderHistorySeed, type CanonicalTurn, type HistorySeedResult } from "../canonical.js"; -import { buildPiEnv } from "../env.js"; +import { buildPiEnv, withExtraEnv } from "../env.js"; import { APPROVAL_TITLE, type BridgeMcpTool, @@ -83,6 +83,8 @@ export interface PiProviderInit { */ resolveAccount?: () => ResolvedAccount | null; onModels?: (models: ReadonlyArray) => void; + /** Caller env for the child process (ProviderSessionInit.extraEnv), merged last via withExtraEnv. */ + extraEnv?: Readonly>; } interface PiSessionStats { @@ -110,6 +112,7 @@ export class PiProvider implements SessionProvider { #backingSessionId: string; #command: string; #resolveAccount: (() => ResolvedAccount | null) | null; + readonly #extraEnv: Readonly> | undefined; #argsPrefix: string[]; #store: Store; #onModels?: PiProviderInit["onModels"]; @@ -144,6 +147,7 @@ export class PiProvider implements SessionProvider { this.#backingSessionId = init.initialBackingId; this.#command = init.command; this.#resolveAccount = init.resolveAccount ?? null; + this.#extraEnv = init.extraEnv; this.#argsPrefix = init.argsPrefix ?? []; this.#store = init.store; this.#onModels = init.onModels; @@ -194,6 +198,11 @@ export class PiProvider implements SessionProvider { this.#hasQueried = value; } + async backendProcessIds(): Promise { + const pid = this.#proc?.pid; + return pid ? [pid] : []; + } + async teardown(): Promise { this.#failTurn("pi session torn down"); this.#proc?.kill(); @@ -330,7 +339,8 @@ export class PiProvider implements SessionProvider { // Allowlisted env (GHSA-38vh): pi gets its credentials (~/.pi via // HOME, provider API keys), never codeoid's own secrets. // pi's thinking-level writes stay in a copy of its settings (#417). - env: this.#piEnv(), + // Then the caller's extra env, which can't override the account (withExtraEnv). + env: withExtraEnv(this.#piEnv(), this.#extraEnv), onEvent: (frame) => this.#onFrame(frame), onExit: ({ code, signal, stderrTail }) => { this.#bridgeReady = false; diff --git a/src/daemon/providers/pi/rpc.ts b/src/daemon/providers/pi/rpc.ts index fdad2469..c368ad52 100644 --- a/src/daemon/providers/pi/rpc.ts +++ b/src/daemon/providers/pi/rpc.ts @@ -100,6 +100,11 @@ export class PiRpcProcess { return !this.#exited; } + /** The child's pid while it runs (process-tree kill on terminateBackend). */ + get pid(): number | undefined { + return this.#exited ? undefined : this.#proc.pid; + } + /** * Strict-LF line assembly. pi warns that Unicode-separator-splitting * readers corrupt frames — we split on `\n` only and strip a trailing diff --git a/src/daemon/providers/qwen/index.ts b/src/daemon/providers/qwen/index.ts index 2c1ecc60..11a97f62 100644 --- a/src/daemon/providers/qwen/index.ts +++ b/src/daemon/providers/qwen/index.ts @@ -54,7 +54,7 @@ import type { CatalogEntry, } from "../interface.js"; import { renderHistorySeed, type CanonicalTurn, type HistorySeedResult } from "../canonical.js"; -import { buildQwenEnv } from "../env.js"; +import { buildQwenEnv, withExtraEnv } from "../env.js"; import type { LLMCallUsage } from "../../context-math.js"; import type { ResolvedAccount } from "../../accounts.js"; import { qwenAccountEnv } from "../../account-apply.js"; @@ -97,6 +97,8 @@ export interface QwenProviderInit { */ resolveAccount?: () => ResolvedAccount | null; onModels?: (models: ReadonlyArray) => void; + /** Caller env for the child process (ProviderSessionInit.extraEnv), merged last via withExtraEnv. */ + extraEnv?: Readonly>; } export class QwenProvider implements SessionProvider { @@ -440,14 +442,20 @@ export class QwenProvider implements SessionProvider { abortController: this.#abortController, // Explicit allowlist + blanking of everything else — the SDK MERGES // this over process.env rather than replacing it (see buildQwenEnv). - env: { - ...buildQwenEnv(), - // An account's key, and none of the daemon's (blanked: the SDK merges over process.env). - // An account: its key and its own home, and none of the daemon's keys. - ...(account ? qwenAccountEnv(account, buildQwenEnv()) : {}), - ...(baseUrl ? { OPENAI_BASE_URL: baseUrl } : {}), - ...(model ? { OPENAI_MODEL: model } : {}), - }, + // The caller's extra env goes on last; withExtraEnv refuses every + // OPENAI_/QWEN_/DASHSCOPE_ and credential-shaped key, so it can't + // undo the blanking or the account. + env: withExtraEnv( + { + ...buildQwenEnv(), + // An account's key, and none of the daemon's (blanked: the SDK merges over process.env). + // An account: its key and its own home, and none of the daemon's keys. + ...(account ? qwenAccountEnv(account, buildQwenEnv()) : {}), + ...(baseUrl ? { OPENAI_BASE_URL: baseUrl } : {}), + ...(model ? { OPENAI_MODEL: model } : {}), + }, + init.extraEnv, + ), authType, ...(model ? { model } : {}), // qwen-code takes a LIST of capacity fallbacks (max 3), where the diff --git a/src/daemon/providers/registry.ts b/src/daemon/providers/registry.ts index d7e48aa5..11c013b9 100644 --- a/src/daemon/providers/registry.ts +++ b/src/daemon/providers/registry.ts @@ -85,6 +85,15 @@ export interface ProviderSessionInit { * HTTP backends call it per turn; process backends when they spawn. */ resolveAccount?: () => ResolvedAccount | null; + /** + * Caller-supplied env for the backend's child process (SessionCreateOptions + * .extraEnv), merged last via `withExtraEnv` (env.ts), which never lets it + * override a credential / account-isolation variable. Honored by every + * process-backed provider — claude, codex, gemini-cli, pi, qwen. In-process + * backends (openai, gemini, llama.cpp, user endpoints) spawn nothing, so it + * has no effect there. + */ + extraEnv?: Readonly>; } /** An HTTP backend's key from the session's account (accounts.ts), or null for the daemon's own. */ @@ -373,6 +382,7 @@ export function createDefaultProviderRegistry( compressionRegistry: init.compressionRegistry, onModels: init.onModels, resolveAccount: init.resolveClaudeAccount, + extraEnv: init.extraEnv, }), }); // Stateless API backends: usable only with a key — in the environment (the @@ -436,6 +446,7 @@ export function createDefaultProviderRegistry( mcpHub: init.mcpHub, onModels: init.onModels, resolveAccount: init.resolveAccount, + extraEnv: init.extraEnv, }), }); } else { @@ -469,6 +480,7 @@ export function createDefaultProviderRegistry( mcpRegistry: init.mcpRegistry, onModels: init.onModels, resolveAccount: init.resolveAccount, + extraEnv: init.extraEnv, }), }); } else { @@ -502,6 +514,7 @@ export function createDefaultProviderRegistry( mcpRegistry: init.mcpRegistry, onModels: init.onModels, resolveAccount: init.resolveAccount, + extraEnv: init.extraEnv, }), }); } else { @@ -536,6 +549,7 @@ export function createDefaultProviderRegistry( config: init.config, onModels: init.onModels, resolveAccount: init.resolveAccount, + extraEnv: init.extraEnv, }), }); } diff --git a/src/daemon/providers/usage-limit.ts b/src/daemon/providers/usage-limit.ts new file mode 100644 index 00000000..bf66da61 --- /dev/null +++ b/src/daemon/providers/usage-limit.ts @@ -0,0 +1,490 @@ +/** + * Usage-limit and rate-limit detection for failed turns (loop design §12.11). + * + * A loop controller must tell "the provider said wait" apart from "the work + * failed": the first is an infrastructure error that consumes no attempt and + * parks the run until a reset time, the second is evidence about the attempt. + * This module is the one place that reads a backend's error text and decides. + * + * Two kinds, deliberately coarse: + * + * - `usage_limit` — a plan or quota allowance is spent (Claude's 5-hour / + * weekly windows, Codex's plan limit, Gemini's daily quota, a billing + * quota). The remedy is to wait for the window to reset or to move to + * another account; the reset is typically hours away, sometimes unknown. + * - `rate_limit` — transient throttling or overload (HTTP 429 / 529, + * RESOURCE_EXHAUSTED with a short retry delay). The remedy is a short wait. + * + * CONSERVATIVE by design: a false positive would park a run whose work + * actually failed, hiding a real failure behind a wait. Every rule below keys + * on a shape a real backend emits — the source is cited beside it — and + * ordinary errors (auth failures, context overflow, codeoid's own + * "Concurrent session limit reached", a model mentioning "rate limiter") + * classify as null. A generic phrase never matches on its own. + * + * `resetAt` is an epoch-milliseconds instant, present only when the error + * itself says when the limit lifts (an epoch, a duration, or a wall-clock + * time). It is sanity-bounded: a parsed time in the past or more than 35 days + * out is dropped rather than trusted. + */ + +export type UsageLimitKind = "usage_limit" | "rate_limit"; + +export interface UsageLimitClassification { + kind: UsageLimitKind; + /** Epoch ms at which the limit is expected to lift, when the error says. */ + resetAt?: number; + /** Backend the shape identifies (or the caller's `opts.provider`). */ + provider?: string; +} + +export interface ClassifyUsageLimitOptions { + /** Reference "now" for relative and wall-clock reset times (tests). Default Date.now(). */ + now?: number; + /** Backend id the error came from; wins over the one a shape implies. */ + provider?: string; +} + +/** + * The Claude Code CLI's "a usage limit was genuinely reached" messages — a + * copy of the Agent SDK's `USAGE_LIMIT_ERROR_PREFIXES` + * (node_modules/@anthropic-ai/claude-agent-sdk/sdk.d.ts, SDK 0.3.281, "the + * error-path outputs of getLimitReachedText (rateLimitMessages.ts)"). Copied + * rather than imported because the export is `@alpha` and a usage-limit + * check must not break on an SDK bump; usage-limit.test.ts fails if the two + * drift. The CLI composes them as `You've hit your ${limit} · resets ${time}` + * with limit ∈ {session limit, weekly limit, Opus limit, Sonnet limit, Fable + * limit} (the CLI bundle's `_h` / `Ide` table). Codex uses the same opener: + * "You've hit your usage limit. … Try again at 3:30 PM." (codex-rs 0.154.0 + * binary strings). + */ +export const CLAUDE_USAGE_LIMIT_PREFIXES: readonly string[] = [ + "You've hit your", + "You've reached your", + "You're out of usage credits", + "Your org is out of usage · add funds to continue", + "Your org is out of usage · contact your admin", + "Your seat type doesn't include usage credits", + "Your seat type doesn't include usage", + "Your usage allocation has been disabled by your admin", + "Your group's usage limit is set to $0", + "Fable 5 requires usage credits", + "You're out of extra usage", + "Your seat type doesn't include extra usage", +]; + +/** + * Openers too generic to trust alone ("You've hit your stride", "You've + * reached your goal: the rate limit test passes"): the limit noun must follow + * within four words, as in every completion the CLIs compose — "session + * limit", "monthly spend limit", "team's shared budget", "usage limit". + */ +const GENERIC_PREFIXES = new Set(["You've hit your", "You've reached your"]); +const GENERIC_TAIL = /^\s+(?:[\w'-]+\s+){0,3}(?:limit|budget)\b/i; + +/** + * gemini-cli treats a quota error whose retry delay is at most this as + * retryable, longer as terminal (`MAX_RETRYABLE_DELAY_SECONDS`, gemini-cli + * 0.50.0 bundle, packages/core googleQuotaErrors). The same line splits + * rate_limit from usage_limit here when only the delay tells them apart. + */ +const RETRYABLE_DELAY_MS = 300_000; + +/** Plausible reset horizon: Claude's longest window is 7 days; a month of slack. */ +const MAX_RESET_HORIZON_MS = 35 * 24 * 3600_000; + +interface Rule { + kind: UsageLimitKind; + test: (text: string) => boolean; + provider?: string; +} + +const re = (pattern: RegExp, kind: UsageLimitKind, provider?: string): Rule => ({ + kind, + test: (t) => pattern.test(t), + ...(provider ? { provider } : {}), +}); + +/** Usage-limit shapes, checked before the rate-limit ones (a 429 carrying a usage text is a usage limit). */ +const USAGE_RULES: readonly Rule[] = [ + // Claude Code before the "You've hit your" wording: `Claude AI usage limit + // reached|` as the result text of a limited turn. + re(/Claude AI usage limit reached/i, "usage_limit", "claude"), + // Older and toast forms in the Claude CLI bundle: "5-hour limit reached ∙ + // resets 3pm", "Usage limit reached …". Only named windows — never + // "Concurrent session limit reached" (codeoid) or "Context limit reached". + re(/(? 3) return; + if (typeof input === "string") { + if (input.length > 0) out.texts.push(input); + return; + } + if (typeof input !== "object") return; + const o = input as Record; + if (typeof o.message === "string" && o.message.length > 0) out.texts.push(o.message); + // A class name carries meaning on its own (gemini-cli's TerminalQuotaError). + if (typeof o.name === "string" && /QuotaError$/.test(o.name)) out.texts.push(o.name); + for (const key of ["status", "statusCode", "code"] as const) { + const v = o[key]; + if (out.status === undefined && (v === 429 || v === 529)) out.status = v; + } + // Machine codes: OpenAI's `code: "insufficient_quota"`, Anthropic's + // `error.type: "rate_limit_error"`, Google's `status: "RESOURCE_EXHAUSTED"`. + for (const key of ["code", "type", "status"] as const) { + const v = o[key]; + if (typeof v === "string" && /^[A-Za-z][\w.]{2,63}$/.test(v)) out.texts.push(v); + } + if (out.retryDelayMs === undefined && typeof o.retryDelayMs === "number" && o.retryDelayMs > 0) { + out.retryDelayMs = o.retryDelayMs; + } + // A structured reset: codex's `resets_at` / `resetsAt` (seconds or ms). + if (out.resetAtMs === undefined) { + const reset = [o.resetsAt, o.resets_at, o.resetAt].find((v): v is number => typeof v === "number" && v > 0); + if (reset !== undefined) out.resetAtMs = toEpochMs(reset); + } + // codex: { codexErrorInfo: "usageLimitExceeded" } or { codexErrorInfo: { usageLimitExceeded: {...} } }. + const info = o.codexErrorInfo; + if (typeof info === "string") out.texts.push(info); + else if (info && typeof info === "object") { + for (const [variant, fields] of Object.entries(info as Record)) { + out.texts.push(variant); + if (fields && typeof fields === "object") collect(fields, out, depth + 1); + } + } + if (o.error !== undefined && o.error !== input) collect(o.error, out, depth + 1); + if (o.cause !== undefined && o.cause !== input) collect(o.cause, out, depth + 1); +} + +/** Normalize separators and apostrophes the CLIs render with typographic glyphs. */ +function normalize(text: string): string { + return text.replace(/[‘’ʼ]/g, "'").replace(/[∙•]/g, "·"); +} + +function prefixRule(text: string): boolean { + for (const prefix of CLAUDE_USAGE_LIMIT_PREFIXES) { + // At the start, or right after a wrapper's colon/bracket/newline + // ("codex: You've hit…", "Error: You've hit…", "[API Error: You've…"). + const at = new RegExp(`(?:^|[:\\]\\n]\\s*)${escapeRegExp(prefix)}`).exec(text); + if (!at) continue; + if (!GENERIC_PREFIXES.has(prefix)) return true; + if (GENERIC_TAIL.test(text.slice(at.index + at[0].length))) return true; + } + return false; +} + +function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * Classify a failed turn's error — an Error, a provider error object, or the + * error text — as a usage limit, a rate limit, or neither (null). + */ +export function classifyUsageLimit( + input: unknown, + opts: ClassifyUsageLimitOptions = {}, +): UsageLimitClassification | null { + const signals: Signals = { texts: [] }; + collect(input, signals, 0); + const text = normalize(signals.texts.join("\n")); + const now = opts.now ?? Date.now(); + + let kind: UsageLimitKind | undefined; + let provider: string | undefined; + const usage = USAGE_RULES.find((r) => r.test(text)); + if (usage) { + kind = "usage_limit"; + provider = usage.provider; + } else if (prefixRule(text)) { + kind = "usage_limit"; + provider = /\bcodex\b/i.test(text) ? "codex" : undefined; + } else if (/You exceeded your current quota/i.test(text)) { + // OpenAI's billing quota and Gemini's free-tier per-minute quota share + // this sentence; Gemini's says "Please retry in 41s". A short delay is a + // throttle, anything else (or none) a spent allowance. + const delay = signals.retryDelayMs ?? relativeDelayMs(text); + kind = delay !== undefined && delay <= RETRYABLE_DELAY_MS && !/\bdaily\b|PerDay/i.test(text) ? "rate_limit" : "usage_limit"; + } else { + const rate = RATE_RULES.find((r) => r.test(text)); + if (rate) { + kind = "rate_limit"; + provider = rate.provider; + } else if (signals.status !== undefined) { + kind = "rate_limit"; + } + } + if (!kind) return null; + + // A "Quota exceeded for quota metric … per day" Google error is a daily allowance. + if (kind === "rate_limit" && /Quota exceeded for quota metric/i.test(text) && /per ?day|PerDay|daily/i.test(text)) { + kind = "usage_limit"; + } + + const resetAt = parseResetAt(text, now, signals); + const resolvedProvider = opts.provider ?? provider; + return { + kind, + ...(resetAt !== undefined ? { resetAt } : {}), + ...(resolvedProvider ? { provider: resolvedProvider } : {}), + }; +} + +/** + * Classify text the CLI may have written AS the turn's reply — Claude's result + * text on an errored turn, or the turn's last assistant message — with the + * CLI-shaped rules only. That text can just as well be the model's own words + * ("the API returned 429 Too Many Requests twice, so I added retries"), so the + * generic rate-limit / 429 / quota rules never apply. A notice counts only + * when the text — or its last block — OPENS with what the CLI composes: + * + * - one of the SDK's USAGE_LIMIT_ERROR_PREFIXES (the generic openers still + * need a limit word on their first line), or the legacy + * "Claude AI usage limit reached"; + * - "API Error:" — the CLI's prefix for an API failure it reports as the + * reply — after which the full classifier reads the rest. + */ +export function classifyCliNotice( + text: string | null | undefined, + opts: ClassifyUsageLimitOptions = {}, +): UsageLimitClassification | null { + if (!text) return null; + const whole = cliNotice(normalize(text).trimStart(), opts); + if (whole) return whole; + // A turn that wrote something before the limit hit ends with the notice as + // its own last block: "I'll start with the tests.\n\nYou've hit your…". + const blocks = normalize(text).split(/\n\s*\n/).map((b) => b.trim()).filter(Boolean); + return blocks.length > 1 ? cliNotice(blocks[blocks.length - 1]!, opts) : null; +} + +function cliNotice(t: string, opts: ClassifyUsageLimitOptions): UsageLimitClassification | null { + if (/^API Error:/.test(t)) return classifyUsageLimit(t, opts); + if (/^Claude AI usage limit reached/i.test(t) || startsWithLimitPrefix(t)) { + const now = opts.now ?? Date.now(); + const resetAt = parseResetAt(t, now, { texts: [t] }); + const provider = opts.provider ?? (t.startsWith("Claude AI") ? "claude" : undefined); + return { kind: "usage_limit", ...(resetAt !== undefined ? { resetAt } : {}), ...(provider ? { provider } : {}) }; + } + return null; +} + +function startsWithLimitPrefix(t: string): boolean { + for (const prefix of CLAUDE_USAGE_LIMIT_PREFIXES) { + if (!t.startsWith(prefix)) continue; + if (!GENERIC_PREFIXES.has(prefix)) return true; + if (GENERIC_TAIL.test(t.slice(prefix.length))) return true; + } + return false; +} + +// ── Reset time ──────────────────────────────────────────────────────────────── + +const UNIT_MS: Record = { + ms: 1, millisecond: 1, milliseconds: 1, + s: 1000, sec: 1000, secs: 1000, second: 1000, seconds: 1000, + m: 60_000, min: 60_000, mins: 60_000, minute: 60_000, minutes: 60_000, + h: 3600_000, hr: 3600_000, hrs: 3600_000, hour: 3600_000, hours: 3600_000, + d: 86_400_000, day: 86_400_000, days: 86_400_000, +}; +const DURATION_PART = /(\d+(?:\.\d+)?)\s*(milliseconds?|ms|seconds?|secs?|s|minutes?|mins?|m|hours?|hrs?|h|days?|d)\b/gi; + +/** A delay the text states relative to now: "try again in 1.5s", "resets in 2h 30m", "Retry-After: 30". */ +function relativeDelayMs(text: string): number | undefined { + const lead = /(?:resets?|try again|retry|available again)\s+(?:in|after)\s+((?:\d+(?:\.\d+)?\s*[a-z]+[\s,]*(?:and\s+)?){1,4})/i.exec(text); + if (lead?.[1]) { + let total = 0; + let matched = false; + for (const m of lead[1].matchAll(DURATION_PART)) { + const unit = UNIT_MS[m[2]!.toLowerCase()]; + if (unit === undefined) continue; + total += Number(m[1]) * unit; + matched = true; + } + if (matched && total > 0) return Math.round(total); + } + const retryAfter = /\bRetry-After:\s*(\d+)\b/i.exec(text); + if (retryAfter) return Number(retryAfter[1]) * 1000; + // Google RetryInfo: "retryDelay": "41s". + const retryInfo = /"retryDelay"\s*:\s*"(\d+(?:\.\d+)?)(ms|s)"/.exec(text); + if (retryInfo) return Number(retryInfo[1]) * (retryInfo[2] === "ms" ? 1 : 1000); + return undefined; +} + +const MONTHS: Record = { + jan: 0, feb: 1, mar: 2, apr: 3, may: 4, jun: 5, jul: 6, aug: 7, sep: 8, oct: 9, nov: 10, dec: 11, +}; + +/** + * A wall-clock reset: the Claude CLI's "resets 3pm (America/Los_Angeles)" / + * "resets Oct 28, 9am (UTC)" / "resets Oct 28, 2027, 9:30am" (its `Yc` + * formatter: en-US time, lower-cased am/pm, the zone in parentheses) and + * Codex's "Try again at 3:30 PM." / "try again at Oct 28th, 2026 3:30 PM." + * (codex-rs `%-I:%M %p` / `%b %-d, %Y %-I:%M %p`, local time), and + * gemini-cli's "Access resets at 3:30 PM PDT." (getResetTimeMessage, gemini-cli + * 0.50.0 interactiveCli bundle: en-US `hour: numeric, minute: 2-digit, + * timeZoneName: short` in the host zone). A short zone name ("PDT") is not a + * zone id and is ignored: these CLIs run on the daemon's host, so host time is + * the zone they printed in. Only a parenthesized IANA id (Claude's) is applied. + */ +const CLOCK = + /(?:resets?(?:\s+at)?|try again at)\s+(?:(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(?:(\d{4}),?\s+)?)?(\d{1,2})(?::(\d{2}))?\s*([ap])\.?\s?m\.?(?:\s*\(([^)]+)\))?/i; + +function clockResetAt(text: string, now: number): number | undefined { + const m = CLOCK.exec(text); + if (!m) return undefined; + const [, mon, day, year, hourText, minuteText, ampm, zone] = m; + let hour = Number(hourText); + if (hour < 1 || hour > 12) return undefined; + if (ampm!.toLowerCase() === "p" && hour !== 12) hour += 12; + if (ampm!.toLowerCase() === "a" && hour === 12) hour = 0; + const minute = minuteText ? Number(minuteText) : 0; + if (minute > 59) return undefined; + const tz = zone && isTimeZone(zone.trim()) ? zone.trim() : undefined; + const today = calendarDate(now, tz); + + if (mon) { + const month = MONTHS[mon.toLowerCase()]!; + const date = Number(day); + let y = year ? Number(year) : today.year; + let at = wallToEpoch(y, month, date, hour, minute, tz); + // "Oct 28" with no year, seen in early January, is next October's. + if (!year && at < now - 86_400_000) at = wallToEpoch(++y, month, date, hour, minute, tz); + return at; + } + let at = wallToEpoch(today.year, today.month, today.day, hour, minute, tz); + // A bare time already passed today means tomorrow (the CLI prints a date only beyond 24h). + if (at <= now - 60_000) at = wallToEpoch(today.year, today.month, today.day + 1, hour, minute, tz); + return at; +} + +function isTimeZone(zone: string): boolean { + try { + new Intl.DateTimeFormat("en-US", { timeZone: zone }); + return true; + } catch { + return false; + } +} + +/** Calendar parts of `instant` in `tz` (the host zone when undefined). */ +function calendarDate(instant: number, tz: string | undefined): { year: number; month: number; day: number } { + if (!tz) { + const d = new Date(instant); + return { year: d.getFullYear(), month: d.getMonth(), day: d.getDate() }; + } + const p = zonedParts(instant, tz); + return { year: p.year, month: p.month, day: p.day }; +} + +function zonedParts(instant: number, tz: string) { + const parts = new Intl.DateTimeFormat("en-US", { + timeZone: tz, + hourCycle: "h23", + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }).formatToParts(new Date(instant)); + const get = (type: string) => Number(parts.find((x) => x.type === type)?.value ?? 0); + return { year: get("year"), month: get("month") - 1, day: get("day"), hour: get("hour") % 24, minute: get("minute"), second: get("second") }; +} + +/** Epoch ms of a wall-clock time in `tz` (host zone when undefined). Day overflow normalizes. */ +function wallToEpoch(year: number, month: number, day: number, hour: number, minute: number, tz: string | undefined): number { + if (!tz) return new Date(year, month, day, hour, minute).getTime(); + const asUtc = Date.UTC(year, month, day, hour, minute); + const offset = (instant: number) => { + const p = zonedParts(instant, tz); + return Date.UTC(p.year, p.month, p.day, p.hour, p.minute, p.second) - Math.floor(instant / 1000) * 1000; + }; + // Two passes settle the offset across a DST boundary. + let at = asUtc - offset(asUtc); + const second = asUtc - offset(at); + if (second !== at) at = second; + return at; +} + +function parseResetAt(text: string, now: number, signals: Signals): number | undefined { + const { retryDelayMs } = signals; + // Data beats prose: an instant the error carried as a field is exact. + const candidates: Array = [signals.resetAtMs]; + // Claude's legacy `Claude AI usage limit reached|1730000000`. + const legacy = /usage limit reached\|(\d{9,13})/i.exec(text); + if (legacy) candidates.push(toEpochMs(Number(legacy[1]))); + // A structured reset in a JSON payload: codex `resets_at`, `resetsAt`. + const field = /"?(?:resets?_?[aA]t)"?\s*[:=]\s*(\d{9,13})\b/.exec(text); + if (field) candidates.push(toEpochMs(Number(field[1]))); + if (retryDelayMs !== undefined) candidates.push(now + retryDelayMs); + const delay = relativeDelayMs(text); + if (delay !== undefined) candidates.push(now + delay); + candidates.push(clockResetAt(text, now)); + for (const at of candidates) { + if (at !== undefined && Number.isFinite(at) && at > now - 60_000 && at <= now + MAX_RESET_HORIZON_MS) return at; + } + return undefined; +} + +/** Seconds or milliseconds since the epoch → milliseconds. */ +function toEpochMs(n: number): number { + return n < 1e12 ? n * 1000 : n; +} diff --git a/src/daemon/session.ts b/src/daemon/session.ts index 72f20cc9..c95f2e79 100644 --- a/src/daemon/session.ts +++ b/src/daemon/session.ts @@ -20,6 +20,9 @@ import { type HistorySeedResult, } from "./providers/canonical.js"; import { targetContextWindow, seedBudgetChars, isPromptTooLongError } from "./providers/context-windows.js"; +import { classifyCliNotice, classifyUsageLimit, type UsageLimitKind } from "./providers/usage-limit.js"; +import { extraEnvViolations } from "./providers/env.js"; +import { killSurvivors, snapshotTree, type ProcessEntry } from "./process-tree.js"; import { CONDUCTOR_SYSTEM_PROMPT_APPEND, isFleetSendTool } from "./fleet.js"; import type { McpSdkServerConfigWithInstance } from "@anthropic-ai/claude-agent-sdk"; import { @@ -39,6 +42,7 @@ import { type TurnOpts, type CatalogEntry, isPlaceholderModel, + BUDGET_STOP_MESSAGE, } from "./providers/interface.js"; import { createDefaultProviderRegistry, type ProviderRegistry } from "./providers/registry.js"; import { selectContextStrategy, renderSessionMap, renderRotationSeed, type ContextStrategy } from "./providers/context-strategy.js"; @@ -109,6 +113,9 @@ const FORK_SETUP_TIMEOUT_MS = 600_000; * the overflow is surfaced as an error. */ const SEED_OVERFLOW_MAX_RETRIES = 3; + +/** How long a hard stop (stall recovery, terminateBackend) waits for the backend's teardown. */ +const HARD_STOP_TEARDOWN_MS = 10_000; import { authToIdentity, CAPABILITIES, @@ -210,6 +217,19 @@ const CANCEL_REASON: Record block. The * digest is sub-agent output, which tool content can steer, so it must not be @@ -285,6 +305,23 @@ export interface AttachedClient { capabilities?: readonly string[]; } +/** + * A failed turn the backend blamed on a usage or rate limit (classifyUsageLimit) + * rather than on the work — the infrastructure error a loop controller records + * without spending an attempt (loop design §12.11). Epoch milliseconds. + */ +export interface TurnInfraError { + kind: UsageLimitKind; + /** When the limit is expected to lift, if the backend said. */ + resetAt?: number; + /** The backend's error text, as the turn reported it. */ + message: string; + /** The backend the turn ran on (the session's provider id at the time). */ + provider: string; + /** When the failure was recorded. */ + at: number; +} + /** Where a context window was observed; the manager scopes what it remembers * to it (see SessionCreateOptions.onModelLimits). */ export interface WindowScope { @@ -455,6 +492,42 @@ export interface SessionCreateOptions { * approval — which the dispatcher detects as a wedge. */ initialMode?: { mode: SessionMode; maxTurns?: number }; + /** + * What happens when the autonomous tool budget runs out (loop design §6). + * + * - `"park"` (default, today's behavior): the mode reverts to guarded and + * the next write/exec call waits for approval — the dispatcher sees the + * session wedge at waiting_approval. + * - `"end_attempt"`: for an unattended worker whose controller owns the + * next step. The call that finds the budget spent is DENIED with a clear + * message, the turn is interrupted, background tasks are stopped where the + * backend can, the session settles idle (never waiting_approval for that + * call), the mode reverts to guarded, and `budgetExhausted` reads true. + * Until a new autonomous budget is set (setMode), EVERY tool call is + * denied and ends its turn — including later prompts and turns the + * backend starts on its own. Only enforceable where `toolBudgetEnforced` + * (not codex); elsewhere a warning is logged and wall time must bound it. + */ + onBudgetExhausted?: "park" | "end_attempt"; + /** + * Extra environment for the backend's child process — every process-backed + * provider (Claude Code CLI, codex, gemini-cli, pi, qwen-code). Merged after + * codeoid builds the backend's env, but a key in codeoid's credential / + * account-isolation / code-loading denylist (`extraEnvViolations`, + * providers/env.ts), or any name that isn't `^[A-Z][A-Z0-9_]{0,63}$`, is + * refused: the constructor throws, so a misconfigured caller learns at once. + * + * DAEMON-CONTROLLED ONLY. The denylist is a backstop, not an authorization: + * a name a user (or a contract, or an agent) supplies must first pass an + * explicit allowlist the daemon owns — a later slice binds it to the trusted + * setup recipe's declared `resources.env`. Never forward client input here. + * No effect on in-process backends (openai, gemini, llama.cpp, endpoints). + * Held in memory only, by design — never persisted (it can carry a run's + * secrets-adjacent settings, and a stale copy must not outlive the run). A + * session restored after a daemon restart has none: the loop controller + * re-supplies it when it re-arms the worker during restart reconciliation. + */ + extraEnv?: Readonly>; /** * Native spend ceiling for this session's backend query (TurnOpts.maxBudgetUsd). * Routines set it on a fresh run session, whose query lifetime is the run. @@ -995,6 +1068,27 @@ export class Session { #collaborationCost?: () => CollaborationCost | undefined; #workerShape?: "ship" | "scout"; #turnsRemaining: number | undefined = undefined; + /** See SessionCreateOptions.onBudgetExhausted. */ + #onBudgetExhausted: "park" | "end_attempt" = "park"; + /** + * end_attempt only: the autonomous budget ran out. While set, EVERY tool + * call is denied — in the turn it ran out in, a later prompted turn, or a + * turn the backend starts on its own — until a new autonomous budget is set + * (setMode). Only an explicit re-arm clears it, never a turn boundary. + */ + #budgetExhausted = false; + /** This turn's end-attempt interrupt has been scheduled (once per turn). */ + #endAttemptInterruptQueued = false; + /** See SessionCreateOptions.extraEnv (validated in the constructor). */ + #extraEnv: Readonly> | undefined; + /** Why the last finished turn failed, when a usage or rate limit was the cause; null after a success. */ + #lastTurnInfraError: TurnInfraError | null = null; + /** + * Claude's `rate_limit_event` reported `rejected` during the current turn, + * with the reset instant it carried — exact where the error text says only + * "resets 3pm". Cleared when a turn starts. + */ + #rateLimitRejection: { resetAt?: number } | null = null; // Cumulative token + cost totals, aggregated from SDK `result` messages // (one per turn). Broadcast via session.info_update so StatusBar-style @@ -1219,6 +1313,18 @@ export class Session { this.#workerShape = opts.workerShape; this.#collaborationCost = opts.collaborationCost; this.#maxBudgetUsd = opts.maxBudgetUsd; + this.#onBudgetExhausted = opts.onBudgetExhausted ?? "park"; + if (opts.extraEnv) { + // Before any side effect: a refused key must fail the create, not run a + // backend with half the env the caller asked for. + const refused = extraEnvViolations(opts.extraEnv); + if (refused.length > 0) { + throw new Error( + `extraEnv may not set ${refused.join(", ")}: codeoid owns credential, account and process-loading variables for the backend`, + ); + } + this.#extraEnv = Object.freeze({ ...opts.extraEnv }); + } if (opts.initialMode) { this.#mode = opts.initialMode.mode; this.#turnsRemaining = @@ -1343,6 +1449,7 @@ export class Session { : (opts.config?.session.defaultEffort ?? null); this.#provider = opts._testProvider ?? this.#createProvider(opts.providerId); + this.#warnIfBudgetUnenforced(); this.#appliedAccount = this.#accountState(this.#provider.id); this.#provider.onSessionEvent = (e) => this.#onSessionScopedEvent(e); @@ -1493,6 +1600,7 @@ export class Session { resolveClaudeAccount: () => this.#claudeAccounts?.resolve(this.#accountChoices.claude ?? null) ?? null, // Every other backend: this session's account for it (accounts.ts). resolveAccount: () => this.#resolveAccount(factory.id), + ...(this.#extraEnv ? { extraEnv: this.#extraEnv } : {}), }); } @@ -1843,6 +1951,7 @@ export class Session { } this.#provider = this.#createProvider(requested); + this.#warnIfBudgetUnenforced(); this.#effortRestartPending = false; // the new backend starts with the session's effort this.#appliedAccount = this.#accountState(requested); this.#resetMcpState({ backendChanged: true }); @@ -1945,6 +2054,21 @@ export class Session { * the failure — a pipeline phase — needs this, not lastAssistantText. */ get lastTurnError(): string | null { return this.#lastTurnError; } + /** + * The most recent finished turn's failure, when the backend blamed a usage or + * rate limit (with its reset time when it gave one); null when that turn + * failed for another reason, succeeded, or a new turn has started. A loop + * controller reads it to record an infrastructure error that spends no + * attempt. Retry behavior is unchanged — this only observes. + */ + get lastTurnInfraError(): TurnInfraError | null { return this.#lastTurnInfraError; } + /** + * True once the autonomous tool budget ran out on a session created with + * `onBudgetExhausted: "end_attempt"` — the attempt was ended, not parked. + * Stays true, and every tool call is denied, until a new autonomous budget + * is set (setMode). + */ + get budgetExhausted(): boolean { return this.#budgetExhausted; } /** * Role of the MOST RECENT canonical turn (undefined before any turn). Right * after `send()` the last turn is our USER prompt/nudge; a transient query-loop @@ -2390,6 +2514,22 @@ export class Session { get agentUri(): string | undefined { return this.#agentIdentity.sub; } get mode(): SessionMode { return this.#mode; } get turnsRemaining(): number | undefined { return this.#turnsRemaining; } + /** + * Whether this session's backend asks codeoid before every tool call in + * autonomous mode, so the tool budget (and `onBudgetExhausted`) actually + * bounds it. False on codex (SessionProvider.enforcesToolBudget): there a + * controller must bound the attempt by wall time (supervisor.ts) instead. + * Read it again after a backend switch. + */ + get toolBudgetEnforced(): boolean { return this.#provider.enforcesToolBudget !== false; } + + /** end_attempt on a backend that can't count tool calls: not refused, but said. */ + #warnIfBudgetUnenforced(): void { + if (this.#onBudgetExhausted !== "end_attempt" || this.toolBudgetEnforced) return; + console.warn( + `[codeoid/session ${this.id}] onBudgetExhausted "end_attempt" requested, but the ${this.#provider.id} backend runs autonomous tool calls without asking codeoid — the tool budget can't be enforced; bound this attempt by wall time (supervise) instead`, + ); + } get pinnedFiles(): readonly string[] { return this.#pinnedFiles; } get model(): string | null { return this.#model; } get fallbackModel(): string | null { return this.#fallbackModel; } @@ -2432,9 +2572,17 @@ export class Session { /** Change execution mode. Resets turn budget if moving out of autonomous. */ setMode(mode: SessionMode, maxTurns?: number, sender?: AuthContext): void { + // A person choosing a mode is taking over: the ended attempt's lock is + // theirs to lift, whatever mode they pick (even the one already set). + if (this.#budgetExhausted && sender && isPersonSender(sender)) { + this.#budgetExhausted = false; + this.#broadcastInfoUpdate(); + } if (this.#mode === mode && this.#turnsRemaining === maxTurns) return; this.#mode = mode; this.#turnsRemaining = mode === "autonomous" ? maxTurns : undefined; + // A fresh autonomous budget is a new attempt's: the spent flag is the old one's. + if (mode === "autonomous") this.#budgetExhausted = false; if (sender) { this.#store.audit(sender.sub, "session.set_mode", this.id, `mode=${mode}`); } @@ -2993,6 +3141,79 @@ export class Session { return this.#preparingSends > 0; } + /** Per-turn loop controls, reset when a turn starts. */ + #resetTurnControls(): void { + this.#endAttemptInterruptQueued = false; + this.#rateLimitRejection = null; + // It describes the last FINISHED turn; a running one has not failed yet. + this.#lastTurnInfraError = null; + } + + /** + * A turn failed: record whether a usage or rate limit was the cause + * (lastTurnInfraError). Any failure overwrites the previous record, so it + * always describes the latest turn. Observation only: nothing here retries + * or changes status. + * + * - `errors`: what the backend reported AS the failure (error text, an + * Error, a structured error) — read with the full classifier. + * - `cliTexts`: text the turn's reply may hold a CLI notice in (Claude's + * result text on an is_error turn, the turn's own assistant text). It + * can be the model's words, so only CLI-shaped rules apply (classifyCliNotice). + * - `explained`: another recognizer already knows the cause (prompt too + * long, the spend ceiling) — then a rate_limit_event alone never makes it + * a usage limit. + */ + #recordTurnFailure( + errors: readonly unknown[], + opts: { cliTexts?: readonly (string | undefined)[]; explained?: boolean } = {}, + ): void { + const provider = this.#provider.id; + const record = (hit: { kind: UsageLimitKind; resetAt?: number }, source: unknown) => { + // The text often says only "resets 3pm"; Claude's rate_limit_event in + // the same turn carries the exact instant. + const resetAt = hit.resetAt ?? this.#rateLimitRejection?.resetAt; + this.#lastTurnInfraError = { + kind: hit.kind, + ...(resetAt !== undefined ? { resetAt } : {}), + message: failureText(source), + provider, + at: Date.now(), + }; + }; + for (const source of errors) { + if (source == null || source === "") continue; + const hit = classifyUsageLimit(source, { provider }); + if (hit) { + record(hit, source); + return; + } + } + for (const text of opts.cliTexts ?? []) { + const hit = classifyCliNotice(text, { provider }); + if (hit) { + record(hit, text); + return; + } + } + // Nothing said so, but the backend reported this turn's request refused + // by a plan limit (Claude's rate_limit_event "rejected", not carried by + // overage) — and no other recognizer explains the failure. + const rejection = this.#rateLimitRejection; + if (rejection && !opts.explained) { + const first = errors.find((e) => e != null && e !== ""); + this.#lastTurnInfraError = { + kind: "usage_limit", + ...(rejection.resetAt !== undefined ? { resetAt: rejection.resetAt } : {}), + message: first === undefined ? "usage limit reached" : failureText(first), + provider, + at: Date.now(), + }; + return; + } + this.#lastTurnInfraError = null; + } + /** * Surface a send-path failure as a visible system message instead of * swallowing it. Called by SessionManager#send when the async send() @@ -3002,6 +3223,7 @@ export class Session { */ reportSendFailure(err: unknown): void { const emsg = err instanceof Error ? err.message : String(err); + this.#recordTurnFailure([err]); console.error( `[codeoid/session ${this.id}] send failed after ack: ${ err instanceof Error ? (err.stack ?? emsg) : emsg @@ -3106,6 +3328,17 @@ export class Session { const joinsRunningTurn = wasWorking && !!this.#activeRun?.pushMidTurn && this.#currentTurnId !== null; const previousTurnId = this.#currentTurnId; let turnId = joinsRunningTurn ? (previousTurnId as string) : this.#beginTurn(); + // Per-turn controls belong to a turn that STARTS here, not to one this + // message only joins (its rate-limit record and interrupt are still live). + if (!joinsRunningTurn) { + this.#resetTurnControls(); + // A person prompting a spent end_attempt worker is taking over; a + // system principal (dispatcher, routines, the loop controller) is not. + if (this.#budgetExhausted && isPersonSender(sender)) { + this.#budgetExhausted = false; + this.#broadcastInfoUpdate(); + } + } const turnAt = new Date().toISOString(); try { const userMsg = this.#makeMessage( @@ -3572,6 +3805,7 @@ export class Session { void this.#checkpointTurn(turnId, 0); this.#turnInterrupted = false; this.#lastTurnError = null; + this.#resetTurnControls(); // A turn with no prompt reads as the agent talking to itself; say why. const note = this.#makeMessage( "info", @@ -3929,6 +4163,11 @@ export class Session { } catch { // fall through — the provider's own fallback is a hard abort } + // While we awaited, the turn may have ended and another started (a + // queued send, a backend-adopted turn, a terminateBackend). Its + // approvals and status are not this interrupt's to settle; an ended + // run's consumer already settled its own. + if (this.#activeRun !== run) return; // Tool cards are NOT closed here: the interrupted turn is still // draining, and a result that lands after interrupt() resolves must // still reach its card, the transcript and memory. The consumer's turn @@ -4831,6 +5070,7 @@ export class Session { ...this.#accountInfo(), mode: this.#mode, turnsRemaining: this.#turnsRemaining, + ...(this.#budgetExhausted ? { budgetExhausted: true as const } : {}), pinnedFiles: [...this.#pinnedFiles], agentUri: this.#agentIdentity.sub, subagents: this.subagentSnapshot, @@ -5727,6 +5967,20 @@ export class Session { return false; } + /** + * True when `toolName` would spend autonomous budget that is already spent, + * on a session that ends the attempt rather than parking + * (`onBudgetExhausted: "end_attempt"`). Side-effect free — shared by the + * gate and the tool_start peek so the card never offers an approval the + * gate will not wait for. The set of budgeted tools is #shouldAutoApprove's. + */ + #budgetSpentFor(toolName: string): boolean { + if (this.#onBudgetExhausted !== "end_attempt" || this.#mode !== "autonomous") return false; + if (this.#turnsRemaining === undefined || this.#turnsRemaining > 0) return false; + if (isFleetSendTool(toolName) || isElicitationTool(toolName)) return false; + return !isSafeTool(toolName) && !this.#isReadonlyMcpTool(toolName); + } + /** * Side-effect-free read of auto-approve state. Used ONLY in the tool_start * event handler to determine the initial UI phase (executing vs @@ -5853,6 +6107,77 @@ export class Session { } } + /** + * Deny a tool call because the autonomous budget is spent on an + * `end_attempt` session, and end the turn it belongs to (loop design §6: + * the attempt ends as budget_exhausted and control returns to the + * controller, rather than the session parking at waiting_approval). + * + * The interrupt is scheduled, not awaited: the deny must reach the backend + * first, and interrupt() awaits the backend, which is waiting on this gate. + * Only a turn's first denial interrupts it; later calls in the draining + * turn are simply refused. Background tasks are stopped too where the + * backend can (Claude: SDK `stopTask`); on a backend without + * stopBackgroundTasks they keep running, but every tool call they make is + * still denied here. + */ + #endAttemptOnBudget(approvalId: string, toolName: string, sender: AuthContext): { behavior: "deny"; message: string } { + if (!this.#budgetExhausted) { + // Same reversion as park (reads frictionless, writes ask again if a + // person takes over); flagged after setMode, which clears the flag only + // for a NEW autonomous budget. + this.setMode("guarded"); + this.#budgetExhausted = true; + this.#broadcastInfoUpdate(); // setMode announced before the flag was set + } + const reason = + "The autonomous tool budget for this attempt is spent, so the attempt has ended. Do not retry; the controller decides what happens next. " + + "(Operator: to take over, set a mode with /mode or send a message yourself; a controller re-arms it with a new autonomous budget.)"; + this.#resolveToolCallMessage(approvalId, { phase: "cancelled", reason: "denied", message: reason }); + this.#approvalPatchKeys.delete(approvalId); + this.#store.audit(sender.sub, "session.budget_deny", this.id, `tool=${toolName}`); + if (!this.#endAttemptInterruptQueued) { + this.#endAttemptInterruptQueued = true; + const infoMsg = this.#makeMessage( + "info", + `⏹ Autonomous tool budget spent — ${toolName} was refused and this attempt has ended.`, + SYSTEM_IDENTITY, + undefined, + undefined, + { event: "budget.end_attempt", tool: toolName }, + ); + this.#persistAndBuffer(infoMsg); + this.#broadcastRaw(infoMsg); + const system: AuthContext = { + sub: "system:budget", + scopes: [], + delegationDepth: 0, + accountId: this.accountId, + projectId: this.projectId, + }; + // Bound to the turn and background work of NOW: if the turn has moved + // on by the time this fires (a person took over and started another), + // it is not this denial's to stop. + const turnAtDenial = this.#currentTurnId; + const tasksAtDenial = [...this.#backgroundTasks.keys()]; + setTimeout(() => { + if (this.#destroyed || this.#currentTurnId !== turnAtDenial) return; + void (async () => { + await this.interrupt(system); + // interrupt() spares background agents while a turn runs; an ended + // attempt must not leave them working on the spent budget. Each + // stopped task settles itself out of the set; one whose stop failed + // stays listed — it is still running. + const live = tasksAtDenial.filter((id) => this.#backgroundTasks.has(id)); + if (live.length > 0 && this.#provider.stopBackgroundTasks) await this.#provider.stopBackgroundTasks(live); + })().catch((e) => { + console.error(`[codeoid/session ${this.id}] ending the attempt on a spent budget failed: ${e instanceof Error ? e.message : String(e)}`); + }); + }, 0); + } + return { behavior: "deny", message: reason }; + } + #makeCanUseToolFn(sender: AuthContext): ToolApprovalFn { // The backend this gate is handed to — the one that will ASK. Captured // here, not at call time, so the rewrite check below refers to it even if @@ -5869,6 +6194,10 @@ export class Session { // approvable card for a call the gate already resolved. if (!this.#turnActive()) await this.#backgroundEventChain; + // The attempt already ended on a spent budget (end_attempt): nothing + // runs — not even a read — until a new budget re-arms the session. + if (this.#budgetExhausted) return this.#endAttemptOnBudget(approvalId, toolName, sender); + // Hook gate — the policy layer, run BEFORE the approval gate. A hook // block is a policy deny that never prompts the user (and never burns // the autonomous budget — #shouldAutoApprove runs after this); an @@ -6009,6 +6338,9 @@ export class Session { // it mid-turn and applies its env, after the turn-start check // (assertClaudeAccountUsable) — so it never runs unasked, in any mode. const settingsWrite = this.#writesClaudeSettingsOnNamedAccount(toolName, effectiveInput); + // end_attempt: a call the spent budget would have parked is denied and + // ends the attempt instead — whatever else would have asked about it. + if (this.#budgetSpentFor(toolName)) return this.#endAttemptOnBudget(approvalId, toolName, sender); const autoApprove = secret?.action !== "ask" && !settingsWrite && this.#shouldAutoApprove(toolName); if (autoApprove) { @@ -6264,6 +6596,7 @@ export class Session { console.error(`[codeoid/session ${this.id}] provider event consumer failed:`, err); // Before the status change: it wakes a pipeline phase waiter that reads it. this.#lastTurnError = emsg; + this.#recordTurnFailure([err]); this.#setStatus("error"); const errorMsg = this.#makeMessage("system", `Error: ${emsg}`, SYSTEM_IDENTITY, undefined, undefined, { event: "agent_error", errorCode: "agent_error" }); this.#persistAndBuffer(errorMsg); @@ -6326,12 +6659,7 @@ export class Session { * run). Idempotent and run-scoped: if `run` is no longer the active run, the * turn already ended and we no-op. * - * Crucially this does NOT route through #teardownProvider (which awaits the - * session event-consumer task — i.e. potentially itself). It nulls the run - * slots up front, surfaces a clear message, resets status to idle, and hard - * tears down the PROVIDER (abort → reap the hung subprocess). The next send() - * recreates a fresh query loop. The abandoned consumer (if any) unblocks when - * teardown closes the turn queue and its finally no-ops via the run guard. + * The stop itself is #hardStop (shared with terminateBackend). */ async #recoverStalledRun( run: TurnRun, @@ -6344,11 +6672,89 @@ export class Session { `[codeoid/session ${this.id}] turn stalled — no provider events for ${stallMs}ms; force-recovering`, ); + await this.#hardStop("stall recovery", { + text: opts?.continuingCurrentSend + ? `⚠️ Previous turn timed out — no activity for ${Math.round(stallMs / 1000)}s. The session was reset and your latest message is being retried in a fresh turn.` + : `⚠️ Turn timed out — no activity for ${Math.round(stallMs / 1000)}s. The session was reset; send your message again to continue.`, + meta: { event: "turn_stalled", errorCode: "turn_stalled" }, + }); + } + + /** + * Stop this session's backend outright — the escalation after an interrupt + * that did not settle the session (supervisor.ts: an attempt's wall time + * ran out and the grace period passed). Whatever is in flight — a tool + * without its own timeout, a wedged turn, background agents — is ended by + * tearing the backend down (its child process is killed), exactly as stall + * recovery does — and then killing whatever that process had started and + * left running (process-tree.ts: the tree under the backend's own pids is + * snapshotted first, survivors are SIGKILLed after the bounded teardown). + * Self-detached commands and backends with unknown pids (qwen) are outside + * that reach; see process-tree.ts for exactly what survives. The session + * survives and settles idle; the next send starts a fresh backend and + * resumes the conversation. Idempotent. + */ + async terminateBackend( + sender: AuthContext, + reason: string, + opts: { teardownTimeoutMs?: number } = {}, + ): Promise { + this.#store.audit(sender.sub, "session.terminate_backend", this.id, reason); + // Stop sends still preparing their turn too, as interrupt() does. + this.#turnInterrupted = true; + this.#stopGen++; + this.#cancelAllUiRequests("interrupted"); + this.#earlyApprovals.clear(); + this.#approvalPatchKeys.clear(); + // Before the teardown: once the backend is gone its children re-parent + // and can no longer be told apart from anyone else's. + let tree: ProcessEntry[] = []; + try { + const roots = (await withTimeout(this.#provider.backendProcessIds?.() ?? Promise.resolve([]), 3_000)) ?? []; + tree = await withTimeout(snapshotTree(roots), 3_000); + } catch (e) { + console.error(`[codeoid/session ${this.id}] could not snapshot the backend's processes: ${e instanceof Error ? e.message : String(e)}`); + } + await this.#hardStop( + "backend terminated", + { + text: `⏹ Backend stopped by ${sender.sub}: ${reason}. Send a new message to continue.`, + meta: { event: "session.backend_terminated", sub: sender.sub }, + }, + opts.teardownTimeoutMs, + ); + // The teardown had its graceful chance (and the backend its own + // escalation); what is still running from the snapshot is left over. + try { + const killed = await killSurvivors(tree); + if (killed.length > 0) { + this.#store.audit(sender.sub, "session.terminate_backend.reaped", this.id, `pids=${killed.join(",")}`); + console.warn(`[codeoid/session ${this.id}] killed ${killed.length} process(es) the backend left running: ${killed.join(", ")}`); + } + } catch (e) { + console.error(`[codeoid/session ${this.id}] killing the backend's leftover processes failed: ${e instanceof Error ? e.message : String(e)}`); + } + } + + /** + * Shared by stall recovery and terminateBackend. Does NOT route through + * #teardownProvider (which awaits the session event-consumer task — i.e. + * potentially the caller). It nulls the run slots up front, surfaces a clear + * message, resets status to idle, and hard tears down the PROVIDER (abort → + * reap the subprocess). The next send() recreates a fresh query loop. The + * abandoned consumer (if any) unblocks when teardown closes the turn queue + * and its finally no-ops via the run guard. + */ + async #hardStop( + label: string, + notice: { text: string; meta: Record }, + teardownTimeoutMs = HARD_STOP_TEARDOWN_MS, + ): Promise { // Stop any spinning UI and release waiters BEFORE we tear down. The // teardown below kills the CLI and every background agent with it, so // nothing is kept: the whole background set is dead. this.#clearBackgroundTasks(); - this.#reconcileWork("stall recovery", { keepBackground: false }); + this.#reconcileWork(label, { keepBackground: false }); this.#flushActiveAssistant(); this.#finalizeActiveThinking(); this.#chunker?.onTurnEnd(); @@ -6359,16 +6765,7 @@ export class Session { this.#activeRun = null; this.#eventConsumerTask = null; - const msg = this.#makeMessage( - "system", - opts?.continuingCurrentSend - ? `⚠️ Previous turn timed out — no activity for ${Math.round(stallMs / 1000)}s. The session was reset and your latest message is being retried in a fresh turn.` - : `⚠️ Turn timed out — no activity for ${Math.round(stallMs / 1000)}s. The session was reset; send your message again to continue.`, - SYSTEM_IDENTITY, - undefined, - undefined, - { event: "turn_stalled", errorCode: "turn_stalled" }, - ); + const msg = this.#makeMessage("system", notice.text, SYSTEM_IDENTITY, undefined, undefined, notice.meta); this.#persistAndBuffer(msg); this.#broadcastRaw(msg); this.#setStatus("idle"); @@ -6376,11 +6773,14 @@ export class Session { // Hard teardown: abort the controller and reap the (presumed hung) // subprocess. Safe to await here — provider.teardown() awaits the // PROVIDER's own pump, never this session's consumer. + // Bounded: a backend that won't acknowledge its teardown must not hang + // the caller (a supervisor's `expired` waits on this). The run slots are + // already cleared, so the session is usable either way. try { - await this.#provider.teardown(); + await withTimeout(this.#provider.teardown(), teardownTimeoutMs); } catch (e) { console.error( - `[codeoid/session ${this.id}] provider teardown during stall recovery failed: ${e instanceof Error ? e.message : String(e)}`, + `[codeoid/session ${this.id}] provider teardown during ${label} failed: ${e instanceof Error ? e.message : String(e)}`, ); } } @@ -6515,6 +6915,8 @@ export class Session { const autoApprove = secretPathVerdict(event.name, event.input, { workdir: this.workdir, configDir: getConfigDir() }) === null && this.#peekAutoApprove(event.name); + // end_attempt: the gate will refuse this call outright — no approval card. + const budgetDenied = this.#budgetExhausted || this.#budgetSpentFor(event.name); const toolMsg = this.#makeMessage( "tool_call", `${event.name}(${Object.keys(event.input).join(", ")})`, @@ -6524,7 +6926,9 @@ export class Session { toolId: event.toolId, name: event.name, input: event.input, - state: autoApprove + state: budgetDenied + ? { phase: "cancelled", reason: "denied", message: "autonomous tool budget spent" } + : autoApprove ? ({ phase: "executing", input: event.input } as unknown as ToolState) : ({ phase: "waiting_confirmation", @@ -6548,7 +6952,7 @@ export class Session { this.#persistAndBuffer(toolMsg); this.#broadcastRaw(toolMsg); this.#maybeInjectRepeatReminder(event.name, event.input, event.sdkAgentId); - if (!autoApprove) this.#setStatus("waiting_approval"); + if (!autoApprove && !budgetDenied) this.#setStatus("waiting_approval"); break; } @@ -6737,6 +7141,15 @@ export class Session { break; } + case "rate_limit": { + // Kept for #recordTurnFailure: if this turn fails, the reset is exact. + // A later "allowed" in the same turn means the rejection was retried past. + // Overage carrying the requests means nothing was refused. + this.#rateLimitRejection = + event.status === "rejected" && !event.overage ? (event.resetsAt !== undefined ? { resetAt: event.resetsAt } : {}) : null; + break; + } + case "custom_message": { // Provider-authored standalone message (extension output, status // cards, rich widgets). Not part of the LLM conversation — the @@ -6790,6 +7203,19 @@ export class Session { break; } this.#lastTurnError = errText; + // Before the status change below, like lastTurnError: an error + // waiter reads it. The CLI puts a limit notice in the result text + // or the turn's own assistant text (only if THIS turn committed it, + // as above) — CLI-shaped rules only there, never the generic ones. + this.#recordTurnFailure([event.result.errorMessage], { + cliTexts: [ + event.result.resultText, + last?.role === "assistant" ? last.content : undefined, + // The reply's final text block on its own (the notice after narration). + last?.role === "assistant" ? (this.#accumulator.lastTextBlock ?? undefined) : undefined, + ], + explained: tooLong || event.result.errorMessage === BUDGET_STOP_MESSAGE, + }); const hint = tooLong ? " — this session's context no longer fits the model. Send /rotate to continue in a fresh context (memory recall keeps the history reachable)." : ""; @@ -6801,6 +7227,7 @@ export class Session { // The seed made it into a successful turn — it's spent. this.#pendingSeed = null; this.#promptTooLong = false; + this.#lastTurnInfraError = null; if (this.#status !== "error") this.#setStatus("idle"); } break; @@ -6833,6 +7260,8 @@ export class Session { case "error": { console.error(`[codeoid/session ${this.id}] provider error:`, event.message); this.#lastTurnError = event.message; // before the status change (see above) + // The structured detail (codex's error info + reset) rides with the text. + this.#recordTurnFailure([event.detail ? { message: event.message, error: event.detail } : event.message]); this.#setStatus("error"); const errorMsg = this.#makeMessage("system", `Error: ${event.message}`, SYSTEM_IDENTITY, undefined, undefined, { event: "agent_error" }); this.#persistAndBuffer(errorMsg); diff --git a/src/daemon/supervisor.ts b/src/daemon/supervisor.ts new file mode 100644 index 00000000..5456f31b --- /dev/null +++ b/src/daemon/supervisor.ts @@ -0,0 +1,161 @@ +/** + * Wall-time deadlines for a session's attempt (loop design §8, §12.11 "Hung work"). + * + * The stall watchdog pauses while a tool runs, and a dispatch lease keeps + * renewing while the session lives, so a tool without its own timeout can hold + * a task forever. A deadline is the bound that holds regardless: it counts + * wall time from `supervise()` and looks at nothing but the clock. + * + * On expiry it interrupts the session (the Esc path: stops the turn, reaps the + * running tool where the backend can). If the session has not settled within + * `graceMs` — the interrupt itself can wedge on a backend that stopped + * answering — it terminates the backend (Session.terminateBackend: the + * provider is torn down and its child process killed, as stall recovery + * does). `expired` then resolves true. + * + * The deadline owns exactly one timer at a time and clears it on every exit + * path; `cancel()` before expiry resolves `expired` false and leaves nothing + * scheduled. + */ + +import type { AuthContext, SessionStatus } from "../protocol/types.js"; + +/** The slice of Session a deadline drives (narrow, so tests can fake it). */ +export interface SupervisedSession { + readonly id: string; + readonly accountId: string; + readonly projectId: string; + readonly status: SessionStatus; + /** A queued send is about to start a turn — not settled even when idle. */ + readonly preparingTurn: boolean; + interrupt(sender: AuthContext): Promise; + terminateBackend(sender: AuthContext, reason: string, opts?: { teardownTimeoutMs?: number }): Promise; +} + +export interface DeadlineOptions { + /** Wall time the attempt may run, from now. */ + wallMs: number; + /** After the interrupt, how long the session gets to settle before its backend is killed. */ + graceMs: number; + /** Who the interrupt and termination are audited as. Default `system:supervisor` in the session's tenant. */ + sender?: AuthContext; + /** Bound on the backend's teardown when it is killed (Session default 10s), so `expired` cannot hang. */ + teardownTimeoutMs?: number; + /** How often the grace period re-checks for a settled session. Default 100ms (bounded by graceMs). */ + pollMs?: number; + /** Observes each step (logging, tests). Exceptions are swallowed. */ + onEvent?: (event: DeadlineEvent) => void; +} + +export type DeadlineEvent = + | { type: "expired"; sessionId: string } + | { type: "settled"; sessionId: string } + | { type: "terminated"; sessionId: string }; + +export interface Deadline { + /** + * Stop supervising. Before expiry: nothing fires and `expired` resolves + * false. During the grace period: the kill escalation is abandoned (the + * interrupt has already been sent) and `expired` resolves true. Idempotent. + */ + cancel(): void; + /** true once the wall time ran out and enforcement finished; false when cancelled first. */ + readonly expired: Promise; +} + +/** A session is settled when no turn is running or about to start. */ +function settled(session: SupervisedSession): boolean { + return (session.status === "idle" || session.status === "error") && !session.preparingTurn; +} + +export function supervise(session: SupervisedSession, opts: DeadlineOptions): Deadline { + const { wallMs, graceMs } = opts; + if (!Number.isFinite(wallMs) || wallMs <= 0) throw new RangeError(`supervise: wallMs must be a positive number (got ${wallMs})`); + if (!Number.isFinite(graceMs) || graceMs < 0) throw new RangeError(`supervise: graceMs must be zero or more (got ${graceMs})`); + const pollMs = Math.max(1, Math.min(opts.pollMs ?? 100, graceMs || 1)); + const sender: AuthContext = opts.sender ?? { + sub: "system:supervisor", + scopes: [], + delegationDepth: 0, + accountId: session.accountId, + projectId: session.projectId, + }; + const note = (event: DeadlineEvent) => { + try { + opts.onEvent?.(event); + } catch { + // observability must not break enforcement + } + }; + + let timer: ReturnType | null = null; + let done = false; + let fired = false; + let resolveExpired!: (v: boolean) => void; + const expired = new Promise((r) => { + resolveExpired = r; + }); + + const arm = (fn: () => void, ms: number) => { + timer = setTimeout(() => { + timer = null; + fn(); + }, ms); + // A deadline must not keep a process alive on its own. + timer.unref?.(); + }; + const finish = (result: boolean) => { + if (done) return; + done = true; + if (timer) clearTimeout(timer); + timer = null; + resolveExpired(result); + }; + + const terminate = () => { + if (done) return; + note({ type: "terminated", sessionId: session.id }); + session + .terminateBackend( + sender, + `attempt wall time of ${Math.round(wallMs / 1000)}s ran out and it did not stop within ${Math.round(graceMs / 1000)}s`, + opts.teardownTimeoutMs !== undefined ? { teardownTimeoutMs: opts.teardownTimeoutMs } : {}, + ) + .catch((e) => { + console.error(`[codeoid/supervisor] terminating session ${session.id} failed: ${e instanceof Error ? e.message : String(e)}`); + }) + .finally(() => finish(true)); + }; + + const graceUntil = (deadline: number) => { + if (done) return; + if (settled(session)) { + note({ type: "settled", sessionId: session.id }); + finish(true); + return; + } + const left = deadline - Date.now(); + if (left <= 0) { + terminate(); + return; + } + arm(() => graceUntil(deadline), Math.min(pollMs, left)); + }; + + arm(() => { + if (done) return; + fired = true; + note({ type: "expired", sessionId: session.id }); + // Not awaited: interrupt() awaits the backend, which may be the thing + // that's hung. The grace clock starts now, whatever it does. + session.interrupt(sender).catch((e) => { + console.error(`[codeoid/supervisor] interrupting session ${session.id} failed: ${e instanceof Error ? e.message : String(e)}`); + }); + graceUntil(Date.now() + graceMs); + }, wallMs); + + return { + cancel: () => finish(fired), + expired, + }; +} diff --git a/src/tests/fixtures/fake-codex.ts b/src/tests/fixtures/fake-codex.ts index 797013d8..acc7cd50 100644 --- a/src/tests/fixtures/fake-codex.ts +++ b/src/tests/fixtures/fake-codex.ts @@ -329,6 +329,9 @@ async function runTurn(threadId: string, prompt: string): Promise { argv: process.argv, }); send({ method: "item/completed", params: { item: { id: "m1", type: "agentMessage", text } } }); + } else if (prompt.includes("echo-env")) { + // The environment the provider spawned us with (extraEnv wiring). + send({ method: "item/completed", params: { item: { id: "m1", type: "agentMessage", text: JSON.stringify(process.env) } } }); } else if (prompt.includes("echo-native")) { send({ method: "item/completed", params: { item: { id: "m1", type: "agentMessage", text: JSON.stringify(nativeCalls) } } }); } else if (prompt.includes("echo-prompt")) { diff --git a/src/tests/process-tree.test.ts b/src/tests/process-tree.test.ts new file mode 100644 index 00000000..674fbc8b --- /dev/null +++ b/src/tests/process-tree.test.ts @@ -0,0 +1,188 @@ +/** + * Tree kill after a backend teardown (process-tree.ts, Session.terminateBackend). + * + * The integration case is the one measured against the Claude CLI and the one + * codex / gemini-cli hit on every teardown: the backend process is SIGKILLed, + * and the command its tool was running is left behind, re-parented. The + * snapshot taken before the teardown is what still identifies it. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { spawn, type ChildProcess } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { killSurvivors, listProcesses, parsePs, procStartTime, snapshotTree, treeOf } from "../daemon/process-tree.js"; +import { argvNamesSession } from "../daemon/providers/claude/index.js"; +import { MockSessionProvider } from "../daemon/providers/mock/session-provider.js"; +import { Session } from "../daemon/session.js"; +import { Store } from "../daemon/store.js"; +import { TranscriptStore } from "../daemon/transcript.js"; +import type { AuthContext } from "../protocol/types.js"; + +const AUTH: AuthContext = { sub: "user:tree", scopes: [], delegationDepth: 0, accountId: "acc", projectId: "proj" }; + +describe("parsePs / treeOf", () => { + const table = parsePs([" 10 1 /sbin/init", " 20 10 backend --serve", " 30 20 /bin/sh -c sleep 99", " 31 30 sleep 99", " 40 10 other", "garbage"].join("\n")); + + test("parses pid, ppid and the full command line", () => { + expect(table).toHaveLength(5); + expect(table[2]).toEqual({ pid: 30, ppid: 20, args: "/bin/sh -c sleep 99" }); + }); + + test("the tree under a root, root included, nothing else", () => { + expect(treeOf(table, [20]).map((p) => p.pid)).toEqual([20, 30, 31]); + expect(treeOf(table, [99])).toEqual([]); + }); + + test("macOS lstart columns: five fields kept as the start token", () => { + expect(parsePs(" 42 1 Sat Oct 24 10:08:43 2026 /bin/sh -c x", { lstart: true })).toEqual([ + { pid: 42, ppid: 1, start: "Sat Oct 24 10:08:43 2026", args: "/bin/sh -c x" }, + ]); + }); + + test.skipIf(process.platform !== "linux")("Linux start time from /proc//stat field 22", () => { + const t = procStartTime(process.pid); + expect(t).toMatch(/^\d+$/); + expect(procStartTime(process.pid)).toBe(t); + expect(procStartTime(2 ** 30)).toBeUndefined(); + }); + + test("never includes this process", () => { + const self = parsePs(`${process.pid} 1 bun test\n77 ${process.pid} child`); + expect(treeOf(self, [process.pid]).map((p) => p.pid)).toEqual([]); + }); +}); + +const alive = (pid: number): boolean => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +}; + +async function until(cond: () => boolean | Promise, ms = 5_000): Promise { + const deadline = Date.now() + ms; + while (!(await cond())) { + if (Date.now() > deadline) throw new Error("condition not reached"); + await new Promise((r) => setTimeout(r, 20)); + } +} + +/** A "backend": a shell running a long command in the background, like a tool call. */ +function spawnBackend(mark: string): Promise<{ proc: ChildProcess; grandchild: number }> { + const proc = spawn("sh", ["-c", `sleep ${mark} & echo $!; wait`], { stdio: ["ignore", "pipe", "ignore"] }); + return new Promise((resolve, reject) => { + proc.stdout!.once("data", (d: Buffer) => resolve({ proc, grandchild: Number(String(d).trim()) })); + proc.once("error", reject); + }); +} + +/** A backend whose teardown SIGKILLs its process, as codex / gemini-cli do. */ +class ProcessBackend extends MockSessionProvider { + proc: ChildProcess | null = null; + async backendProcessIds(): Promise { + return this.proc?.pid ? [this.proc.pid] : []; + } + override async teardown(): Promise { + this.teardownCount++; + this.proc?.kill("SIGKILL"); + } +} + +describe.skipIf(process.platform === "win32")("killing what a backend left running", () => { + let tmp: string; + let store: Store; + let transcriptStore: TranscriptStore; + const leftovers: number[] = []; + beforeEach(() => { + tmp = mkdtempSync(join(tmpdir(), "codeoid-tree-")); + store = new Store(join(tmp, "codeoid.db")); + transcriptStore = new TranscriptStore(join(tmp, "transcripts")); + }); + afterEach(async () => { + for (const pid of leftovers.splice(0)) try { process.kill(pid, "SIGKILL"); } catch {} + await new Promise((r) => setTimeout(r, 50)); + try { await transcriptStore.flush(); } catch {} + try { store.close(); } catch {} + try { rmSync(tmp, { recursive: true, force: true }); } catch {} + }); + + test("killSurvivors: the re-parented grandchild is killed; a pid whose command changed is spared", async () => { + const mark = String(800000 + Math.floor(Math.random() * 99999)); + const { proc, grandchild } = await spawnBackend(mark); + leftovers.push(grandchild); + const snap = await snapshotTree([proc.pid!]); + expect(snap.map((p) => p.pid)).toContain(grandchild); + proc.kill("SIGKILL"); + await until(async () => !(await listProcesses()).some((p) => p.pid === proc.pid && !p.args.includes("defunct") && p.args.startsWith("sh"))); + expect(alive(grandchild)).toBe(true); // the teardown alone leaves it + const forged = snap.map((p) => (p.pid === grandchild ? p : { ...p, args: `${p.args} (other)` })); + expect(await killSurvivors(forged)).toEqual([grandchild]); + await until(() => !alive(grandchild)); + }); + + test.skipIf(process.platform !== "linux" && process.platform !== "darwin")("a different start time means a different process: spared", async () => { + const mark = String(800000 + Math.floor(Math.random() * 99999)); + const { proc, grandchild } = await spawnBackend(mark); + leftovers.push(grandchild, proc.pid!); + const snap = await snapshotTree([proc.pid!]); + expect(snap.every((p) => p.start !== undefined)).toBe(true); + expect(await killSurvivors(snap.map((p) => ({ ...p, start: `${p.start}0` })))).toEqual([]); + expect(alive(grandchild)).toBe(true); + }); + + test("descendants spawned AFTER the snapshot are killed with the survivor", async () => { + const mark = String(800000 + Math.floor(Math.random() * 99999)); + // The "backend" only starts its long command after we have snapshotted it. + const proc = spawn("sh", ["-c", `sleep 0.3; sleep ${mark} & echo $!; wait`], { stdio: ["ignore", "pipe", "ignore"] }); + leftovers.push(proc.pid!); + const snap = await snapshotTree([proc.pid!]); + expect(snap.some((p) => p.args === `sleep ${mark}`)).toBe(false); + const late = await new Promise((r) => proc.stdout!.once("data", (d: Buffer) => r(Number(String(d).trim())))); + leftovers.push(late); + const killed = await killSurvivors(snap); + expect(killed).toContain(proc.pid!); + expect(killed).toContain(late); + await until(() => !alive(late)); + }); + + test("Session.terminateBackend snapshots before the teardown and kills the leftover", async () => { + const mark = String(800000 + Math.floor(Math.random() * 99999)); + const backend = new ProcessBackend("codex", [[{ type: "tool_start", toolId: "t1", sdkToolUseId: "s1", name: "Bash", input: { command: "x" }, approvalId: "a1" }]], { stall: true }); + const { proc, grandchild } = await spawnBackend(mark); + leftovers.push(grandchild); + backend.proc = proc; + const id = randomUUID(); + store.createSession({ + id, name: "tree", workdir: tmp, status: "idle", createdBy: AUTH.sub, + createdAt: new Date().toISOString(), attachedClients: 0, accountId: AUTH.accountId, projectId: AUTH.projectId, + }); + const session = new Session({ name: "tree", workdir: tmp, auth: AUTH, store, transcriptStore, existingId: id, _testProvider: backend, initialMode: { mode: "autonomous" } }); + await session.send("work", AUTH); + await until(() => session.status === "tool_running"); + await session.terminateBackend(AUTH, "test"); + expect(backend.teardownCount).toBe(1); + await until(() => !alive(grandchild)); + expect(session.status).toBe("idle"); + }); +}); + +describe("argvNamesSession", () => { + const id = "0b9a3c1e-1111-4222-8333-444455556666"; + test("the forms the SDK emits, and their two-token spelling", () => { + expect(argvNamesSession(`/x/claude --output-format stream-json --resume=${id} --verbose`, id)).toBe(true); + expect(argvNamesSession(`/x/claude --session-id=${id}`, id)).toBe(true); + expect(argvNamesSession(`/x/claude --resume ${id}`, id)).toBe(true); + expect(argvNamesSession(`/x/claude --session-id ${id} -p`, id)).toBe(true); + }); + test("a mention of the id is not a match", () => { + expect(argvNamesSession(`grep ${id} /tmp/log`, id)).toBe(false); + expect(argvNamesSession(`/x/claude --resume=${id}-fork`, id)).toBe(false); + expect(argvNamesSession(`/x/claude --append-system-prompt=--resume=${id}`, id)).toBe(false); + expect(argvNamesSession(`/x/claude --resume=other ${id}`, id)).toBe(false); + }); +}); diff --git a/src/tests/provider-claude.test.ts b/src/tests/provider-claude.test.ts index def6d7f5..2220a380 100644 --- a/src/tests/provider-claude.test.ts +++ b/src/tests/provider-claude.test.ts @@ -327,6 +327,25 @@ describe("translateSDKMessage – result", () => { }); }); + it("carries the result text of a success-subtype is_error turn as resultText (where the CLI puts a limit notice)", () => { + const events = collectEmits({ + type: "result", + subtype: "success", + is_error: true, + result: "You've hit your session limit · resets 3pm (UTC)", + usage: { input_tokens: 0, output_tokens: 0 }, + modelUsage: {}, + }); + const result = (events[0] as { result: { resultText?: string; errorMessage?: string } }).result; + expect(result.resultText).toBe("You've hit your session limit · resets 3pm (UTC)"); + expect(result.errorMessage).toBeUndefined(); + }); + + it("no resultText on a turn that did not fail", () => { + const events = collectEmits({ type: "result", subtype: "success", result: "all done", usage: {}, modelUsage: {} }); + expect((events[0] as { result: { resultText?: string } }).result.resultText).toBeUndefined(); + }); + it("sets errorMessage when subtype is not 'success' and result is set", () => { const events = collectEmits({ type: "result", subtype: "error_during_execution", result: "tool failed" }); expect(events[0]).toMatchObject({ type: "turn_done", result: { errorMessage: "tool failed" } }); @@ -1329,6 +1348,29 @@ describe("translateSDKMessage – system", () => { }); }); +describe("translateSDKMessage – rate_limit_event", () => { + it("emits the plan limit state with resetsAt converted from epoch seconds to ms", () => { + const events = collectEmits({ + type: "rate_limit_event", + rate_limit_info: { status: "rejected", resetsAt: 1_800_000_000, rateLimitType: "five_hour" }, + }); + expect(events).toEqual([{ type: "rate_limit", status: "rejected", resetsAt: 1_800_000_000_000, limitType: "five_hour" }]); + }); + + it("flags a rejection that overage is carrying", () => { + for (const info of [{ isUsingOverage: true }, { overageInUse: true }, { overageStatus: "allowed" }, { overageStatus: "allowed_warning" }]) { + const events = collectEmits({ type: "rate_limit_event", rate_limit_info: { status: "rejected", ...info } }); + expect(events[0]).toMatchObject({ type: "rate_limit", status: "rejected", overage: true }); + } + const plain = collectEmits({ type: "rate_limit_event", rate_limit_info: { status: "rejected", overageStatus: "rejected" } }); + expect((plain[0] as { overage?: boolean }).overage).toBeUndefined(); + }); + + it("emits nothing for a status it doesn't know", () => { + expect(collectEmits({ type: "rate_limit_event", rate_limit_info: { status: "mystery" } })).toHaveLength(0); + }); +}); + describe("translateSDKMessage – user (tool_result)", () => { it("emits tool_complete for each tool_result block", () => { const events = collectEmits({ @@ -2041,9 +2083,11 @@ describe("ClaudeProvider – Claude accounts", () => { function makeAccountProvider( current: { account: ResolvedClaudeAccount | null }, onModels?: (models: ReadonlyArray) => void, + extraEnv?: Record, ): ClaudeProvider { return new ClaudeProvider({ ...(onModels ? { onModels } : {}), + ...(extraEnv ? { extraEnv } : {}), sessionId: "acct-session", initialBackingId: "acct-backing", workspaceId: "ws_test", @@ -2093,6 +2137,37 @@ describe("ClaudeProvider – Claude accounts", () => { } }); + it("extraEnv reaches the CLI's env last, but never overrides the account's credential, HOME or a Node preload", async () => { + const saved = process.env.ANTHROPIC_API_KEY; + process.env.ANTHROPIC_API_KEY = "daemon-key-would-win"; + try { + const current = { account: account({ name: "work", kind: "subscription", token: "sk-ant-oat01-WORK" }) }; + const provider = makeAccountProvider(current, undefined, { + LOOP_RUN_ID: "run-7", + DATABASE_URL: "postgres://test", + CLAUDE_CODE_OAUTH_TOKEN: "sk-ant-oat01-EVIL", + ANTHROPIC_API_KEY: "sk-evil", + CLAUDE_CONFIG_DIR: "/tmp/other-login", + HOME: "/tmp/evil-home", + NODE_OPTIONS: "--require /tmp/evil.js", + }); + turnOn(provider, "t1"); + const env = envOf(); + expect(env.LOOP_RUN_ID).toBe("run-7"); + expect(env.DATABASE_URL).toBe("postgres://test"); + expect(env.CLAUDE_CODE_OAUTH_TOKEN).toBe("sk-ant-oat01-WORK"); + expect(env.ANTHROPIC_API_KEY).toBeUndefined(); + expect(env.CLAUDE_CONFIG_DIR).not.toBe("/tmp/other-login"); + expect(env.HOME).toBe(process.env.HOME!); + expect(env.NODE_OPTIONS).not.toBe("--require /tmp/evil.js"); + openGate(); + await provider.teardown(); + } finally { + if (saved === undefined) delete process.env.ANTHROPIC_API_KEY; + else process.env.ANTHROPIC_API_KEY = saved; + } + }); + it("a Bedrock account: mapped families go to the CLI as aliases; its catalog never reaches the shared cache", async () => { const reported: unknown[] = []; const current = { diff --git a/src/tests/provider-codex.test.ts b/src/tests/provider-codex.test.ts index b7076c1d..0fbce963 100644 --- a/src/tests/provider-codex.test.ts +++ b/src/tests/provider-codex.test.ts @@ -27,7 +27,7 @@ import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:f import { tmpdir } from "node:os"; import { join } from "node:path"; import { CodexRpcProcess } from "../daemon/providers/codex/rpc.js"; -import { CodexProvider, describeCodexError } from "../daemon/providers/codex/index.js"; +import { CodexProvider, codexErrorDetail, describeCodexError } from "../daemon/providers/codex/index.js"; import { MemoryMcpHttp, MEMORY_MCP_SERVER_NAME } from "../daemon/memory/mcp-http.js"; import { McpRegistry } from "../daemon/mcp/registry.js"; import { MCP_PROXY_PATH_PREFIX, MCP_PROXY_TOKEN_ENV, McpProxy } from "../daemon/mcp/proxy.js"; @@ -118,6 +118,40 @@ async function collect(run: TurnRun): Promise { } describe("CodexProvider over fake-codex", () => { + it("extraEnv reaches the spawned codex; credential, home and codeoid-owned keys keep codeoid's value", async () => { + const saved = process.env.OPENAI_API_KEY; + process.env.OPENAI_API_KEY = "sk-daemon"; + try { + const p = new CodexProvider({ + sessionId: "sess-env", + initialBackingId: "sess-env", + command: process.execPath, + argsPrefix: [FIXTURE], + store: {} as Store, + extraEnv: { + LOOP_RUN_ID: "run-9", + OPENAI_API_KEY: "sk-evil", + CODEX_HOME: "/tmp/other-login", + HOME: "/tmp/evil-home", + PATH: "/tmp/evil-bin", + CODEOID_MEMORY_TOKEN: "forged", + }, + }); + const events = await collect(p.runTurn(turnOpts("echo-env"))); + await p.teardown(); + const env = JSON.parse((events.find((e) => e.type === "text_done") as { content: string }).content) as Record; + expect(env.LOOP_RUN_ID).toBe("run-9"); + expect(env.OPENAI_API_KEY).toBe("sk-daemon"); + expect(env.CODEX_HOME).toBeUndefined(); + expect(env.HOME).toBe(process.env.HOME!); + expect(env.PATH).toBe(process.env.PATH!); + expect(env.CODEOID_MEMORY_TOKEN).toBeUndefined(); + } finally { + if (saved === undefined) delete process.env.OPENAI_API_KEY; + else process.env.OPENAI_API_KEY = saved; + } + }); + it("C1: text turn streams thinking + text and completes with usage", async () => { const p = makeProvider(); const events = await collect(p.runTurn(turnOpts("hello"))); @@ -1296,6 +1330,31 @@ describe("describeCodexError", () => { }); }); +describe("codexErrorDetail", () => { + it("extracts a unit variant and a reset instant from the error object", () => { + expect(codexErrorDetail({ error: { message: "x", codexErrorInfo: "usageLimitExceeded", resets_at: 1_800_000_000 } })).toEqual({ + codexErrorInfo: "usageLimitExceeded", + resetsAt: 1_800_000_000, + }); + }); + + it("extracts a struct variant's fields (reset, HTTP status)", () => { + expect(codexErrorDetail({ error: { message: "x", codexErrorInfo: { usageLimitExceeded: { resetsAt: 1_800_000_000_000 } } } })).toEqual({ + codexErrorInfo: "usageLimitExceeded", + resetsAt: 1_800_000_000_000, + }); + expect(codexErrorDetail({ error: { message: "x", codexErrorInfo: { httpConnectionFailed: { httpStatusCode: 429 } } } })).toEqual({ + codexErrorInfo: "httpConnectionFailed", + status: 429, + }); + }); + + it("undefined without error info", () => { + expect(codexErrorDetail({ error: { message: "x" } })).toBeUndefined(); + expect(codexErrorDetail({})).toBeUndefined(); + }); +}); + describe("CodexProvider – thinking effort (#417)", () => { const effortSeen = async (effort?: EffortLevel): Promise => { const p = makeProvider(); diff --git a/src/tests/provider-env.test.ts b/src/tests/provider-env.test.ts index 53f924b8..89a8393f 100644 --- a/src/tests/provider-env.test.ts +++ b/src/tests/provider-env.test.ts @@ -5,7 +5,7 @@ */ import { describe, expect, it } from "bun:test"; -import { buildPiEnv, buildSubprocessEnv, withGatewayCredential } from "../daemon/providers/env.js"; +import { buildPiEnv, buildSubprocessEnv, extraEnvViolations, withExtraEnv, withGatewayCredential } from "../daemon/providers/env.js"; import { buildAgentEnv } from "../daemon/providers/claude/index.js"; const DAEMON_ENV: Record = { @@ -178,3 +178,64 @@ describe("withGatewayCredential (forge#111)", () => { expect(env.ANTHROPIC_AUTH_TOKEN).toBe("zid_sk_session"); }); }); + +describe("extraEnv denylist (SessionCreateOptions.extraEnv)", () => { + // Every variable codeoid sets or scrubs for account isolation, plus + // process-loading and request-routing ones. None may be overridden. + const DENIED = [ + "HOME", "USER", "LOGNAME", "PATH", "SHELL", "XDG_CONFIG_HOME", "XDG_DATA_HOME", + "NODE_OPTIONS", "NODE_PATH", "LD_PRELOAD", "LD_LIBRARY_PATH", "DYLD_INSERT_LIBRARIES", + "HTTPS_PROXY", "NODE_EXTRA_CA_CERTS", "NODE_TLS_REJECT_UNAUTHORIZED", "SSL_CERT_FILE", + "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_BASE_URL", "ANTHROPIC_CUSTOM_HEADERS", + "CLAUDE_CODE_OAUTH_TOKEN", "CLAUDE_CONFIG_DIR", "CLAUDE_CODE_USE_BEDROCK", + "AWS_BEARER_TOKEN_BEDROCK", "AWS_PROFILE", "OTEL_EXPORTER_OTLP_ENDPOINT", + "CODEX_HOME", "OPENAI_API_KEY", "OPENAI_BASE_URL", + "GEMINI_API_KEY", "GEMINI_CLI_HOME", "GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_CLOUD_PROJECT", "CODE_ASSIST_ENDPOINT", + "QWEN_HOME", "DASHSCOPE_API_KEY", "BAILIAN_API_KEY", "PI_CODING_AGENT_DIR", + "CODEOID_API_KEY", "CODEOID_AGENT_ENV_ALLOW", "CODEOID_MEMORY_TOKEN", "ZEROID_URL", "TELEGRAM_BOT_TOKEN", + "GROQ_API_KEY", "MISTRAL_API_TOKEN", "SOME_AUTH_TOKEN", "VENDOR_ACCESS_TOKEN", + // Tool configuration that can run code, and runtime knobs. + "RIPGREP_CONFIG_PATH", "BASH_ENV", "ENV", "PROMPT_COMMAND", "PAGER", "GIT_PAGER", "LESSOPEN", "LESSCLOSE", + "EDITOR", "VISUAL", "GCONV_PATH", "RUST_LOG", "RUST_BACKTRACE", "TMPDIR", "TEMP", "TMP", "TZ", + "GIT_CONFIG_GLOBAL", "GIT_SSH_COMMAND", "NPM_CONFIG_USERCONFIG", "PYTHONPATH", "PYTHONSTARTUP", "RUBYOPT", "RUBYLIB", + "PERL5LIB", "PERL5OPT", "PERL_MB_OPT", "JAVA_TOOL_OPTIONS", "_JAVA_OPTIONS", "JDK_JAVA_OPTIONS", "CLOUDSDK_CONFIG", "LC_ALL", + "RUSTC_WRAPPER", "RUSTFLAGS", "CARGO_HOME", "CARGO_BUILD_RUSTC_WRAPPER", "GOFLAGS", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE", + "SSH_AUTH_SOCK", "SHELLOPTS", "BASHOPTS", "PS4", + ]; + const ALLOWED = ["LOOP_RUN_ID", "DATABASE_URL", "CI", "NODE_ENV", "GITHUB_TOKEN", "LANG", "MY_FLAG", "X"]; + + it("refuses every denylisted key and none of the ordinary ones", () => { + const extra = Object.fromEntries([...DENIED, ...ALLOWED].map((k) => [k, "x"])); + expect(extraEnvViolations(extra).sort()).toEqual([...DENIED].sort()); + }); + + it("refuses malformed names and values, and every lower-case or over-long name", () => { + expect( + extraEnvViolations({ + "BAD-NAME": "x", + "1X": "x", + OK: "x", + NUL: "a\0b", + NUM: 1 as unknown as string, + http_proxy: "x", + npm_config_registry: "x", + Mixed_Case: "x", + _LEADING: "x", + ["A".repeat(65)]: "x", + ["A".repeat(64)]: "x", + }), + ).toEqual(["BAD-NAME", "1X", "NUL", "NUM", "http_proxy", "npm_config_registry", "Mixed_Case", "_LEADING", "A".repeat(65)]); + }); + + it("withExtraEnv merges allowed keys over the built env and keeps codeoid's value for refused ones", () => { + const built = { PATH: "/usr/bin", HOME: "/home/u", ANTHROPIC_API_KEY: "ant-key", TZ: "UTC", LOOP_FLAG: "a" }; + const merged = withExtraEnv(built, { PATH: "/evil", HOME: "/evil", ANTHROPIC_API_KEY: "evil", TZ: "Asia/Kolkata", LOOP_FLAG: "b", LOOP_RUN_ID: "r1" }); + expect(merged).toEqual({ PATH: "/usr/bin", HOME: "/home/u", ANTHROPIC_API_KEY: "ant-key", TZ: "UTC", LOOP_FLAG: "b", LOOP_RUN_ID: "r1" }); + expect(built.LOOP_FLAG).toBe("a"); // pure + expect(withExtraEnv(built, undefined)).toBe(built); + }); + + it("a refused key absent from the built env is not added either", () => { + expect(withExtraEnv({ PATH: "/usr/bin" }, { OPENAI_API_KEY: "evil" })).toEqual({ PATH: "/usr/bin" }); + }); +}); diff --git a/src/tests/provider-qwen-accounts.test.ts b/src/tests/provider-qwen-accounts.test.ts index a39f7b99..923b7e05 100644 --- a/src/tests/provider-qwen-accounts.test.ts +++ b/src/tests/provider-qwen-accounts.test.ts @@ -105,4 +105,23 @@ describe("QwenProvider on a named account", () => { void run; await provider.teardown(); }); + + test("extraEnv rides on top, but can't override a daemon key or move the Qwen home", async () => { + const before = queryCalls.length; + const provider = new QwenProvider({ + sessionId: "sess-q2", + initialBackingId: "22222222-2222-4222-8222-222222222222", + workspaceId: "ws", + store: { audit: () => {} } as never, + extraEnv: { LOOP_RUN_ID: "run-q", DASHSCOPE_API_KEY: "evil", QWEN_HOME: "/tmp/evil", OPENAI_ORG_ID: "org-evil" }, + }); + void provider.runTurn({ history: [], userMessage: "hi", workdir: dir, canUseTool: async () => ({ behavior: "allow" as const }) }); + await Bun.sleep(100); + const env = queryCalls[before]!.env!; + expect(env.LOOP_RUN_ID).toBe("run-q"); + expect(env.DASHSCOPE_API_KEY).toBe("daemon-dashscope"); + expect(env.QWEN_HOME).toBe(process.env.QWEN_HOME!); + expect(env.OPENAI_ORG_ID).toBe("org-daemon"); + await provider.teardown(); + }); }); diff --git a/src/tests/session-loop-controls.test.ts b/src/tests/session-loop-controls.test.ts new file mode 100644 index 00000000..07c8e725 --- /dev/null +++ b/src/tests/session-loop-controls.test.ts @@ -0,0 +1,612 @@ +/** + * Session-level controls a loop controller drives (loop design §6, §8, §12.11), + * through a real Session over MockSessionProvider: + * + * - lastTurnInfraError: a turn failed on a usage / rate limit (with its + * reset), set on failure, cleared by the next successful turn; + * - onBudgetExhausted: "end_attempt" denies and ends the turn, settling idle, + * while the default "park" still waits for approval; + * - extraEnv: refused keys fail the create; allowed ones reach the provider init; + * - supervise(): a wall-time deadline that fires while a tool is running + * (stall watchdog paused), kills the backend after the grace period, and + * never fires once cancelled. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { randomUUID } from "node:crypto"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Session, type SessionCreateOptions } from "../daemon/session.js"; +import { MockSessionProvider } from "../daemon/providers/mock/session-provider.js"; +import { mockResult } from "../daemon/providers/mock/index.js"; +import { ProviderRegistry, type ProviderSessionInit } from "../daemon/providers/registry.js"; +import type { ProviderEvent, TurnOpts, TurnRun } from "../daemon/providers/interface.js"; +import { supervise, type DeadlineEvent, type SupervisedSession } from "../daemon/supervisor.js"; +import { Store } from "../daemon/store.js"; +import { TranscriptStore } from "../daemon/transcript.js"; +import type { AuthContext, SessionStatus } from "../protocol/types.js"; +import type { CodeoidConfig } from "../config.js"; + +const AUTH: AuthContext = { sub: "user:loops", scopes: [], delegationDepth: 0, accountId: "acc", projectId: "proj" }; +const SYSTEM: AuthContext = { ...AUTH, sub: "system:dispatch" }; + +let tmp: string; +let store: Store; +let transcriptStore: TranscriptStore; + +beforeEach(() => { + tmp = mkdtempSync(join(tmpdir(), "codeoid-loopctl-")); + store = new Store(join(tmp, "codeoid.db")); + transcriptStore = new TranscriptStore(join(tmp, "transcripts")); +}); +afterEach(async () => { + await new Promise((r) => setTimeout(r, 50)); + try { await transcriptStore.flush(); } catch {} + try { store.close(); } catch {} + try { rmSync(tmp, { recursive: true, force: true }); } catch {} +}); + +function makeSession(extra: Partial = {}): Session { + const id = randomUUID(); + store.createSession({ + id, name: "loop", workdir: tmp, status: "idle", createdBy: AUTH.sub, + createdAt: new Date().toISOString(), attachedClients: 0, accountId: AUTH.accountId, projectId: AUTH.projectId, + }); + return new Session({ name: "loop", workdir: tmp, auth: AUTH, store, transcriptStore, existingId: id, ...extra }); +} + +async function until(cond: () => boolean, ms = 3_000): Promise { + const deadline = Date.now() + ms; + while (!cond()) { + if (Date.now() > deadline) throw new Error("condition not reached"); + await new Promise((r) => setTimeout(r, 5)); + } +} + +const done = (overrides: Parameters[0] = {}): ProviderEvent => ({ type: "turn_done", result: mockResult(overrides) }); +const bash = (n: number): ProviderEvent => ({ + type: "tool_start", toolId: `t${n}`, sdkToolUseId: `sdk-t${n}`, name: "Bash", input: { command: `echo ${n}` }, approvalId: `ap-${n}`, +}); +const bashDone = (n: number): ProviderEvent => ({ type: "tool_complete", sdkToolUseId: `sdk-t${n}`, output: `${n}`, success: true }); + +/** Config with a short stall watchdog (autoRotate off — #shouldRotate reads it on every send). */ +function stallConfig(turnStallTimeoutMs: number): CodeoidConfig { + return { + session: { turnStallTimeoutMs }, + autoRotate: { enabled: false, warnPct: 0.75, rotatePct: 0.9, hardRotatePct: 0.95, minTurnsBeforeRotate: 1, strategy: "task-anchor" }, + } as unknown as CodeoidConfig; +} + +// ── lastTurnInfraError ────────────────────────────────────────────────────── + +describe("lastTurnInfraError", () => { + test("a usage-limit turn records kind + reset; the next successful turn clears it", async () => { + const provider = new MockSessionProvider("claude", [ + [done({ isError: true, errorMessage: "You've hit your session limit · resets 3pm (UTC)" })], + [{ type: "text_done", content: "ok" }, done()], + ]); + const session = makeSession({ _testProvider: provider }); + expect(session.lastTurnInfraError).toBeNull(); + + await session.send("work", AUTH); + await until(() => session.status === "error"); + const infra = session.lastTurnInfraError; + expect(infra?.kind).toBe("usage_limit"); + expect(infra?.provider).toBe("claude"); + expect(infra?.message).toContain("session limit"); + expect(new Date(infra!.resetAt!).getUTCHours()).toBe(15); + expect(infra!.resetAt!).toBeGreaterThan(Date.now() - 60_000); + + await session.send("again", AUTH); + await until(() => session.status === "idle" && session.lastTurnInfraError === null); + }); + + test("the Claude CLI's limit notice in the turn's own assistant text is recognized", async () => { + const provider = new MockSessionProvider("claude", [ + [{ type: "text_done", content: "API Error: Repeated 529 Overloaded errors" }, done({ isError: true })], + ]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError?.kind).toBe("rate_limit"); + }); + + test("a provider error event, and an ordinary failure afterwards resets it to null", async () => { + const provider = new MockSessionProvider("codex", [ + [{ type: "error", message: "codex: You've hit your usage limit. Try again in 2h." }], + [{ type: "error", message: "codex exited unexpectedly (code=1 signal=null)" }], + ]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + const infra = session.lastTurnInfraError; + expect(infra?.kind).toBe("usage_limit"); + expect(infra!.resetAt! - infra!.at).toBeGreaterThan(2 * 3600_000 - 5_000); + + await session.send("again", AUTH); + await until(() => session.status === "error" && session.lastTurnInfraError === null); + }); + + test("a provider error's structured detail (codex) gives the exact reset", async () => { + const resetSec = Math.floor(Date.now() / 1000) + 3 * 3600; + const provider = new MockSessionProvider("codex", [ + [{ type: "error", message: "codex: You've hit your usage limit.", detail: { codexErrorInfo: "usageLimitExceeded", resetsAt: resetSec } }], + ]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).toMatchObject({ + kind: "usage_limit", + resetAt: resetSec * 1000, + message: "codex: You've hit your usage limit.", + provider: "codex", + }); + }); + + test("Claude's rate_limit_event supplies the exact reset when the error text has none", async () => { + const resetsAt = Date.now() + 4 * 3600_000; + const provider = new MockSessionProvider("claude", [ + [{ type: "rate_limit", status: "rejected", resetsAt, limitType: "five_hour" }, done({ isError: true, errorMessage: "Claude AI usage limit reached" })], + ]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).toMatchObject({ kind: "usage_limit", resetAt: resetsAt }); + }); + + test("an ordinary failed turn records nothing", async () => { + const provider = new MockSessionProvider("claude", [[done({ isError: true, errorMessage: "Prompt is too long" })]]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).toBeNull(); + }); +}); + +// ── onBudgetExhausted ────────────────────────────────────────────────────── + +describe("onBudgetExhausted", () => { + // Budget 1: the first Bash spends it, the second finds it spent. `stall` + // keeps the turn open afterwards, so only an interrupt can end it. + const script = () => [[bash(1), bashDone(1), bash(2)]]; + + test("default park: the call waits for approval, the session wedges at waiting_approval", async () => { + const provider = new MockSessionProvider("claude", script(), { stall: true }); + const session = makeSession({ _testProvider: provider, initialMode: { mode: "autonomous", maxTurns: 1 } }); + await session.send("work", AUTH); + await until(() => session.pendingToolApprovals().length === 1); + expect(session.status).toBe("waiting_approval"); + expect(session.pendingToolApprovals()[0]!.toolName).toBe("Bash"); + expect(provider.canUseToolResults.map((r) => r.behavior)).toEqual(["allow"]); + expect(session.budgetExhausted).toBe(false); + expect(session.mode).toBe("guarded"); + await session.interrupt(AUTH); + }); + + test("end_attempt: the call is denied, the turn interrupted, the session idle with budgetExhausted", async () => { + const provider = new MockSessionProvider("claude", script(), { stall: true }); + const session = makeSession({ + _testProvider: provider, + initialMode: { mode: "autonomous", maxTurns: 1 }, + onBudgetExhausted: "end_attempt", + }); + const seen: SessionStatus[] = []; + session.attach({ id: "c", auth: AUTH, send: (m) => { if (m.type === "session.status_change") seen.push(m.status); } }); + await session.send("work", AUTH); + await until(() => provider.canUseToolResults.length === 2); + expect(provider.canUseToolResults[1]!.behavior).toBe("deny"); + expect(provider.canUseToolResults[1]!.message).toMatch(/budget/i); + await until(() => session.status === "idle" && session.turnInterrupted); + expect(seen).not.toContain("waiting_approval"); + expect(session.budgetExhausted).toBe(true); + expect(session.mode).toBe("guarded"); + expect(session.pendingToolApprovals()).toEqual([]); + + // A new autonomous budget is a new attempt. + session.setMode("autonomous", 3); + expect(session.budgetExhausted).toBe(false); + }); + + test("end_attempt with budget to spare behaves exactly like park", async () => { + const provider = new MockSessionProvider("claude", [[bash(1), bashDone(1), done()]]); + const session = makeSession({ + _testProvider: provider, + initialMode: { mode: "autonomous", maxTurns: 5 }, + onBudgetExhausted: "end_attempt", + }); + await session.send("work", AUTH); + await until(() => session.status === "idle" && provider.canUseToolResults.length === 1); + expect(provider.canUseToolResults[0]!.behavior).toBe("allow"); + expect(session.turnsRemaining).toBe(4); + expect(session.budgetExhausted).toBe(false); + }); +}); + +// ── extraEnv ─────────────────────────────────────────────────────────────── + +describe("extraEnv", () => { + test("refused keys fail the create, naming them", () => { + expect(() => + makeSession({ _testProvider: new MockSessionProvider("claude"), extraEnv: { LOOP_RUN_ID: "r", ANTHROPIC_API_KEY: "k", HOME: "/x" } }), + ).toThrow(/ANTHROPIC_API_KEY, HOME/); + }); + + test("allowed keys reach the provider through ProviderSessionInit", () => { + let captured: ProviderSessionInit | undefined; + const providers = new ProviderRegistry("cap"); + providers.register({ + id: "cap", + displayName: "Capture", + create: (init) => { + captured = init; + return new MockSessionProvider("cap"); + }, + }); + makeSession({ providers, providerId: "cap", extraEnv: { LOOP_RUN_ID: "run-1", DATABASE_URL: "postgres://t" } }); + expect(captured?.extraEnv).toEqual({ LOOP_RUN_ID: "run-1", DATABASE_URL: "postgres://t" }); + }); +}); + +// ── supervise() ──────────────────────────────────────────────────────────── + +/** A backend whose interrupt never returns — the wedge that makes the kill necessary. */ +class WedgedProvider extends MockSessionProvider { + override runTurn(opts: TurnOpts): TurnRun { + const run = super.runTurn(opts); + return { ...run, interrupt: () => new Promise(() => {}) }; + } +} + +describe("supervise()", () => { + test("fires while a tool is running (watchdog paused) and kills the backend after the grace period", async () => { + // The tool starts and never completes: tool_running, the stall watchdog + // (30ms here) paused for the whole time. + const provider = new WedgedProvider("claude", [[bash(1)]], { stall: true }); + const session = makeSession({ _testProvider: provider, config: stallConfig(30), initialMode: { mode: "autonomous" } }); + await session.send("long tool", AUTH); + await until(() => session.status === "tool_running"); + await new Promise((r) => setTimeout(r, 80)); // well past the stall window + expect(session.status).toBe("tool_running"); + expect(provider.teardownCount).toBe(0); + + const events: DeadlineEvent[] = []; + const deadline = supervise(session, { wallMs: 50, graceMs: 60, onEvent: (e) => events.push(e) }); + expect(await deadline.expired).toBe(true); + expect(events.map((e) => e.type)).toEqual(["expired", "terminated"]); + expect(provider.teardownCount).toBe(1); + expect(session.status).toBe("idle"); + expect(session.turnInterrupted).toBe(true); + }); + + test("a session the interrupt settles is not killed", async () => { + const provider = new MockSessionProvider("claude", [[bash(1)]], { stall: true }); + const session = makeSession({ _testProvider: provider, initialMode: { mode: "autonomous" } }); + await session.send("long tool", AUTH); + await until(() => session.status === "tool_running"); + const events: DeadlineEvent[] = []; + const deadline = supervise(session, { wallMs: 20, graceMs: 2_000, onEvent: (e) => events.push(e) }); + expect(await deadline.expired).toBe(true); + expect(events.map((e) => e.type)).toEqual(["expired", "settled"]); + expect(provider.teardownCount).toBe(0); + }); +}); + +describe("supervise() — timers", () => { + function fake(status: SessionStatus = "tool_running") { + const calls: string[] = []; + const s: SupervisedSession & { status: SessionStatus } = { + id: "s1", + accountId: "a", + projectId: "p", + status, + preparingTurn: false, + interrupt: async (sender) => { calls.push(`interrupt:${sender.sub}`); }, + terminateBackend: async (sender) => { calls.push(`terminate:${sender.sub}`); s.status = "idle"; }, + }; + return { s, calls }; + } + + test("cancel before expiry: nothing fires, expired resolves false", async () => { + const { s, calls } = fake(); + const d = supervise(s, { wallMs: 30, graceMs: 10 }); + d.cancel(); + expect(await d.expired).toBe(false); + await new Promise((r) => setTimeout(r, 80)); + expect(calls).toEqual([]); + d.cancel(); // idempotent + }); + + test("interrupt, then kill when still busy after grace; audited as the supervisor", async () => { + const { s, calls } = fake(); + const d = supervise(s, { wallMs: 10, graceMs: 20 }); + expect(await d.expired).toBe(true); + expect(calls).toEqual(["interrupt:system:supervisor", "terminate:system:supervisor"]); + }); + + test("cancel during the grace period abandons the kill", async () => { + const { s, calls } = fake(); + const d = supervise(s, { wallMs: 10, graceMs: 200 }); + await until(() => calls.length === 1); + d.cancel(); + expect(await d.expired).toBe(true); + await new Promise((r) => setTimeout(r, 250)); + expect(calls).toEqual(["interrupt:system:supervisor"]); + }); + + test("a queued send counts as busy", async () => { + const { s, calls } = fake("idle"); + (s as { preparingTurn: boolean }).preparingTurn = true; + const d = supervise(s, { wallMs: 5, graceMs: 15 }); + expect(await d.expired).toBe(true); + expect(calls.at(-1)).toBe("terminate:system:supervisor"); + }); + + test("rejects a non-positive wall time or a negative grace", () => { + const { s } = fake(); + expect(() => supervise(s, { wallMs: 0, graceMs: 0 })).toThrow(RangeError); + expect(() => supervise(s, { wallMs: 10, graceMs: -1 })).toThrow(RangeError); + }); +}); + +// ── Round-1 audit fixes ────────────────────────────────────────────────────── + +const rejected = (extra: Partial> = {}): ProviderEvent => ({ + type: "rate_limit", + status: "rejected", + resetsAt: Date.now() + 3600_000, + ...extra, +}); + +describe("rate_limit fallback (audit #1)", () => { + test("a rejection that overage is carrying is not a limit", async () => { + const provider = new MockSessionProvider("claude", [[rejected({ overage: true }), done({ isError: true, errorMessage: "tool crashed" })]]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).toBeNull(); + }); + + for (const [label, errorMessage] of [ + ["prompt too long", "Prompt is too long"], + ["the spend ceiling", "Stopped at the spend ceiling for this run (maxBudgetUsd)."], + ] as const) { + test(`a rejection does not relabel a failure already explained by ${label}`, async () => { + const provider = new MockSessionProvider("claude", [[rejected(), done({ isError: true, errorMessage })]]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).toBeNull(); + }); + } + + test("an unexplained failure after a plain rejection is still a usage limit", async () => { + const r = rejected(); + const provider = new MockSessionProvider("claude", [[r, done({ isError: true })]]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).toMatchObject({ kind: "usage_limit", resetAt: (r as { resetsAt: number }).resetsAt }); + }); +}); + +describe("CLI notice in result / assistant text (audit #2)", () => { + test("Claude's result text on an is_error turn is classified", async () => { + const provider = new MockSessionProvider("claude", [ + [done({ isError: true, resultText: "You've hit your weekly limit · resets Oct 28, 9am (UTC)" })], + ]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError?.kind).toBe("usage_limit"); + }); + + for (const reply of [ + "The upstream API returned 429 Too Many Requests twice, so I added retries.", + "HTTP 429 handler added; tests pass", + "You've reached your goal: the rate limit test passes", + ]) { + test(`a model reply in a failed turn is not a limit: ${JSON.stringify(reply)}`, async () => { + const provider = new MockSessionProvider("claude", [[{ type: "text_done", content: reply }, done({ isError: true, resultText: reply })]]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).toBeNull(); + }); + } +}); + +describe("turn-start clearing (audit #3, #5)", () => { + test("lastTurnInfraError is cleared when the next turn starts, not when it ends", async () => { + const provider = new MockSessionProvider("claude", [ + [done({ isError: true, errorMessage: "You've hit your session limit · resets 3pm (UTC)" })], + [bash(1)], + ]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).not.toBeNull(); + await session.send("again", AUTH); + await until(() => session.pendingToolApprovals().length === 1); // turn 2 running, parked on approval + expect(session.lastTurnInfraError).toBeNull(); + await session.interrupt(AUTH); + }); + + test("a message that joins the running turn keeps that turn's rate-limit record", async () => { + const r = rejected(); + const provider = new MockSessionProvider("claude", [[r, { type: "text_delta", content: "working" }]], { stall: true, midTurn: true }); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "thinking"); + await new Promise((res) => setTimeout(res, 20)); + await session.send("also this", AUTH); // joins (mid-turn push), no new turn + expect(provider.midTurnPushes.length).toBe(1); + // The interrupted partial turn's boundary is absorbed; the continuation fails. + provider.emitLive(done()); + provider.emitLive(done({ isError: true })); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).toMatchObject({ kind: "usage_limit", resetAt: (r as { resetsAt: number }).resetsAt }); + }); +}); + +describe("end_attempt keeps the attempt ended (audit #4)", () => { + test("later prompted and backend-started turns are denied too, and background tasks are stopped", async () => { + const provider = new MockSessionProvider("claude", [[bash(1), bashDone(1), bash(2)], [bash(3)]], { stall: true }); + const session = makeSession({ + _testProvider: provider, + initialMode: { mode: "autonomous", maxTurns: 1 }, + onBudgetExhausted: "end_attempt", + }); + provider.onSessionEvent?.({ type: "background_tasks", tasks: [{ id: "bg-1", kind: "subagent", description: "bg", status: "running" }] }); + await session.send("work", AUTH); + await until(() => provider.canUseToolResults.length === 2 && session.status === "idle"); + await until(() => provider.stoppedTasks.includes("bg-1")); + expect(session.budgetExhausted).toBe(true); + + // A later prompt from a system principal (the dispatcher, a controller): still ended. + await session.send("try again", SYSTEM); + await until(() => provider.canUseToolResults.length === 3); + expect(provider.canUseToolResults[2]!.behavior).toBe("deny"); + await until(() => session.status === "idle"); + expect(session.budgetExhausted).toBe(true); + + // A turn the backend starts on its own: still ended, flag kept. + provider.startOwnTurn([bash(4)]); + await until(() => provider.boundGates.length === 1); + await new Promise((res) => setTimeout(res, 20)); + const verdict = await provider.boundGates[0]!.canUseTool("t4", "ap-4", "Bash", { command: "echo 4" }); + expect(verdict.behavior).toBe("deny"); + expect(session.budgetExhausted).toBe(true); + + session.setMode("autonomous", 2); + expect(session.budgetExhausted).toBe(false); + }); +}); + +class UnbudgetedProvider extends MockSessionProvider { + readonly enforcesToolBudget = false; +} + +describe("toolBudgetEnforced (audit #7)", () => { + test("true on a gated backend; false, with a warning, on one that skips approvals", () => { + expect(makeSession({ _testProvider: new MockSessionProvider("claude") }).toolBudgetEnforced).toBe(true); + const warn = console.warn; + const warnings: string[] = []; + console.warn = (m: unknown) => { warnings.push(String(m)); }; + try { + const s = makeSession({ _testProvider: new UnbudgetedProvider("codex"), onBudgetExhausted: "end_attempt" }); + expect(s.toolBudgetEnforced).toBe(false); + expect(warnings.some((w) => w.includes("can't be enforced"))).toBe(true); + warnings.length = 0; + makeSession({ _testProvider: new UnbudgetedProvider("codex") }); // park: nothing to warn about + expect(warnings).toEqual([]); + } finally { + console.warn = warn; + } + }); +}); + +/** A backend whose interrupt waits for the test to release it. */ +class SlowInterruptProvider extends MockSessionProvider { + release!: () => void; + override runTurn(opts: TurnOpts): TurnRun { + const run = super.runTurn(opts); + const gate = new Promise((r) => { this.release = r; }); + return { ...run, interrupt: () => gate }; + } +} + +/** A backend whose teardown never completes. */ +class StuckTeardownProvider extends MockSessionProvider { + override async teardown(): Promise { + this.teardownCount++; + await new Promise(() => {}); + } +} + +describe("interrupt and terminate races (audit #8)", () => { + test("an interrupt that returns after another turn took over leaves that turn alone", async () => { + const provider = new SlowInterruptProvider("claude", [[bash(1)], [bash(2)]], { stall: true }); + const session = makeSession({ _testProvider: provider }); + await session.send("first", AUTH); + await until(() => session.pendingToolApprovals().length === 1); + const releaseFirst = provider.release; // turn 1's gate (turn 2 gets its own) + const pending = session.interrupt(AUTH); + await session.terminateBackend(AUTH, "test"); + expect(session.status).toBe("idle"); + await session.send("second", AUTH); + await until(() => session.pendingToolApprovals().some((a) => a.approvalId === "ap-2")); + releaseFirst(); + await pending; + expect(session.status).toBe("waiting_approval"); + expect(session.pendingToolApprovals().map((a) => a.approvalId)).toEqual(["ap-2"]); + await session.terminateBackend(AUTH, "cleanup"); + }); + + test("terminateBackend does not hang on a teardown that never finishes", async () => { + const provider = new StuckTeardownProvider("claude", [[bash(1)]], { stall: true }); + const session = makeSession({ _testProvider: provider, initialMode: { mode: "autonomous" } }); + await session.send("work", AUTH); + await until(() => session.status === "tool_running"); + const t0 = Date.now(); + await session.terminateBackend(AUTH, "test", { teardownTimeoutMs: 50 }); + expect(Date.now() - t0).toBeLessThan(2_000); + expect(session.status).toBe("idle"); + expect(provider.teardownCount).toBe(1); + }); +}); + +// ── Round-2 audit fixes ────────────────────────────────────────────────────── + +describe("a person can take over a spent end_attempt worker (audit-2 #1)", () => { + async function spentWorker() { + const provider = new MockSessionProvider("claude", [[bash(1), bashDone(1), bash(2)], [bash(3)]], { stall: true }); + const session = makeSession({ + _testProvider: provider, + initialMode: { mode: "autonomous", maxTurns: 1 }, + onBudgetExhausted: "end_attempt", + }); + const infos: Array> = []; + session.attach({ id: "c", auth: AUTH, send: (m) => { if (m.type === "session.info_update") infos.push(m as unknown as Record); } }); + await session.send("work", SYSTEM); + await until(() => provider.canUseToolResults.length === 2 && session.status === "idle"); + expect(session.budgetExhausted).toBe(true); + return { provider, session, infos }; + } + + test("session info says so, and the denial says how to re-arm", async () => { + const { provider, session, infos } = await spentWorker(); + const last = infos.at(-1) as { session?: { budgetExhausted?: boolean } } | undefined; + expect(JSON.stringify(last)).toContain('"budgetExhausted":true'); + expect(provider.canUseToolResults[1]!.message).toContain("/mode"); + void session; + }); + + test("a person's setMode lifts it — even to the mode already set; a system one does not", async () => { + const { session } = await spentWorker(); + session.setMode("guarded", undefined, SYSTEM); + expect(session.budgetExhausted).toBe(true); + session.setMode("guarded", undefined, AUTH); // already guarded + expect(session.budgetExhausted).toBe(false); + }); + + test("a person's new turn lifts it: the call is gated normally, not denied", async () => { + const { provider, session } = await spentWorker(); + await session.send("let me look", AUTH); + await until(() => session.pendingToolApprovals().length === 1); + expect(session.budgetExhausted).toBe(false); + expect(provider.canUseToolResults).toHaveLength(2); // the third call is waiting on a person, not refused + await session.interrupt(AUTH); + }); +}); + +describe("the CLI notice as the reply's last block (audit-2 #6)", () => { + test("narration, then the limit notice", async () => { + const reply = "I'll start with the failing test.\n\nYou've hit your session limit · resets 3pm (UTC)"; + const provider = new MockSessionProvider("claude", [[{ type: "text_done", content: reply }, done({ isError: true })]]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError?.kind).toBe("usage_limit"); + }); +}); diff --git a/src/tests/usage-limit.test.ts b/src/tests/usage-limit.test.ts new file mode 100644 index 00000000..2555e7f3 --- /dev/null +++ b/src/tests/usage-limit.test.ts @@ -0,0 +1,267 @@ +/** + * classifyUsageLimit — the real message shapes each backend emits for a usage + * or rate limit (sources cited in usage-limit.ts), and the ordinary errors that + * must NOT be mistaken for one. + */ + +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { CLAUDE_USAGE_LIMIT_PREFIXES, classifyCliNotice, classifyUsageLimit } from "../daemon/providers/usage-limit.js"; + +// 2026-10-25 12:00 UTC — 05:00 in Los Angeles (PDT, UTC-7: DST ends Nov 1). +const NOW = Date.UTC(2026, 9, 25, 12, 0); +const at = (input: unknown) => classifyUsageLimit(input, { now: NOW }); + +describe("classifyUsageLimit — usage limits", () => { + test("Claude legacy `Claude AI usage limit reached|`", () => { + const resetSec = Math.floor(NOW / 1000) + 3600; + expect(at(`Claude AI usage limit reached|${resetSec}`)).toEqual({ kind: "usage_limit", resetAt: resetSec * 1000, provider: "claude" }); + }); + + test("Claude Code: You've hit your session limit · resets 3pm (America/Los_Angeles)", () => { + expect(at("You've hit your session limit · resets 3pm (America/Los_Angeles)")).toEqual({ + kind: "usage_limit", + resetAt: Date.UTC(2026, 9, 25, 22, 0), + }); + }); + + test("Claude Code: weekly limit with a date and a minute, typographic apostrophe", () => { + expect(at("You’ve hit your weekly limit · resets Oct 28, 9:30am (UTC)")?.resetAt).toBe(Date.UTC(2026, 9, 28, 9, 30)); + }); + + test("a bare time already past today resets tomorrow", () => { + // 4am UTC is 8 hours before NOW. + expect(at("You've hit your Opus limit · resets 4am (UTC)")?.resetAt).toBe(Date.UTC(2026, 9, 26, 4, 0)); + }); + + test("older CLI form: 5-hour limit reached ∙ resets 3pm (host time zone)", () => { + const r = at("5-hour limit reached ∙ resets 3pm"); + expect(r?.kind).toBe("usage_limit"); + expect(new Date(r!.resetAt!).getHours()).toBe(15); + }); + + test("Claude org / seat usage messages", () => { + expect(at("Your org is out of usage · add funds to continue")?.kind).toBe("usage_limit"); + expect(at("You're out of extra usage")?.kind).toBe("usage_limit"); + expect(at("You've hit your monthly spend limit.")?.kind).toBe("usage_limit"); + }); + + test("Codex: You've hit your usage limit … try again at 3:30 PM (local time)", () => { + const r = at( + "codex: You've hit your usage limit. Upgrade to Plus to continue using Codex (https://chatgpt.com/explore/plus), or try again at 3:30 PM.", + ); + expect(r?.kind).toBe("usage_limit"); + expect(r?.provider).toBe("codex"); + const d = new Date(r!.resetAt!); + expect([d.getHours(), d.getMinutes()]).toEqual([15, 30]); + }); + + test("Codex: a reset on another day", () => { + const r = at("You've hit your usage limit. To get more access now, send a request to your admin or try again at Oct 28th, 2026 3:30 PM."); + expect(r?.kind).toBe("usage_limit"); + expect(r?.resetAt).toBe(new Date(2026, 9, 28, 15, 30).getTime()); + }); + + test("Codex structured error info", () => { + expect(at({ message: "turn failed", codexErrorInfo: "usageLimitExceeded" })).toEqual({ kind: "usage_limit", provider: "codex" }); + }); + + test("Codex structured error info carries the exact reset (seconds or ms, unit or struct variant)", () => { + const resetSec = Math.floor(NOW / 1000) + 7200; + expect(at({ message: "codex: You've hit your usage limit.", error: { codexErrorInfo: "usageLimitExceeded", resetsAt: resetSec } })).toEqual({ + kind: "usage_limit", + resetAt: resetSec * 1000, + provider: "codex", + }); + // The struct field beats the prose time. + expect( + at({ message: "codex: You've hit your usage limit. Try again at 3:30 PM.", error: { codexErrorInfo: { usageLimitExceeded: { resets_at: resetSec * 1000 } } } }) + ?.resetAt, + ).toBe(resetSec * 1000); + expect(at({ message: "codex: slow down", error: { codexErrorInfo: "rateLimitExceeded" } })).toEqual({ kind: "rate_limit", provider: "codex" }); + expect(at({ message: "codex: request failed", error: { codexErrorInfo: "httpConnectionFailed", status: 429 } })?.kind).toBe("rate_limit"); + expect(at({ message: "codex: boom", error: { codexErrorInfo: "contextWindowExceeded" } })).toBeNull(); + }); + + test("gemini-cli: Usage limit reached … Access resets at 3:30 PM PDT (host time; the short zone name is ignored)", () => { + const r = at("Usage limit reached for gemini-2.5-pro.\nAccess resets at 3:30 PM PDT.\n/stats model for usage details\n/model to switch models."); + expect(r?.kind).toBe("usage_limit"); + const d = new Date(r!.resetAt!); + expect([d.getHours(), d.getMinutes()]).toEqual([15, 30]); + }); + + test("gemini-cli TerminalQuotaError (message, class name, retry delay)", () => { + expect(at("gemini-cli: You have exhausted your daily quota on this model.")).toEqual({ kind: "usage_limit", provider: "gemini" }); + const err = Object.assign(new Error("quota"), { name: "TerminalQuotaError", retryDelayMs: 3_600_000 }); + expect(at(err)).toEqual({ kind: "usage_limit", resetAt: NOW + 3_600_000, provider: "gemini" }); + }); + + test("Google daily quota metric", () => { + expect(at("Quota exceeded for quota metric 'Gemini 2.5 Pro Requests' and limit 'Gemini 2.5 Pro Requests per day per user'")?.kind).toBe( + "usage_limit", + ); + }); + + test("OpenAI billing quota", () => { + expect(at("429 You exceeded your current quota, please check your plan and billing details.")?.kind).toBe("usage_limit"); + expect(at({ status: 429, error: { code: "insufficient_quota", message: "quota" } })?.kind).toBe("usage_limit"); + }); + + test("Qwen / DashScope allocation quota", () => { + expect(at("[API Error: Throttling.AllocationQuota: free allocated quota exceeded]")).toEqual({ kind: "usage_limit", provider: "qwen" }); + expect(at("Qwen OAuth quota exceeded: Your free daily quota has been reached.")?.kind).toBe("usage_limit"); + }); + + test("the caller's provider wins over the one a shape implies", () => { + expect(classifyUsageLimit("You've hit your usage limit.", { now: NOW, provider: "claude" })?.provider).toBe("claude"); + }); + + test("an implausible reset (in the past, or months out) is dropped, the kind kept", () => { + expect(at("Claude AI usage limit reached|1000")).toEqual({ kind: "usage_limit", provider: "claude" }); + expect(at(`Claude AI usage limit reached|${Math.floor(NOW / 1000) + 90 * 86_400}`)?.resetAt).toBeUndefined(); + }); +}); + +describe("classifyUsageLimit — rate limits", () => { + test("Anthropic API errors: 429 rate_limit_error and 529 overloaded_error", () => { + expect(at('429 {"type":"error","error":{"type":"rate_limit_error","message":"Number of request tokens has exceeded your per-minute rate limit"}}')).toEqual({ + kind: "rate_limit", + provider: "claude", + }); + expect(at('API Error: 529 {"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}')?.kind).toBe("rate_limit"); + }); + + test("Claude CLI after its own retries", () => { + expect(at("API Error: Repeated 529 Overloaded errors")?.kind).toBe("rate_limit"); + expect(at("API Error: Request rejected (429) · this may be a temporary capacity issue.")?.kind).toBe("rate_limit"); + // Says "usage limit" — but explicitly NOT one. + expect(at("API Error: Server is temporarily limiting requests (not your usage limit) · Rate limited")?.kind).toBe("rate_limit"); + }); + + test("Gemini free tier: RESOURCE_EXHAUSTED with a short retry delay", () => { + const body = JSON.stringify({ + error: { + code: 429, + message: "You exceeded your current quota, please check your plan and billing details. Please retry in 41.5s.", + status: "RESOURCE_EXHAUSTED", + }, + }); + expect(at(`GeminiProvider: ${body}`)).toEqual({ kind: "rate_limit", resetAt: NOW + 41_500 }); + }); + + test("qwen-code formatted 429", () => { + expect( + at("[API Error: Requests rate limit exceeded, please try again later. (Status: RESOURCE_EXHAUSTED)]\nPossible quota limitations in place or slow response times detected. Please wait and try again later.")?.kind, + ).toBe("rate_limit"); + }); + + test("OpenAI rate limit with its retry hint", () => { + expect( + at("429 Rate limit reached for gpt-4.1 in organization org-x on tokens per min (TPM): Limit 30000, Used 29000, Requested 2000. Please try again in 2.5s."), + ).toEqual({ kind: "rate_limit", resetAt: NOW + 2500 }); + }); + + test("codex retry exhaustion and a bare structured 429", () => { + expect(at("codex: exceeded retry limit, last status: 429 Too Many Requests")?.kind).toBe("rate_limit"); + expect(at({ status: 429, message: "request failed" })?.kind).toBe("rate_limit"); + expect(at({ message: "slow down", cause: { statusCode: 529 } })?.kind).toBe("rate_limit"); + }); + + test("durations in several units", () => { + expect(at("Rate limit exceeded. Try again in 1h 30m")?.resetAt).toBe(NOW + 90 * 60_000); + expect(at("Too Many Requests\nRetry-After: 30")?.resetAt).toBe(NOW + 30_000); + }); +}); + +describe("classifyUsageLimit — ordinary errors are not limits", () => { + const negatives: unknown[] = [ + "Error: 401 Invalid API key · Please run /login", + "Prompt is too long: 210000 tokens > 200000 maximum", + "Concurrent session limit reached (max 3)", + "Context limit reached", + "Budget limit reached ($5.00) — raise maxBudgetUsd to continue", + "codex exited unexpectedly (code=1 signal=null)", + "Build a rate limiter for the API", + "You've hit your stride — keep going", + "ECONNRESET socket hang up", + "Turn ended with an error", + "TypeError: cannot read properties of undefined (line 429)", + "Qwen OAuth free tier has been discontinued as of 2026-04-15.", + "Subagent nesting limit reached (depth 3)", + { status: 500, message: "Internal server error" }, + "", + null, + undefined, + {}, + ]; + for (const input of negatives) { + test(`null for ${JSON.stringify(input)}`, () => { + expect(at(input)).toBeNull(); + }); + } +}); + +// Replies a model might write that discuss limits — the audit's false +// positives when assistant text was read with the generic rules. +const MODEL_REPLIES = [ + "The upstream API returned 429 Too Many Requests twice, so I added retries.", + "HTTP 429 handler added; tests pass", + "You've reached your goal: the rate limit test passes", + "Rate limit exceeded errors are now retried with backoff (RESOURCE_EXHAUSTED included).", + "I hit the usage limit reached branch in the parser and fixed it.", + "You've hit your target — the overloaded_error path is covered.", +]; + +describe("classifyCliNotice — CLI-shaped rules only", () => { + test("a notice the CLI composes, at the start of the text", () => { + expect(classifyCliNotice("You've hit your session limit · resets 3pm (UTC)", { now: NOW })).toEqual({ + kind: "usage_limit", + resetAt: Date.UTC(2026, 9, 25, 15, 0), + }); + expect(classifyCliNotice("You've reached your Fable limit.", { now: NOW })?.kind).toBe("usage_limit"); + expect(classifyCliNotice("You've hit your team's shared budget. Switch to another model to continue.", { now: NOW })?.kind).toBe("usage_limit"); + expect(classifyCliNotice(`Claude AI usage limit reached|${Math.floor(NOW / 1000) + 60}`, { now: NOW })?.resetAt).toBe(NOW + 60_000); + expect(classifyCliNotice("API Error: Repeated 529 Overloaded errors", { now: NOW })?.kind).toBe("rate_limit"); + expect(classifyCliNotice("API Error: Request rejected (429) · this may be a temporary capacity issue.", { now: NOW })?.kind).toBe("rate_limit"); + }); + + test("the last block of a reply that narrated first", () => { + expect(classifyCliNotice("Working on it.\n\nYou've hit your weekly limit · resets Oct 28, 9am (UTC)", { now: NOW })?.resetAt).toBe( + Date.UTC(2026, 9, 28, 9, 0), + ); + expect(classifyCliNotice("Done.\n\nThe API returned 429 Too Many Requests, so I added retries.", { now: NOW })).toBeNull(); + }); + + test("not when it is not the opening of the text", () => { + expect(classifyCliNotice("Done. You've hit your session limit · resets 3pm (UTC)", { now: NOW })).toBeNull(); + expect(classifyCliNotice("The log said API Error: Repeated 529 Overloaded errors", { now: NOW })).toBeNull(); + }); + + test("an API Error that is not a limit is not one", () => { + expect(classifyCliNotice("API Error: 401 Invalid API key · Please run /login", { now: NOW })).toBeNull(); + }); + + for (const reply of MODEL_REPLIES) { + test(`a model reply is not a notice: ${JSON.stringify(reply)}`, () => { + expect(classifyCliNotice(reply, { now: NOW })).toBeNull(); + }); + } + + test("the generic openers need the limit noun right after them in the full classifier too", () => { + expect(at("You've reached your goal: the rate limit test passes")).toBeNull(); + expect(at("codex: You've hit your usage limit.")?.kind).toBe("usage_limit"); + }); +}); + +describe("CLAUDE_USAGE_LIMIT_PREFIXES", () => { + test("matches the installed Agent SDK's USAGE_LIMIT_ERROR_PREFIXES", () => { + // Read from the SDK's declarations, not imported: another suite mocks the + // module, and the export is @alpha (the copy exists so a bump can't break us). + const dts = readFileSync(join(import.meta.dir, "..", "..", "node_modules", "@anthropic-ai", "claude-agent-sdk", "sdk.d.ts"), "utf8"); + const decl = /USAGE_LIMIT_ERROR_PREFIXES: readonly \[([^\]]*)\]/.exec(dts); + expect(decl).not.toBeNull(); + const sdk = [...decl![1]!.matchAll(/"([^"]*)"|'([^']*)'/g)].map((m) => m[1] ?? m[2]); + expect([...CLAUDE_USAGE_LIMIT_PREFIXES]).toEqual(sdk); + }); +});