From 03ee605aac49447b1147657f6a045b42203b9000 Mon Sep 17 00:00:00 2001 From: Yash Datta Date: Sun, 25 Oct 2026 08:09:11 +0800 Subject: [PATCH 1/4] =?UTF-8?q?feat:=20controls=20for=20unattended=20worke?= =?UTF-8?q?rs=20=E2=80=94=20usage-limit=20detection,=20end-attempt=20budge?= =?UTF-8?q?ts,=20extra=20env,=20wall-time=20deadlines?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Session-level primitives a loop controller will drive (loop design §6, §8, §12.11). Each stands alone; default behavior is unchanged. - providers/usage-limit.ts: classifyUsageLimit() tells a usage limit or a rate limit (with its reset time) from an ordinary failure, keyed on the shapes Claude Code, Codex, gemini-cli, Gemini, OpenAI and qwen-code actually emit. Session records it as lastTurnInfraError on every failed turn and clears it on the next success; Claude's rate_limit_event now surfaces as a provider event so the exact reset instant is kept. - onBudgetExhausted: "end_attempt" denies the call that finds the autonomous budget spent, interrupts the turn, settles idle and sets budgetExhausted, instead of parking at waiting_approval ("park" stays the default). - extraEnv: caller env for every process-backed backend, merged last but never over a credential, vendor home, HOME/PATH, proxy/TLS or process-loading variable (explicit denylist; the create fails on one). - supervisor.ts: supervise(session, { wallMs, graceMs }) interrupts on expiry regardless of tool state, then terminates the backend (Session.terminateBackend, sharing stall recovery's hard stop) if the session has not settled within the grace period. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/daemon/providers/acp/index.ts | 12 +- src/daemon/providers/claude/index.ts | 36 +- src/daemon/providers/codex/index.ts | 19 +- src/daemon/providers/env.ts | 79 +++++ src/daemon/providers/interface.ts | 7 + src/daemon/providers/pi/index.ts | 9 +- src/daemon/providers/qwen/index.ts | 26 +- src/daemon/providers/registry.ts | 14 + src/daemon/providers/usage-limit.ts | 418 +++++++++++++++++++++++ src/daemon/session.ts | 314 +++++++++++++++-- src/daemon/supervisor.ts | 155 +++++++++ src/tests/fixtures/fake-codex.ts | 3 + src/tests/provider-claude.test.ts | 47 +++ src/tests/provider-codex.test.ts | 34 ++ src/tests/provider-env.test.ts | 44 ++- src/tests/provider-qwen-accounts.test.ts | 19 ++ src/tests/session-loop-controls.test.ts | 333 ++++++++++++++++++ src/tests/usage-limit.test.ts | 191 +++++++++++ 18 files changed, 1715 insertions(+), 45 deletions(-) create mode 100644 src/daemon/providers/usage-limit.ts create mode 100644 src/daemon/supervisor.ts create mode 100644 src/tests/session-loop-controls.test.ts create mode 100644 src/tests/usage-limit.test.ts diff --git a/src/daemon/providers/acp/index.ts b/src/daemon/providers/acp/index.ts index 1afae808..b56cd2fe 100644 --- a/src/daemon/providers/acp/index.ts +++ b/src/daemon/providers/acp/index.ts @@ -38,7 +38,7 @@ import type { CatalogEntry, } from "../interface.js"; import { renderHistorySeed, type CanonicalTurn, type HistorySeedResult } from "../canonical.js"; -import { buildGeminiCliEnv } from "../env.js"; +import { buildGeminiCliEnv, withExtraEnv } from "../env.js"; import { StdioJsonRpcProcess } from "../jsonrpc-stdio.js"; import { MEMORY_MCP_SERVER_NAME, type MemoryMcpMount } from "../../memory/mcp-http.js"; import { BLACKBOARD_MCP_SERVER_NAME } from "../../blackboard/mcp-http.js"; @@ -89,6 +89,8 @@ export interface GeminiAcpProviderInit { */ resolveAccount?: () => ResolvedAccount | null; onModels?: (models: ReadonlyArray) => void; + /** Caller env for the child process (ProviderSessionInit.extraEnv), merged last via withExtraEnv. */ + extraEnv?: Readonly>; } export class GeminiAcpProvider implements SessionProvider { @@ -109,6 +111,7 @@ export class GeminiAcpProvider implements SessionProvider { #command: string; #argsPrefix: string[]; #resolveAccount: (() => ResolvedAccount | null) | null; + readonly #extraEnv: Readonly> | undefined; #workspaceId: string; #memoryMcp: MemoryMcpMount | null; /** Role-scoped blackboard mount; token minted+revoked by the SessionManager. */ @@ -144,6 +147,7 @@ export class GeminiAcpProvider implements SessionProvider { this.#backingSessionId = init.initialBackingId; this.#command = init.command; this.#resolveAccount = init.resolveAccount ?? null; + this.#extraEnv = init.extraEnv; this.#argsPrefix = init.argsPrefix ?? []; this.#workspaceId = init.workspaceId ?? init.sessionId; this.#memoryMcp = init.memoryMcp ?? null; @@ -315,7 +319,11 @@ export class GeminiAcpProvider implements SessionProvider { cwd: opts.workdir, // The account's own GEMINI_CLI_HOME (an overlay selecting its auth type). // ...and, for a thinking effort, an overlay of that whose settings set it (#417). - env: applyGeminiCliEffort(opts.effort, applyGeminiCliAccount(this.#resolveAccount?.() ?? null, buildGeminiCliEnv())), + // Then the caller's extra env, which can't override either (withExtraEnv). + env: withExtraEnv( + applyGeminiCliEffort(opts.effort, applyGeminiCliAccount(this.#resolveAccount?.() ?? null, buildGeminiCliEnv())), + this.#extraEnv, + ), onNotification: (method, params) => this.#onNotification(method, params), onServerRequest: (method, params) => this.#onServerRequest(method, params), onExit: ({ code, signal, stderrTail }) => { diff --git a/src/daemon/providers/claude/index.ts b/src/daemon/providers/claude/index.ts index f7366ca5..f161c74a 100644 --- a/src/daemon/providers/claude/index.ts +++ b/src/daemon/providers/claude/index.ts @@ -66,7 +66,7 @@ import { removeClaudeSession, toClaudeSessionEntries, writeClaudeSession } from import { BUDGET_STOP_MESSAGE, isBackgroundLifecycleEvent } from "../interface.js"; import { isUngrantableSkillCommand, redactCommand } from "../../skill-command.js"; import { renderHistorySeed, type CanonicalTurn, type HistorySeedResult } from "../canonical.js"; -import { buildSubprocessEnv, withGatewayCredential } from "../env.js"; +import { buildSubprocessEnv, withExtraEnv, withGatewayCredential } from "../env.js"; import type { LLMCallUsage } from "../../context-math.js"; import type { PackSubagent } from "../../pipeline/subagents.js"; @@ -132,6 +132,8 @@ export interface ClaudeProviderInit { * switch takes effect on the next turn and RESUMES the same backing session. */ resolveAccount?: () => ResolvedClaudeAccount | null; + /** Caller env for the CLI child process (ProviderSessionInit.extraEnv), merged last via withExtraEnv. */ + extraEnv?: Readonly>; /** codeoid session id — used as the initial backing id and for audit/logging. */ sessionId: string; /** Test seam for native continuation (#387); defaults to the real SDK. */ @@ -859,11 +861,18 @@ export class ClaudeProvider implements SessionProvider { // applyAccountEnv). The forge gateway re-point applies only to the // daemon's own environment: a named account carries the user's own // gateway token, which must never be swapped for an agent identity. - env: account - ? account.apply(buildAgentEnv(process.env)) - : buildAgentEnv(process.env, { - gatewayCredential: init.identityManager?.getGatewayCredential(init.sessionId), - }), + // + // The caller's extra env (a loop worker's) goes on last; withExtraEnv + // refuses every ANTHROPIC_/CLAUDE_ and credential-shaped key, so it + // can never undo the account or the gateway re-point. + env: withExtraEnv( + account + ? account.apply(buildAgentEnv(process.env)) + : buildAgentEnv(process.env, { + gatewayCredential: init.identityManager?.getGatewayCredential(init.sessionId), + }), + init.extraEnv, + ), allowedTools: [ // Every EXACT TOOL NAME here is auto-approved by the SDK BEFORE // canUseTool is consulted, so it never reaches our gate — the SDK @@ -1877,6 +1886,21 @@ export function translateSDKMessage( emit({ type: "tool_progress", toolName: p.tool_name, elapsedSeconds: p.elapsed_time_seconds }); break; } + + case "rate_limit_event": { + // SDKRateLimitInfo.resetsAt is epoch SECONDS (the CLI renders it with + // `new Date(resetsAt * 1000)`); ProviderEvent carries milliseconds. + const info = (msg as { rate_limit_info?: { status?: string; resetsAt?: number; rateLimitType?: string } }).rate_limit_info; + const status = info?.status; + if (status !== "allowed" && status !== "allowed_warning" && status !== "rejected") break; + emit({ + type: "rate_limit", + status, + ...(typeof info?.resetsAt === "number" && info.resetsAt > 0 ? { resetsAt: info.resetsAt * 1000 } : {}), + ...(info?.rateLimitType ? { limitType: info.rateLimitType } : {}), + }); + break; + } } } diff --git a/src/daemon/providers/codex/index.ts b/src/daemon/providers/codex/index.ts index 576040ac..a53a089c 100644 --- a/src/daemon/providers/codex/index.ts +++ b/src/daemon/providers/codex/index.ts @@ -65,7 +65,7 @@ import type { } from "../interface.js"; import { renderHistorySeed, type CanonicalTurn, type HistorySeedResult } from "../canonical.js"; import { type ResponsesItem, toResponsesItems } from "./native-history.js"; -import { buildCodexEnv } from "../env.js"; +import { buildCodexEnv, withExtraEnv } from "../env.js"; import { CodexRpcProcess } from "./rpc.js"; import { EFFORT_LEVELS, type EffortLevel, type SessionMode } from "../../../protocol/types.js"; import { MEMORY_MCP_SERVER_NAME, MEMORY_MCP_TOKEN_ENV, type MemoryMcpMount } from "../../memory/mcp-http.js"; @@ -130,6 +130,8 @@ export interface CodexProviderInit { */ resolveAccount?: () => ResolvedAccount | null; onModels?: (models: ReadonlyArray) => void; + /** Caller env for the child process (ProviderSessionInit.extraEnv), merged last via withExtraEnv. */ + extraEnv?: Readonly>; } /** Item types that surface as tool records (vs text/reasoning streams). */ @@ -282,6 +284,7 @@ export class CodexProvider implements SessionProvider { #command: string; #argsPrefix: string[]; #resolveAccount: (() => ResolvedAccount | null) | null; + readonly #extraEnv: Readonly> | undefined; #onModels?: CodexProviderInit["onModels"]; #workspaceId: string; #memoryMcp: MemoryMcpMount | null; @@ -374,6 +377,7 @@ export class CodexProvider implements SessionProvider { this.#backingSessionId = init.initialBackingId; this.#command = init.command; this.#resolveAccount = init.resolveAccount ?? null; + this.#extraEnv = init.extraEnv; this.#argsPrefix = init.argsPrefix ?? []; this.#onModels = init.onModels; this.#workspaceId = init.workspaceId ?? init.sessionId; @@ -821,10 +825,15 @@ export class CodexProvider implements SessionProvider { cwd: opts.workdir, // The account's own CODEX_HOME (an overlay sharing everything but its // auth.json), applied last so nothing re-adds a daemon key. - env: await applyCodexAccount( - this.#resolveAccount?.() ?? null, - { ...buildCodexEnv(), ...mcp.env, ...bb.env, ...reg.env, ...oauth.env }, - { command: this.#command, argsPrefix: this.#argsPrefix }, + // The caller's extra env goes on last (withExtraEnv refuses anything + // credential- or home-shaped, so the account stays the account). + env: withExtraEnv( + await applyCodexAccount( + this.#resolveAccount?.() ?? null, + { ...buildCodexEnv(), ...mcp.env, ...bb.env, ...reg.env, ...oauth.env }, + { command: this.#command, argsPrefix: this.#argsPrefix }, + ), + this.#extraEnv, ), onNotification: (method, params) => this.#onNotification(method, params), onServerRequest: (method, params) => this.#onServerRequest(method, params), diff --git a/src/daemon/providers/env.ts b/src/daemon/providers/env.ts index b0d44a23..d6ad7e62 100644 --- a/src/daemon/providers/env.ts +++ b/src/daemon/providers/env.ts @@ -80,6 +80,85 @@ export function buildSubprocessEnv( return out; } +// ── Caller-supplied extra env (SessionCreateOptions.extraEnv) ──────────────── + +/** + * What a caller's extra env may NEVER set on a backend's child process: the + * variables codeoid owns for credential selection and account isolation, and + * the ones that change what the backend process loads or where its requests + * go. A loop controller passes extra env for the work (a test DB URL, a build + * flag); none of it may swap the account a session runs on, re-point a vendor + * home at another login, inject code into the backend (NODE_OPTIONS, + * LD_PRELOAD), or route its bearer token through a proxy. + * + * Matched case-insensitively, by exact name, namespace prefix, or credential + * suffix. Deliberately broad: a controller that needs one of these has to get + * it through the account system, not an env side door. + */ +const EXTRA_ENV_DENY_EXACT = new Set([ + // Identity and the vendor homes every account overlay lives under. + "HOME", "USER", "LOGNAME", "USERPROFILE", + "XDG_CONFIG_HOME", "XDG_DATA_HOME", "XDG_STATE_HOME", "XDG_CACHE_HOME", "XDG_RUNTIME_DIR", + // What runs: the binary lookup, the shell tools run in, Node preloads. + "PATH", "SHELL", "NODE_OPTIONS", "NODE_PATH", "BUN_OPTIONS", + // Where requests go, and which TLS roots let something read them in flight + // (the REDIRECTING_ENV sets in claude-accounts.ts / account-apply.ts). + "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", + "NODE_EXTRA_CA_CERTS", "NODE_TLS_REJECT_UNAUTHORIZED", "SSL_CERT_FILE", "SSL_CERT_DIR", + "CODE_ASSIST_ENDPOINT", +]); +const EXTRA_ENV_DENY_PREFIXES = [ + // codeoid's own namespaces (DENY_PREFIXES above) — tokens, the escape hatch. + "CODEOID_", "ZEROID_", "TELEGRAM_", + // Every backend's credential / home / endpoint namespace the account + // appliers set or scrub (claude-accounts.ts, account-apply.ts). + "ANTHROPIC_", "CLAUDE_", "CODEX_", "OPENAI_", "GEMINI_", "GOOGLE_", "QWEN_", "DASHSCOPE_", "BAILIAN_", "PI_", + // Cloud credentials (Bedrock, Vertex, Azure) and where telemetry goes. + "AWS_", "AZURE_", "VERTEX_", "OTEL_", + // Dynamic-loader injection. + "LD_", "DYLD_", +] as const; +const EXTRA_ENV_DENY_SUFFIXES = ["_API_KEY", "_API_TOKEN", "_AUTH_TOKEN", "_ACCESS_TOKEN", "_REFRESH_TOKEN", "_OAUTH_TOKEN"] as const; +const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; + +function extraEnvRefused(name: string, value: unknown): boolean { + if (!ENV_NAME.test(name) || typeof value !== "string" || value.includes("\0")) return true; + const upper = name.toUpperCase(); + return ( + EXTRA_ENV_DENY_EXACT.has(upper) || + EXTRA_ENV_DENY_PREFIXES.some((p) => upper.startsWith(p)) || + EXTRA_ENV_DENY_SUFFIXES.some((s) => upper.endsWith(s)) + ); +} + +/** + * The keys of `extra` codeoid refuses to pass to a backend (denylisted, not a + * valid variable name, or a non-string / NUL-bearing value). Empty = all allowed. + */ +export function extraEnvViolations(extra: Readonly>): string[] { + return Object.entries(extra) + .filter(([k, v]) => extraEnvRefused(k, v)) + .map(([k]) => k); +} + +/** + * `env` (what codeoid built for the backend, account applied) with the + * caller's extra env merged over it — minus every refused key, which keeps + * codeoid's value. Pure. Session refuses such keys up front; filtering here + * again means a provider built by some other path can't be talked into it. + */ +export function withExtraEnv( + env: Record, + extra: Readonly> | undefined, +): Record { + if (!extra) return env; + const out: Record = { ...env }; + for (const [k, v] of Object.entries(extra)) { + if (!extraEnvRefused(k, v)) out[k] = v; + } + return out; +} + /** * The header Highflame's AI gateway (firehog) authenticates a sandboxed agent * on. Forge injects it at launch carrying the SANDBOX BADGE; see diff --git a/src/daemon/providers/interface.ts b/src/daemon/providers/interface.ts index 36ce95e1..8a128891 100644 --- a/src/daemon/providers/interface.ts +++ b/src/daemon/providers/interface.ts @@ -336,6 +336,13 @@ export type ProviderEvent = /** Per-LLM-call usage, split by primary vs subagent (null parent_tool_use_id = primary). */ | { type: "llm_call"; usage: LLMCallUsage; isPrimary: boolean } | { type: "api_retry"; attempt?: number; retryDelayMs?: number; errorStatus?: number | null } + /** + * The backend's plan rate-limit state changed (Claude's `rate_limit_event`, + * subscription accounts only). `rejected` means requests are refused until + * `resetsAt` (epoch ms) — the exact reset a usage-limit error's text often + * gives only as a wall-clock time. Observational: nothing retries on it. + */ + | { type: "rate_limit"; status: "allowed" | "allowed_warning" | "rejected"; resetsAt?: number; limitType?: string } | { type: "tool_progress"; toolName?: string; elapsedSeconds?: number } /** * Provider-authored standalone message — extension output, status cards, diff --git a/src/daemon/providers/pi/index.ts b/src/daemon/providers/pi/index.ts index f3b34de6..70db03a3 100644 --- a/src/daemon/providers/pi/index.ts +++ b/src/daemon/providers/pi/index.ts @@ -36,7 +36,7 @@ import type { } from "../interface.js"; import { EFFORT_LEVELS, type EffortLevel, type ProviderCommand } from "../../../protocol/types.js"; import { renderHistorySeed, type CanonicalTurn, type HistorySeedResult } from "../canonical.js"; -import { buildPiEnv } from "../env.js"; +import { buildPiEnv, withExtraEnv } from "../env.js"; import { APPROVAL_TITLE, type BridgeMcpTool, @@ -83,6 +83,8 @@ export interface PiProviderInit { */ resolveAccount?: () => ResolvedAccount | null; onModels?: (models: ReadonlyArray) => void; + /** Caller env for the child process (ProviderSessionInit.extraEnv), merged last via withExtraEnv. */ + extraEnv?: Readonly>; } interface PiSessionStats { @@ -110,6 +112,7 @@ export class PiProvider implements SessionProvider { #backingSessionId: string; #command: string; #resolveAccount: (() => ResolvedAccount | null) | null; + readonly #extraEnv: Readonly> | undefined; #argsPrefix: string[]; #store: Store; #onModels?: PiProviderInit["onModels"]; @@ -144,6 +147,7 @@ export class PiProvider implements SessionProvider { this.#backingSessionId = init.initialBackingId; this.#command = init.command; this.#resolveAccount = init.resolveAccount ?? null; + this.#extraEnv = init.extraEnv; this.#argsPrefix = init.argsPrefix ?? []; this.#store = init.store; this.#onModels = init.onModels; @@ -330,7 +334,8 @@ export class PiProvider implements SessionProvider { // Allowlisted env (GHSA-38vh): pi gets its credentials (~/.pi via // HOME, provider API keys), never codeoid's own secrets. // pi's thinking-level writes stay in a copy of its settings (#417). - env: this.#piEnv(), + // Then the caller's extra env, which can't override the account (withExtraEnv). + env: withExtraEnv(this.#piEnv(), this.#extraEnv), onEvent: (frame) => this.#onFrame(frame), onExit: ({ code, signal, stderrTail }) => { this.#bridgeReady = false; diff --git a/src/daemon/providers/qwen/index.ts b/src/daemon/providers/qwen/index.ts index 2c1ecc60..11a97f62 100644 --- a/src/daemon/providers/qwen/index.ts +++ b/src/daemon/providers/qwen/index.ts @@ -54,7 +54,7 @@ import type { CatalogEntry, } from "../interface.js"; import { renderHistorySeed, type CanonicalTurn, type HistorySeedResult } from "../canonical.js"; -import { buildQwenEnv } from "../env.js"; +import { buildQwenEnv, withExtraEnv } from "../env.js"; import type { LLMCallUsage } from "../../context-math.js"; import type { ResolvedAccount } from "../../accounts.js"; import { qwenAccountEnv } from "../../account-apply.js"; @@ -97,6 +97,8 @@ export interface QwenProviderInit { */ resolveAccount?: () => ResolvedAccount | null; onModels?: (models: ReadonlyArray) => void; + /** Caller env for the child process (ProviderSessionInit.extraEnv), merged last via withExtraEnv. */ + extraEnv?: Readonly>; } export class QwenProvider implements SessionProvider { @@ -440,14 +442,20 @@ export class QwenProvider implements SessionProvider { abortController: this.#abortController, // Explicit allowlist + blanking of everything else — the SDK MERGES // this over process.env rather than replacing it (see buildQwenEnv). - env: { - ...buildQwenEnv(), - // An account's key, and none of the daemon's (blanked: the SDK merges over process.env). - // An account: its key and its own home, and none of the daemon's keys. - ...(account ? qwenAccountEnv(account, buildQwenEnv()) : {}), - ...(baseUrl ? { OPENAI_BASE_URL: baseUrl } : {}), - ...(model ? { OPENAI_MODEL: model } : {}), - }, + // The caller's extra env goes on last; withExtraEnv refuses every + // OPENAI_/QWEN_/DASHSCOPE_ and credential-shaped key, so it can't + // undo the blanking or the account. + env: withExtraEnv( + { + ...buildQwenEnv(), + // An account's key, and none of the daemon's (blanked: the SDK merges over process.env). + // An account: its key and its own home, and none of the daemon's keys. + ...(account ? qwenAccountEnv(account, buildQwenEnv()) : {}), + ...(baseUrl ? { OPENAI_BASE_URL: baseUrl } : {}), + ...(model ? { OPENAI_MODEL: model } : {}), + }, + init.extraEnv, + ), authType, ...(model ? { model } : {}), // qwen-code takes a LIST of capacity fallbacks (max 3), where the diff --git a/src/daemon/providers/registry.ts b/src/daemon/providers/registry.ts index d7e48aa5..11c013b9 100644 --- a/src/daemon/providers/registry.ts +++ b/src/daemon/providers/registry.ts @@ -85,6 +85,15 @@ export interface ProviderSessionInit { * HTTP backends call it per turn; process backends when they spawn. */ resolveAccount?: () => ResolvedAccount | null; + /** + * Caller-supplied env for the backend's child process (SessionCreateOptions + * .extraEnv), merged last via `withExtraEnv` (env.ts), which never lets it + * override a credential / account-isolation variable. Honored by every + * process-backed provider — claude, codex, gemini-cli, pi, qwen. In-process + * backends (openai, gemini, llama.cpp, user endpoints) spawn nothing, so it + * has no effect there. + */ + extraEnv?: Readonly>; } /** An HTTP backend's key from the session's account (accounts.ts), or null for the daemon's own. */ @@ -373,6 +382,7 @@ export function createDefaultProviderRegistry( compressionRegistry: init.compressionRegistry, onModels: init.onModels, resolveAccount: init.resolveClaudeAccount, + extraEnv: init.extraEnv, }), }); // Stateless API backends: usable only with a key — in the environment (the @@ -436,6 +446,7 @@ export function createDefaultProviderRegistry( mcpHub: init.mcpHub, onModels: init.onModels, resolveAccount: init.resolveAccount, + extraEnv: init.extraEnv, }), }); } else { @@ -469,6 +480,7 @@ export function createDefaultProviderRegistry( mcpRegistry: init.mcpRegistry, onModels: init.onModels, resolveAccount: init.resolveAccount, + extraEnv: init.extraEnv, }), }); } else { @@ -502,6 +514,7 @@ export function createDefaultProviderRegistry( mcpRegistry: init.mcpRegistry, onModels: init.onModels, resolveAccount: init.resolveAccount, + extraEnv: init.extraEnv, }), }); } else { @@ -536,6 +549,7 @@ export function createDefaultProviderRegistry( config: init.config, onModels: init.onModels, resolveAccount: init.resolveAccount, + extraEnv: init.extraEnv, }), }); } diff --git a/src/daemon/providers/usage-limit.ts b/src/daemon/providers/usage-limit.ts new file mode 100644 index 00000000..9d94b201 --- /dev/null +++ b/src/daemon/providers/usage-limit.ts @@ -0,0 +1,418 @@ +/** + * Usage-limit and rate-limit detection for failed turns (loop design §12.11). + * + * A loop controller must tell "the provider said wait" apart from "the work + * failed": the first is an infrastructure error that consumes no attempt and + * parks the run until a reset time, the second is evidence about the attempt. + * This module is the one place that reads a backend's error text and decides. + * + * Two kinds, deliberately coarse: + * + * - `usage_limit` — a plan or quota allowance is spent (Claude's 5-hour / + * weekly windows, Codex's plan limit, Gemini's daily quota, a billing + * quota). The remedy is to wait for the window to reset or to move to + * another account; the reset is typically hours away, sometimes unknown. + * - `rate_limit` — transient throttling or overload (HTTP 429 / 529, + * RESOURCE_EXHAUSTED with a short retry delay). The remedy is a short wait. + * + * CONSERVATIVE by design: a false positive would park a run whose work + * actually failed, hiding a real failure behind a wait. Every rule below keys + * on a shape a real backend emits — the source is cited beside it — and + * ordinary errors (auth failures, context overflow, codeoid's own + * "Concurrent session limit reached", a model mentioning "rate limiter") + * classify as null. A generic phrase never matches on its own. + * + * `resetAt` is an epoch-milliseconds instant, present only when the error + * itself says when the limit lifts (an epoch, a duration, or a wall-clock + * time). It is sanity-bounded: a parsed time in the past or more than 35 days + * out is dropped rather than trusted. + */ + +export type UsageLimitKind = "usage_limit" | "rate_limit"; + +export interface UsageLimitClassification { + kind: UsageLimitKind; + /** Epoch ms at which the limit is expected to lift, when the error says. */ + resetAt?: number; + /** Backend the shape identifies (or the caller's `opts.provider`). */ + provider?: string; +} + +export interface ClassifyUsageLimitOptions { + /** Reference "now" for relative and wall-clock reset times (tests). Default Date.now(). */ + now?: number; + /** Backend id the error came from; wins over the one a shape implies. */ + provider?: string; +} + +/** + * The Claude Code CLI's "a usage limit was genuinely reached" messages — a + * copy of the Agent SDK's `USAGE_LIMIT_ERROR_PREFIXES` + * (node_modules/@anthropic-ai/claude-agent-sdk/sdk.d.ts, SDK 0.3.281, "the + * error-path outputs of getLimitReachedText (rateLimitMessages.ts)"). Copied + * rather than imported because the export is `@alpha` and a usage-limit + * check must not break on an SDK bump; usage-limit.test.ts fails if the two + * drift. The CLI composes them as `You've hit your ${limit} · resets ${time}` + * with limit ∈ {session limit, weekly limit, Opus limit, Sonnet limit, Fable + * limit} (the CLI bundle's `_h` / `Ide` table). Codex uses the same opener: + * "You've hit your usage limit. … Try again at 3:30 PM." (codex-rs 0.154.0 + * binary strings). + */ +export const CLAUDE_USAGE_LIMIT_PREFIXES: readonly string[] = [ + "You've hit your", + "You've reached your", + "You're out of usage credits", + "Your org is out of usage · add funds to continue", + "Your org is out of usage · contact your admin", + "Your seat type doesn't include usage credits", + "Your seat type doesn't include usage", + "Your usage allocation has been disabled by your admin", + "Your group's usage limit is set to $0", + "Fable 5 requires usage credits", + "You're out of extra usage", + "Your seat type doesn't include extra usage", +]; + +/** Openers too generic to trust alone ("You've hit your stride"): they need a limit word after them. */ +const GENERIC_PREFIXES = new Set(["You've hit your", "You've reached your"]); +const LIMIT_WORD = /\b(?:limit|usage|quota|budget|credits?|spend)\b/i; + +/** + * gemini-cli treats a quota error whose retry delay is at most this as + * retryable, longer as terminal (`MAX_RETRYABLE_DELAY_SECONDS`, gemini-cli + * 0.50.0 bundle, packages/core googleQuotaErrors). The same line splits + * rate_limit from usage_limit here when only the delay tells them apart. + */ +const RETRYABLE_DELAY_MS = 300_000; + +/** Plausible reset horizon: Claude's longest window is 7 days; a month of slack. */ +const MAX_RESET_HORIZON_MS = 35 * 24 * 3600_000; + +interface Rule { + kind: UsageLimitKind; + test: (text: string) => boolean; + provider?: string; +} + +const re = (pattern: RegExp, kind: UsageLimitKind, provider?: string): Rule => ({ + kind, + test: (t) => pattern.test(t), + ...(provider ? { provider } : {}), +}); + +/** Usage-limit shapes, checked before the rate-limit ones (a 429 carrying a usage text is a usage limit). */ +const USAGE_RULES: readonly Rule[] = [ + // Claude Code before the "You've hit your" wording: `Claude AI usage limit + // reached|` as the result text of a limited turn. + re(/Claude AI usage limit reached/i, "usage_limit", "claude"), + // Older and toast forms in the Claude CLI bundle: "5-hour limit reached ∙ + // resets 3pm", "Usage limit reached …". Only named windows — never + // "Concurrent session limit reached" (codeoid) or "Context limit reached". + re(/(? 3) return; + if (typeof input === "string") { + if (input.length > 0) out.texts.push(input); + return; + } + if (typeof input !== "object") return; + const o = input as Record; + if (typeof o.message === "string" && o.message.length > 0) out.texts.push(o.message); + // A class name carries meaning on its own (gemini-cli's TerminalQuotaError). + if (typeof o.name === "string" && /QuotaError$/.test(o.name)) out.texts.push(o.name); + for (const key of ["status", "statusCode", "code"] as const) { + const v = o[key]; + if (out.status === undefined && (v === 429 || v === 529)) out.status = v; + } + // Machine codes: OpenAI's `code: "insufficient_quota"`, Anthropic's + // `error.type: "rate_limit_error"`, Google's `status: "RESOURCE_EXHAUSTED"`. + for (const key of ["code", "type", "status"] as const) { + const v = o[key]; + if (typeof v === "string" && /^[A-Za-z][\w.]{2,63}$/.test(v)) out.texts.push(v); + } + if (out.retryDelayMs === undefined && typeof o.retryDelayMs === "number" && o.retryDelayMs > 0) { + out.retryDelayMs = o.retryDelayMs; + } + // codex: { codexErrorInfo: "usageLimitExceeded" } or { usageLimitExceeded: {...} }. + const info = o.codexErrorInfo; + if (typeof info === "string") out.texts.push(info); + else if (info && typeof info === "object") out.texts.push(Object.keys(info).join(" ")); + if (o.error !== undefined && o.error !== input) collect(o.error, out, depth + 1); + if (o.cause !== undefined && o.cause !== input) collect(o.cause, out, depth + 1); +} + +/** Normalize separators and apostrophes the CLIs render with typographic glyphs. */ +function normalize(text: string): string { + return text.replace(/[‘’ʼ]/g, "'").replace(/[∙•]/g, "·"); +} + +function prefixRule(text: string): boolean { + for (const prefix of CLAUDE_USAGE_LIMIT_PREFIXES) { + // At the start, or right after a wrapper's colon/bracket/newline + // ("codex: You've hit…", "Error: You've hit…", "[API Error: You've…"). + const at = new RegExp(`(?:^|[:\\]\\n]\\s*)${escapeRegExp(prefix)}`).exec(text); + if (!at) continue; + if (!GENERIC_PREFIXES.has(prefix)) return true; + const rest = text.slice(at.index + at[0].length).split("\n", 1)[0] ?? ""; + if (LIMIT_WORD.test(rest)) return true; + } + return false; +} + +function escapeRegExp(s: string): string { + return s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +/** + * Classify a failed turn's error — an Error, a provider error object, or the + * error text — as a usage limit, a rate limit, or neither (null). + */ +export function classifyUsageLimit( + input: unknown, + opts: ClassifyUsageLimitOptions = {}, +): UsageLimitClassification | null { + const signals: Signals = { texts: [] }; + collect(input, signals, 0); + const text = normalize(signals.texts.join("\n")); + const now = opts.now ?? Date.now(); + + let kind: UsageLimitKind | undefined; + let provider: string | undefined; + const usage = USAGE_RULES.find((r) => r.test(text)); + if (usage) { + kind = "usage_limit"; + provider = usage.provider; + } else if (prefixRule(text)) { + kind = "usage_limit"; + provider = /\bcodex\b/i.test(text) ? "codex" : undefined; + } else if (/You exceeded your current quota/i.test(text)) { + // OpenAI's billing quota and Gemini's free-tier per-minute quota share + // this sentence; Gemini's says "Please retry in 41s". A short delay is a + // throttle, anything else (or none) a spent allowance. + const delay = signals.retryDelayMs ?? relativeDelayMs(text); + kind = delay !== undefined && delay <= RETRYABLE_DELAY_MS && !/\bdaily\b|PerDay/i.test(text) ? "rate_limit" : "usage_limit"; + } else { + const rate = RATE_RULES.find((r) => r.test(text)); + if (rate) { + kind = "rate_limit"; + provider = rate.provider; + } else if (signals.status !== undefined) { + kind = "rate_limit"; + } + } + if (!kind) return null; + + // A "Quota exceeded for quota metric … per day" Google error is a daily allowance. + if (kind === "rate_limit" && /Quota exceeded for quota metric/i.test(text) && /per ?day|PerDay|daily/i.test(text)) { + kind = "usage_limit"; + } + + const resetAt = parseResetAt(text, now, signals.retryDelayMs); + const resolvedProvider = opts.provider ?? provider; + return { + kind, + ...(resetAt !== undefined ? { resetAt } : {}), + ...(resolvedProvider ? { provider: resolvedProvider } : {}), + }; +} + +// ── Reset time ──────────────────────────────────────────────────────────────── + +const UNIT_MS: Record = { + ms: 1, millisecond: 1, milliseconds: 1, + s: 1000, sec: 1000, secs: 1000, second: 1000, seconds: 1000, + m: 60_000, min: 60_000, mins: 60_000, minute: 60_000, minutes: 60_000, + h: 3600_000, hr: 3600_000, hrs: 3600_000, hour: 3600_000, hours: 3600_000, + d: 86_400_000, day: 86_400_000, days: 86_400_000, +}; +const DURATION_PART = /(\d+(?:\.\d+)?)\s*(milliseconds?|ms|seconds?|secs?|s|minutes?|mins?|m|hours?|hrs?|h|days?|d)\b/gi; + +/** A delay the text states relative to now: "try again in 1.5s", "resets in 2h 30m", "Retry-After: 30". */ +function relativeDelayMs(text: string): number | undefined { + const lead = /(?:resets?|try again|retry|available again)\s+(?:in|after)\s+((?:\d+(?:\.\d+)?\s*[a-z]+[\s,]*(?:and\s+)?){1,4})/i.exec(text); + if (lead?.[1]) { + let total = 0; + let matched = false; + for (const m of lead[1].matchAll(DURATION_PART)) { + const unit = UNIT_MS[m[2]!.toLowerCase()]; + if (unit === undefined) continue; + total += Number(m[1]) * unit; + matched = true; + } + if (matched && total > 0) return Math.round(total); + } + const retryAfter = /\bRetry-After:\s*(\d+)\b/i.exec(text); + if (retryAfter) return Number(retryAfter[1]) * 1000; + // Google RetryInfo: "retryDelay": "41s". + const retryInfo = /"retryDelay"\s*:\s*"(\d+(?:\.\d+)?)(ms|s)"/.exec(text); + if (retryInfo) return Number(retryInfo[1]) * (retryInfo[2] === "ms" ? 1 : 1000); + return undefined; +} + +const MONTHS: Record = { + jan: 0, feb: 1, mar: 2, apr: 3, may: 4, jun: 5, jul: 6, aug: 7, sep: 8, oct: 9, nov: 10, dec: 11, +}; + +/** + * A wall-clock reset: the Claude CLI's "resets 3pm (America/Los_Angeles)" / + * "resets Oct 28, 9am (UTC)" / "resets Oct 28, 2027, 9:30am" (its `Yc` + * formatter: en-US time, lower-cased am/pm, the zone in parentheses) and + * Codex's "Try again at 3:30 PM." / "try again at Oct 28th, 2026 3:30 PM." + * (codex-rs `%-I:%M %p` / `%b %-d, %Y %-I:%M %p`, local time). + */ +const CLOCK = + /(?:resets?(?:\s+at)?|try again at)\s+(?:(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(?:(\d{4}),?\s+)?)?(\d{1,2})(?::(\d{2}))?\s*([ap])\.?\s?m\.?(?:\s*\(([^)]+)\))?/i; + +function clockResetAt(text: string, now: number): number | undefined { + const m = CLOCK.exec(text); + if (!m) return undefined; + const [, mon, day, year, hourText, minuteText, ampm, zone] = m; + let hour = Number(hourText); + if (hour < 1 || hour > 12) return undefined; + if (ampm!.toLowerCase() === "p" && hour !== 12) hour += 12; + if (ampm!.toLowerCase() === "a" && hour === 12) hour = 0; + const minute = minuteText ? Number(minuteText) : 0; + if (minute > 59) return undefined; + const tz = zone && isTimeZone(zone.trim()) ? zone.trim() : undefined; + const today = calendarDate(now, tz); + + if (mon) { + const month = MONTHS[mon.toLowerCase()]!; + const date = Number(day); + let y = year ? Number(year) : today.year; + let at = wallToEpoch(y, month, date, hour, minute, tz); + // "Oct 28" with no year, seen in early January, is next October's. + if (!year && at < now - 86_400_000) at = wallToEpoch(++y, month, date, hour, minute, tz); + return at; + } + let at = wallToEpoch(today.year, today.month, today.day, hour, minute, tz); + // A bare time already passed today means tomorrow (the CLI prints a date only beyond 24h). + if (at <= now - 60_000) at = wallToEpoch(today.year, today.month, today.day + 1, hour, minute, tz); + return at; +} + +function isTimeZone(zone: string): boolean { + try { + new Intl.DateTimeFormat("en-US", { timeZone: zone }); + return true; + } catch { + return false; + } +} + +/** Calendar parts of `instant` in `tz` (the host zone when undefined). */ +function calendarDate(instant: number, tz: string | undefined): { year: number; month: number; day: number } { + if (!tz) { + const d = new Date(instant); + return { year: d.getFullYear(), month: d.getMonth(), day: d.getDate() }; + } + const p = zonedParts(instant, tz); + return { year: p.year, month: p.month, day: p.day }; +} + +function zonedParts(instant: number, tz: string) { + const parts = new Intl.DateTimeFormat("en-US", { + timeZone: tz, + hourCycle: "h23", + year: "numeric", + month: "2-digit", + day: "2-digit", + hour: "2-digit", + minute: "2-digit", + second: "2-digit", + }).formatToParts(new Date(instant)); + const get = (type: string) => Number(parts.find((x) => x.type === type)?.value ?? 0); + return { year: get("year"), month: get("month") - 1, day: get("day"), hour: get("hour") % 24, minute: get("minute"), second: get("second") }; +} + +/** Epoch ms of a wall-clock time in `tz` (host zone when undefined). Day overflow normalizes. */ +function wallToEpoch(year: number, month: number, day: number, hour: number, minute: number, tz: string | undefined): number { + if (!tz) return new Date(year, month, day, hour, minute).getTime(); + const asUtc = Date.UTC(year, month, day, hour, minute); + const offset = (instant: number) => { + const p = zonedParts(instant, tz); + return Date.UTC(p.year, p.month, p.day, p.hour, p.minute, p.second) - Math.floor(instant / 1000) * 1000; + }; + // Two passes settle the offset across a DST boundary. + let at = asUtc - offset(asUtc); + const second = asUtc - offset(at); + if (second !== at) at = second; + return at; +} + +function parseResetAt(text: string, now: number, retryDelayMs: number | undefined): number | undefined { + const candidates: Array = []; + // Claude's legacy `Claude AI usage limit reached|1730000000`. + const legacy = /usage limit reached\|(\d{9,13})/i.exec(text); + if (legacy) candidates.push(toEpochMs(Number(legacy[1]))); + // A structured reset in a JSON payload: codex `resets_at`, `resetsAt`. + const field = /"?(?:resets?_?[aA]t)"?\s*[:=]\s*(\d{9,13})\b/.exec(text); + if (field) candidates.push(toEpochMs(Number(field[1]))); + if (retryDelayMs !== undefined) candidates.push(now + retryDelayMs); + const delay = relativeDelayMs(text); + if (delay !== undefined) candidates.push(now + delay); + candidates.push(clockResetAt(text, now)); + for (const at of candidates) { + if (at !== undefined && Number.isFinite(at) && at > now - 60_000 && at <= now + MAX_RESET_HORIZON_MS) return at; + } + return undefined; +} + +/** Seconds or milliseconds since the epoch → milliseconds. */ +function toEpochMs(n: number): number { + return n < 1e12 ? n * 1000 : n; +} diff --git a/src/daemon/session.ts b/src/daemon/session.ts index 72f20cc9..66c88a86 100644 --- a/src/daemon/session.ts +++ b/src/daemon/session.ts @@ -20,6 +20,8 @@ import { type HistorySeedResult, } from "./providers/canonical.js"; import { targetContextWindow, seedBudgetChars, isPromptTooLongError } from "./providers/context-windows.js"; +import { classifyUsageLimit, type UsageLimitKind } from "./providers/usage-limit.js"; +import { extraEnvViolations } from "./providers/env.js"; import { CONDUCTOR_SYSTEM_PROMPT_APPEND, isFleetSendTool } from "./fleet.js"; import type { McpSdkServerConfigWithInstance } from "@anthropic-ai/claude-agent-sdk"; import { @@ -210,6 +212,14 @@ const CANCEL_REASON: Record block. The * digest is sub-agent output, which tool content can steer, so it must not be @@ -285,6 +295,23 @@ export interface AttachedClient { capabilities?: readonly string[]; } +/** + * A failed turn the backend blamed on a usage or rate limit (classifyUsageLimit) + * rather than on the work — the infrastructure error a loop controller records + * without spending an attempt (loop design §12.11). Epoch milliseconds. + */ +export interface TurnInfraError { + kind: UsageLimitKind; + /** When the limit is expected to lift, if the backend said. */ + resetAt?: number; + /** The backend's error text, as the turn reported it. */ + message: string; + /** The backend the turn ran on (the session's provider id at the time). */ + provider: string; + /** When the failure was recorded. */ + at: number; +} + /** Where a context window was observed; the manager scopes what it remembers * to it (see SessionCreateOptions.onModelLimits). */ export interface WindowScope { @@ -455,6 +482,30 @@ export interface SessionCreateOptions { * approval — which the dispatcher detects as a wedge. */ initialMode?: { mode: SessionMode; maxTurns?: number }; + /** + * What happens when the autonomous tool budget runs out (loop design §6). + * + * - `"park"` (default, today's behavior): the mode reverts to guarded and + * the next write/exec call waits for approval — the dispatcher sees the + * session wedge at waiting_approval. + * - `"end_attempt"`: for an unattended worker whose controller owns the + * next step. The call that finds the budget spent is DENIED with a clear + * message, the turn is interrupted, the session settles idle (never + * waiting_approval for that call), the mode reverts to guarded, and + * `budgetExhausted` reads true until a new autonomous budget is set. + */ + onBudgetExhausted?: "park" | "end_attempt"; + /** + * Extra environment for the backend's child process — every process-backed + * provider (Claude Code CLI, codex, gemini-cli, pi, qwen-code). Merged after + * codeoid builds the backend's env, but a key in codeoid's credential / + * account-isolation denylist (`extraEnvViolations`, providers/env.ts) is + * refused: the constructor throws, so a misconfigured caller learns at once. + * No effect on in-process backends (openai, gemini, llama.cpp, endpoints). + * Held in memory only — not persisted, so a session restored after a daemon + * restart runs without it unless its owner supplies it again. + */ + extraEnv?: Readonly>; /** * Native spend ceiling for this session's backend query (TurnOpts.maxBudgetUsd). * Routines set it on a fresh run session, whose query lifetime is the run. @@ -995,6 +1046,24 @@ export class Session { #collaborationCost?: () => CollaborationCost | undefined; #workerShape?: "ship" | "scout"; #turnsRemaining: number | undefined = undefined; + /** See SessionCreateOptions.onBudgetExhausted. */ + #onBudgetExhausted: "park" | "end_attempt" = "park"; + /** end_attempt only: the autonomous budget ran out; reset by a new autonomous budget (setMode). */ + #budgetExhausted = false; + /** end_attempt only: the turn the budget ran out in is being ended — deny every further call in it. */ + #endingAttempt = false; + /** The ending turn's interrupt has been scheduled (once per ending). */ + #endAttemptInterruptQueued = false; + /** See SessionCreateOptions.extraEnv (validated in the constructor). */ + #extraEnv: Readonly> | undefined; + /** Why the last finished turn failed, when a usage or rate limit was the cause; null after a success. */ + #lastTurnInfraError: TurnInfraError | null = null; + /** + * Claude's `rate_limit_event` reported `rejected` during the current turn, + * with the reset instant it carried — exact where the error text says only + * "resets 3pm". Cleared when a turn starts. + */ + #rateLimitRejection: { resetAt?: number } | null = null; // Cumulative token + cost totals, aggregated from SDK `result` messages // (one per turn). Broadcast via session.info_update so StatusBar-style @@ -1219,6 +1288,18 @@ export class Session { this.#workerShape = opts.workerShape; this.#collaborationCost = opts.collaborationCost; this.#maxBudgetUsd = opts.maxBudgetUsd; + this.#onBudgetExhausted = opts.onBudgetExhausted ?? "park"; + if (opts.extraEnv) { + // Before any side effect: a refused key must fail the create, not run a + // backend with half the env the caller asked for. + const refused = extraEnvViolations(opts.extraEnv); + if (refused.length > 0) { + throw new Error( + `extraEnv may not set ${refused.join(", ")}: codeoid owns credential, account and process-loading variables for the backend`, + ); + } + this.#extraEnv = Object.freeze({ ...opts.extraEnv }); + } if (opts.initialMode) { this.#mode = opts.initialMode.mode; this.#turnsRemaining = @@ -1493,6 +1574,7 @@ export class Session { resolveClaudeAccount: () => this.#claudeAccounts?.resolve(this.#accountChoices.claude ?? null) ?? null, // Every other backend: this session's account for it (accounts.ts). resolveAccount: () => this.#resolveAccount(factory.id), + ...(this.#extraEnv ? { extraEnv: this.#extraEnv } : {}), }); } @@ -1945,6 +2027,20 @@ export class Session { * the failure — a pipeline phase — needs this, not lastAssistantText. */ get lastTurnError(): string | null { return this.#lastTurnError; } + /** + * The most recent finished turn's failure, when the backend blamed a usage or + * rate limit (with its reset time when it gave one); null when that turn + * failed for another reason, and cleared by the next successful turn. A loop + * controller reads it to record an infrastructure error that spends no + * attempt. Retry behavior is unchanged — this only observes. + */ + get lastTurnInfraError(): TurnInfraError | null { return this.#lastTurnInfraError; } + /** + * True once the autonomous tool budget ran out on a session created with + * `onBudgetExhausted: "end_attempt"` — the attempt was ended, not parked. + * Stays true until a new autonomous budget is set (setMode). + */ + get budgetExhausted(): boolean { return this.#budgetExhausted; } /** * Role of the MOST RECENT canonical turn (undefined before any turn). Right * after `send()` the last turn is our USER prompt/nudge; a transient query-loop @@ -2435,6 +2531,8 @@ export class Session { if (this.#mode === mode && this.#turnsRemaining === maxTurns) return; this.#mode = mode; this.#turnsRemaining = mode === "autonomous" ? maxTurns : undefined; + // A fresh autonomous budget is a new attempt's: the spent flag is the old one's. + if (mode === "autonomous") this.#budgetExhausted = false; if (sender) { this.#store.audit(sender.sub, "session.set_mode", this.id, `mode=${mode}`); } @@ -2993,6 +3091,56 @@ export class Session { return this.#preparingSends > 0; } + /** Per-turn loop controls, reset when a turn starts. */ + #resetTurnControls(): void { + this.#endingAttempt = false; + this.#endAttemptInterruptQueued = false; + this.#rateLimitRejection = null; + } + + /** + * A turn failed: record whether a usage or rate limit was the cause + * (lastTurnInfraError). `sources` are what the turn reported, most + * authoritative first — the error, then (when this turn wrote it) the + * assistant text the Claude CLI puts its limit message in. Any failure + * overwrites the previous record, so it always describes the latest turn. + * Observation only: nothing here retries or changes status. + */ + #recordTurnFailure(sources: readonly unknown[]): void { + const provider = this.#provider.id; + for (const source of sources) { + if (source == null || source === "") continue; + const hit = classifyUsageLimit(source, { provider }); + if (!hit) continue; + // The text often says only "resets 3pm"; Claude's rate_limit_event in + // the same turn carries the exact instant. + const resetAt = hit.resetAt ?? this.#rateLimitRejection?.resetAt; + this.#lastTurnInfraError = { + kind: hit.kind, + ...(resetAt !== undefined ? { resetAt } : {}), + message: failureText(source), + provider, + at: Date.now(), + }; + return; + } + // No text said so, but the backend reported this turn's request rejected + // by a plan limit (Claude's rate_limit_event, status "rejected"). + const rejection = this.#rateLimitRejection; + if (rejection) { + const first = sources.find((s) => s != null && s !== ""); + this.#lastTurnInfraError = { + kind: "usage_limit", + ...(rejection.resetAt !== undefined ? { resetAt: rejection.resetAt } : {}), + message: first === undefined ? "usage limit reached" : failureText(first), + provider, + at: Date.now(), + }; + return; + } + this.#lastTurnInfraError = null; + } + /** * Surface a send-path failure as a visible system message instead of * swallowing it. Called by SessionManager#send when the async send() @@ -3002,6 +3150,7 @@ export class Session { */ reportSendFailure(err: unknown): void { const emsg = err instanceof Error ? err.message : String(err); + this.#recordTurnFailure([err]); console.error( `[codeoid/session ${this.id}] send failed after ack: ${ err instanceof Error ? (err.stack ?? emsg) : emsg @@ -3031,6 +3180,7 @@ export class Session { // A new turn is starting — clear any interrupt from the previous one. this.#turnInterrupted = false; this.#lastTurnError = null; + this.#resetTurnControls(); // Any inbound message invalidates a repeat run: the guard only claims // "N identical calls with nothing else happening", and this is something // else happening. Applies to system principals too (a background-task @@ -3572,6 +3722,7 @@ export class Session { void this.#checkpointTurn(turnId, 0); this.#turnInterrupted = false; this.#lastTurnError = null; + this.#resetTurnControls(); // A turn with no prompt reads as the agent talking to itself; say why. const note = this.#makeMessage( "info", @@ -5727,6 +5878,20 @@ export class Session { return false; } + /** + * True when `toolName` would spend autonomous budget that is already spent, + * on a session that ends the attempt rather than parking + * (`onBudgetExhausted: "end_attempt"`). Side-effect free — shared by the + * gate and the tool_start peek so the card never offers an approval the + * gate will not wait for. The set of budgeted tools is #shouldAutoApprove's. + */ + #budgetSpentFor(toolName: string): boolean { + if (this.#onBudgetExhausted !== "end_attempt" || this.#mode !== "autonomous") return false; + if (this.#turnsRemaining === undefined || this.#turnsRemaining > 0) return false; + if (isFleetSendTool(toolName) || isElicitationTool(toolName)) return false; + return !isSafeTool(toolName) && !this.#isReadonlyMcpTool(toolName); + } + /** * Side-effect-free read of auto-approve state. Used ONLY in the tool_start * event handler to determine the initial UI phase (executing vs @@ -5853,6 +6018,59 @@ export class Session { } } + /** + * Deny a tool call because the autonomous budget is spent on an + * `end_attempt` session, and end the turn it belongs to (loop design §6: + * the attempt ends as budget_exhausted and control returns to the + * controller, rather than the session parking at waiting_approval). + * + * The interrupt is scheduled, not awaited: the deny must reach the backend + * first, and interrupt() awaits the backend, which is waiting on this gate. + * Only the first denial interrupts; later calls in the draining turn are + * simply refused. + */ + #endAttemptOnBudget(approvalId: string, toolName: string, sender: AuthContext): { behavior: "deny"; message: string } { + if (!this.#endingAttempt) { + // Same reversion as park (reads frictionless, writes ask again if a + // person takes over); flagged after setMode, which clears the flag only + // for a NEW autonomous budget. + this.setMode("guarded"); + this.#budgetExhausted = true; + this.#endingAttempt = true; + } + const reason = "The autonomous tool budget for this attempt is spent, so the attempt has ended. Do not retry; the controller decides what happens next."; + this.#resolveToolCallMessage(approvalId, { phase: "cancelled", reason: "denied", message: reason }); + this.#approvalPatchKeys.delete(approvalId); + this.#store.audit(sender.sub, "session.budget_deny", this.id, `tool=${toolName}`); + if (!this.#endAttemptInterruptQueued) { + this.#endAttemptInterruptQueued = true; + const infoMsg = this.#makeMessage( + "info", + `⏹ Autonomous tool budget spent — ${toolName} was refused and this attempt has ended.`, + SYSTEM_IDENTITY, + undefined, + undefined, + { event: "budget.end_attempt", tool: toolName }, + ); + this.#persistAndBuffer(infoMsg); + this.#broadcastRaw(infoMsg); + const system: AuthContext = { + sub: "system:budget", + scopes: [], + delegationDepth: 0, + accountId: this.accountId, + projectId: this.projectId, + }; + setTimeout(() => { + if (this.#destroyed) return; + this.interrupt(system).catch((e) => { + console.error(`[codeoid/session ${this.id}] ending the attempt on a spent budget failed: ${e instanceof Error ? e.message : String(e)}`); + }); + }, 0); + } + return { behavior: "deny", message: reason }; + } + #makeCanUseToolFn(sender: AuthContext): ToolApprovalFn { // The backend this gate is handed to — the one that will ASK. Captured // here, not at call time, so the rewrite check below refers to it even if @@ -5869,6 +6087,10 @@ export class Session { // approvable card for a call the gate already resolved. if (!this.#turnActive()) await this.#backgroundEventChain; + // The attempt already ended on a spent budget (end_attempt): nothing + // else runs in it — not even a read — while the interrupt lands. + if (this.#endingAttempt) return this.#endAttemptOnBudget(approvalId, toolName, sender); + // Hook gate — the policy layer, run BEFORE the approval gate. A hook // block is a policy deny that never prompts the user (and never burns // the autonomous budget — #shouldAutoApprove runs after this); an @@ -6009,6 +6231,9 @@ export class Session { // it mid-turn and applies its env, after the turn-start check // (assertClaudeAccountUsable) — so it never runs unasked, in any mode. const settingsWrite = this.#writesClaudeSettingsOnNamedAccount(toolName, effectiveInput); + // end_attempt: a call the spent budget would have parked is denied and + // ends the attempt instead — whatever else would have asked about it. + if (this.#budgetSpentFor(toolName)) return this.#endAttemptOnBudget(approvalId, toolName, sender); const autoApprove = secret?.action !== "ask" && !settingsWrite && this.#shouldAutoApprove(toolName); if (autoApprove) { @@ -6264,6 +6489,7 @@ export class Session { console.error(`[codeoid/session ${this.id}] provider event consumer failed:`, err); // Before the status change: it wakes a pipeline phase waiter that reads it. this.#lastTurnError = emsg; + this.#recordTurnFailure([err]); this.#setStatus("error"); const errorMsg = this.#makeMessage("system", `Error: ${emsg}`, SYSTEM_IDENTITY, undefined, undefined, { event: "agent_error", errorCode: "agent_error" }); this.#persistAndBuffer(errorMsg); @@ -6326,12 +6552,7 @@ export class Session { * run). Idempotent and run-scoped: if `run` is no longer the active run, the * turn already ended and we no-op. * - * Crucially this does NOT route through #teardownProvider (which awaits the - * session event-consumer task — i.e. potentially itself). It nulls the run - * slots up front, surfaces a clear message, resets status to idle, and hard - * tears down the PROVIDER (abort → reap the hung subprocess). The next send() - * recreates a fresh query loop. The abandoned consumer (if any) unblocks when - * teardown closes the turn queue and its finally no-ops via the run guard. + * The stop itself is #hardStop (shared with terminateBackend). */ async #recoverStalledRun( run: TurnRun, @@ -6344,11 +6565,52 @@ export class Session { `[codeoid/session ${this.id}] turn stalled — no provider events for ${stallMs}ms; force-recovering`, ); + await this.#hardStop("stall recovery", { + text: opts?.continuingCurrentSend + ? `⚠️ Previous turn timed out — no activity for ${Math.round(stallMs / 1000)}s. The session was reset and your latest message is being retried in a fresh turn.` + : `⚠️ Turn timed out — no activity for ${Math.round(stallMs / 1000)}s. The session was reset; send your message again to continue.`, + meta: { event: "turn_stalled", errorCode: "turn_stalled" }, + }); + } + + /** + * Stop this session's backend outright — the escalation after an interrupt + * that did not settle the session (supervisor.ts: an attempt's wall time + * ran out and the grace period passed). Whatever is in flight — a tool + * without its own timeout, a wedged turn, background agents — is ended by + * tearing the backend down (its child process is killed), exactly as stall + * recovery does. The session survives and settles idle; the next send starts + * a fresh backend and resumes the conversation. Idempotent. + */ + async terminateBackend(sender: AuthContext, reason: string): Promise { + this.#store.audit(sender.sub, "session.terminate_backend", this.id, reason); + // Stop sends still preparing their turn too, as interrupt() does. + this.#turnInterrupted = true; + this.#stopGen++; + this.#cancelAllUiRequests("interrupted"); + this.#earlyApprovals.clear(); + this.#approvalPatchKeys.clear(); + await this.#hardStop("backend terminated", { + text: `⏹ Backend stopped by ${sender.sub}: ${reason}. Send a new message to continue.`, + meta: { event: "session.backend_terminated", sub: sender.sub }, + }); + } + + /** + * Shared by stall recovery and terminateBackend. Does NOT route through + * #teardownProvider (which awaits the session event-consumer task — i.e. + * potentially the caller). It nulls the run slots up front, surfaces a clear + * message, resets status to idle, and hard tears down the PROVIDER (abort → + * reap the subprocess). The next send() recreates a fresh query loop. The + * abandoned consumer (if any) unblocks when teardown closes the turn queue + * and its finally no-ops via the run guard. + */ + async #hardStop(label: string, notice: { text: string; meta: Record }): Promise { // Stop any spinning UI and release waiters BEFORE we tear down. The // teardown below kills the CLI and every background agent with it, so // nothing is kept: the whole background set is dead. this.#clearBackgroundTasks(); - this.#reconcileWork("stall recovery", { keepBackground: false }); + this.#reconcileWork(label, { keepBackground: false }); this.#flushActiveAssistant(); this.#finalizeActiveThinking(); this.#chunker?.onTurnEnd(); @@ -6359,16 +6621,7 @@ export class Session { this.#activeRun = null; this.#eventConsumerTask = null; - const msg = this.#makeMessage( - "system", - opts?.continuingCurrentSend - ? `⚠️ Previous turn timed out — no activity for ${Math.round(stallMs / 1000)}s. The session was reset and your latest message is being retried in a fresh turn.` - : `⚠️ Turn timed out — no activity for ${Math.round(stallMs / 1000)}s. The session was reset; send your message again to continue.`, - SYSTEM_IDENTITY, - undefined, - undefined, - { event: "turn_stalled", errorCode: "turn_stalled" }, - ); + const msg = this.#makeMessage("system", notice.text, SYSTEM_IDENTITY, undefined, undefined, notice.meta); this.#persistAndBuffer(msg); this.#broadcastRaw(msg); this.#setStatus("idle"); @@ -6380,7 +6633,7 @@ export class Session { await this.#provider.teardown(); } catch (e) { console.error( - `[codeoid/session ${this.id}] provider teardown during stall recovery failed: ${e instanceof Error ? e.message : String(e)}`, + `[codeoid/session ${this.id}] provider teardown during ${label} failed: ${e instanceof Error ? e.message : String(e)}`, ); } } @@ -6515,6 +6768,8 @@ export class Session { const autoApprove = secretPathVerdict(event.name, event.input, { workdir: this.workdir, configDir: getConfigDir() }) === null && this.#peekAutoApprove(event.name); + // end_attempt: the gate will refuse this call outright — no approval card. + const budgetDenied = this.#endingAttempt || this.#budgetSpentFor(event.name); const toolMsg = this.#makeMessage( "tool_call", `${event.name}(${Object.keys(event.input).join(", ")})`, @@ -6524,7 +6779,9 @@ export class Session { toolId: event.toolId, name: event.name, input: event.input, - state: autoApprove + state: budgetDenied + ? { phase: "cancelled", reason: "denied", message: "autonomous tool budget spent" } + : autoApprove ? ({ phase: "executing", input: event.input } as unknown as ToolState) : ({ phase: "waiting_confirmation", @@ -6548,7 +6805,7 @@ export class Session { this.#persistAndBuffer(toolMsg); this.#broadcastRaw(toolMsg); this.#maybeInjectRepeatReminder(event.name, event.input, event.sdkAgentId); - if (!autoApprove) this.#setStatus("waiting_approval"); + if (!autoApprove && !budgetDenied) this.#setStatus("waiting_approval"); break; } @@ -6737,6 +6994,14 @@ export class Session { break; } + case "rate_limit": { + // Kept for #recordTurnFailure: if this turn fails, the reset is exact. + // A later "allowed" in the same turn means the rejection was retried past. + this.#rateLimitRejection = + event.status === "rejected" ? (event.resetsAt !== undefined ? { resetAt: event.resetsAt } : {}) : null; + break; + } + case "custom_message": { // Provider-authored standalone message (extension output, status // cards, rich widgets). Not part of the LLM conversation — the @@ -6790,6 +7055,13 @@ export class Session { break; } this.#lastTurnError = errText; + // Before the status change below, like lastTurnError: an error + // waiter reads it. The CLI puts a limit message in the turn's + // assistant text (only if THIS turn committed it, as above). + // Short only: the CLI's limit notice is the whole reply, and a long + // reply that happens to discuss rate limits is the model's work. + const notice = last?.role === "assistant" && last.content.length <= 500 ? last.content : undefined; + this.#recordTurnFailure([event.result.errorMessage, notice, errText]); const hint = tooLong ? " — this session's context no longer fits the model. Send /rotate to continue in a fresh context (memory recall keeps the history reachable)." : ""; @@ -6801,6 +7073,7 @@ export class Session { // The seed made it into a successful turn — it's spent. this.#pendingSeed = null; this.#promptTooLong = false; + this.#lastTurnInfraError = null; if (this.#status !== "error") this.#setStatus("idle"); } break; @@ -6833,6 +7106,7 @@ export class Session { case "error": { console.error(`[codeoid/session ${this.id}] provider error:`, event.message); this.#lastTurnError = event.message; // before the status change (see above) + this.#recordTurnFailure([event.message]); this.#setStatus("error"); const errorMsg = this.#makeMessage("system", `Error: ${event.message}`, SYSTEM_IDENTITY, undefined, undefined, { event: "agent_error" }); this.#persistAndBuffer(errorMsg); diff --git a/src/daemon/supervisor.ts b/src/daemon/supervisor.ts new file mode 100644 index 00000000..13719f68 --- /dev/null +++ b/src/daemon/supervisor.ts @@ -0,0 +1,155 @@ +/** + * Wall-time deadlines for a session's attempt (loop design §8, §12.11 "Hung work"). + * + * The stall watchdog pauses while a tool runs, and a dispatch lease keeps + * renewing while the session lives, so a tool without its own timeout can hold + * a task forever. A deadline is the bound that holds regardless: it counts + * wall time from `supervise()` and looks at nothing but the clock. + * + * On expiry it interrupts the session (the Esc path: stops the turn, reaps the + * running tool where the backend can). If the session has not settled within + * `graceMs` — the interrupt itself can wedge on a backend that stopped + * answering — it terminates the backend (Session.terminateBackend: the + * provider is torn down and its child process killed, as stall recovery + * does). `expired` then resolves true. + * + * The deadline owns exactly one timer at a time and clears it on every exit + * path; `cancel()` before expiry resolves `expired` false and leaves nothing + * scheduled. + */ + +import type { AuthContext, SessionStatus } from "../protocol/types.js"; + +/** The slice of Session a deadline drives (narrow, so tests can fake it). */ +export interface SupervisedSession { + readonly id: string; + readonly accountId: string; + readonly projectId: string; + readonly status: SessionStatus; + /** A queued send is about to start a turn — not settled even when idle. */ + readonly preparingTurn: boolean; + interrupt(sender: AuthContext): Promise; + terminateBackend(sender: AuthContext, reason: string): Promise; +} + +export interface DeadlineOptions { + /** Wall time the attempt may run, from now. */ + wallMs: number; + /** After the interrupt, how long the session gets to settle before its backend is killed. */ + graceMs: number; + /** Who the interrupt and termination are audited as. Default `system:supervisor` in the session's tenant. */ + sender?: AuthContext; + /** How often the grace period re-checks for a settled session. Default 100ms (bounded by graceMs). */ + pollMs?: number; + /** Observes each step (logging, tests). Exceptions are swallowed. */ + onEvent?: (event: DeadlineEvent) => void; +} + +export type DeadlineEvent = + | { type: "expired"; sessionId: string } + | { type: "settled"; sessionId: string } + | { type: "terminated"; sessionId: string }; + +export interface Deadline { + /** + * Stop supervising. Before expiry: nothing fires and `expired` resolves + * false. During the grace period: the kill escalation is abandoned (the + * interrupt has already been sent) and `expired` resolves true. Idempotent. + */ + cancel(): void; + /** true once the wall time ran out and enforcement finished; false when cancelled first. */ + readonly expired: Promise; +} + +/** A session is settled when no turn is running or about to start. */ +function settled(session: SupervisedSession): boolean { + return (session.status === "idle" || session.status === "error") && !session.preparingTurn; +} + +export function supervise(session: SupervisedSession, opts: DeadlineOptions): Deadline { + const { wallMs, graceMs } = opts; + if (!Number.isFinite(wallMs) || wallMs <= 0) throw new RangeError(`supervise: wallMs must be a positive number (got ${wallMs})`); + if (!Number.isFinite(graceMs) || graceMs < 0) throw new RangeError(`supervise: graceMs must be zero or more (got ${graceMs})`); + const pollMs = Math.max(1, Math.min(opts.pollMs ?? 100, graceMs || 1)); + const sender: AuthContext = opts.sender ?? { + sub: "system:supervisor", + scopes: [], + delegationDepth: 0, + accountId: session.accountId, + projectId: session.projectId, + }; + const note = (event: DeadlineEvent) => { + try { + opts.onEvent?.(event); + } catch { + // observability must not break enforcement + } + }; + + let timer: ReturnType | null = null; + let done = false; + let fired = false; + let resolveExpired!: (v: boolean) => void; + const expired = new Promise((r) => { + resolveExpired = r; + }); + + const arm = (fn: () => void, ms: number) => { + timer = setTimeout(() => { + timer = null; + fn(); + }, ms); + // A deadline must not keep a process alive on its own. + timer.unref?.(); + }; + const finish = (result: boolean) => { + if (done) return; + done = true; + if (timer) clearTimeout(timer); + timer = null; + resolveExpired(result); + }; + + const terminate = () => { + if (done) return; + note({ type: "terminated", sessionId: session.id }); + session + .terminateBackend(sender, `attempt wall time of ${Math.round(wallMs / 1000)}s ran out and it did not stop within ${Math.round(graceMs / 1000)}s`) + .catch((e) => { + console.error(`[codeoid/supervisor] terminating session ${session.id} failed: ${e instanceof Error ? e.message : String(e)}`); + }) + .finally(() => finish(true)); + }; + + const graceUntil = (deadline: number) => { + if (done) return; + if (settled(session)) { + note({ type: "settled", sessionId: session.id }); + finish(true); + return; + } + const left = deadline - Date.now(); + if (left <= 0) { + terminate(); + return; + } + arm(() => graceUntil(deadline), Math.min(pollMs, left)); + }; + + arm(() => { + if (done) return; + fired = true; + note({ type: "expired", sessionId: session.id }); + // Not awaited: interrupt() awaits the backend, which may be the thing + // that's hung. The grace clock starts now, whatever it does. + session.interrupt(sender).catch((e) => { + console.error(`[codeoid/supervisor] interrupting session ${session.id} failed: ${e instanceof Error ? e.message : String(e)}`); + }); + graceUntil(Date.now() + graceMs); + }, wallMs); + + return { + cancel: () => finish(fired), + expired, + }; +} diff --git a/src/tests/fixtures/fake-codex.ts b/src/tests/fixtures/fake-codex.ts index 797013d8..acc7cd50 100644 --- a/src/tests/fixtures/fake-codex.ts +++ b/src/tests/fixtures/fake-codex.ts @@ -329,6 +329,9 @@ async function runTurn(threadId: string, prompt: string): Promise { argv: process.argv, }); send({ method: "item/completed", params: { item: { id: "m1", type: "agentMessage", text } } }); + } else if (prompt.includes("echo-env")) { + // The environment the provider spawned us with (extraEnv wiring). + send({ method: "item/completed", params: { item: { id: "m1", type: "agentMessage", text: JSON.stringify(process.env) } } }); } else if (prompt.includes("echo-native")) { send({ method: "item/completed", params: { item: { id: "m1", type: "agentMessage", text: JSON.stringify(nativeCalls) } } }); } else if (prompt.includes("echo-prompt")) { diff --git a/src/tests/provider-claude.test.ts b/src/tests/provider-claude.test.ts index def6d7f5..4fff0a7a 100644 --- a/src/tests/provider-claude.test.ts +++ b/src/tests/provider-claude.test.ts @@ -1329,6 +1329,20 @@ describe("translateSDKMessage – system", () => { }); }); +describe("translateSDKMessage – rate_limit_event", () => { + it("emits the plan limit state with resetsAt converted from epoch seconds to ms", () => { + const events = collectEmits({ + type: "rate_limit_event", + rate_limit_info: { status: "rejected", resetsAt: 1_800_000_000, rateLimitType: "five_hour" }, + }); + expect(events).toEqual([{ type: "rate_limit", status: "rejected", resetsAt: 1_800_000_000_000, limitType: "five_hour" }]); + }); + + it("emits nothing for a status it doesn't know", () => { + expect(collectEmits({ type: "rate_limit_event", rate_limit_info: { status: "mystery" } })).toHaveLength(0); + }); +}); + describe("translateSDKMessage – user (tool_result)", () => { it("emits tool_complete for each tool_result block", () => { const events = collectEmits({ @@ -2041,9 +2055,11 @@ describe("ClaudeProvider – Claude accounts", () => { function makeAccountProvider( current: { account: ResolvedClaudeAccount | null }, onModels?: (models: ReadonlyArray) => void, + extraEnv?: Record, ): ClaudeProvider { return new ClaudeProvider({ ...(onModels ? { onModels } : {}), + ...(extraEnv ? { extraEnv } : {}), sessionId: "acct-session", initialBackingId: "acct-backing", workspaceId: "ws_test", @@ -2093,6 +2109,37 @@ describe("ClaudeProvider – Claude accounts", () => { } }); + it("extraEnv reaches the CLI's env last, but never overrides the account's credential, HOME or a Node preload", async () => { + const saved = process.env.ANTHROPIC_API_KEY; + process.env.ANTHROPIC_API_KEY = "daemon-key-would-win"; + try { + const current = { account: account({ name: "work", kind: "subscription", token: "sk-ant-oat01-WORK" }) }; + const provider = makeAccountProvider(current, undefined, { + LOOP_RUN_ID: "run-7", + DATABASE_URL: "postgres://test", + CLAUDE_CODE_OAUTH_TOKEN: "sk-ant-oat01-EVIL", + ANTHROPIC_API_KEY: "sk-evil", + CLAUDE_CONFIG_DIR: "/tmp/other-login", + HOME: "/tmp/evil-home", + NODE_OPTIONS: "--require /tmp/evil.js", + }); + turnOn(provider, "t1"); + const env = envOf(); + expect(env.LOOP_RUN_ID).toBe("run-7"); + expect(env.DATABASE_URL).toBe("postgres://test"); + expect(env.CLAUDE_CODE_OAUTH_TOKEN).toBe("sk-ant-oat01-WORK"); + expect(env.ANTHROPIC_API_KEY).toBeUndefined(); + expect(env.CLAUDE_CONFIG_DIR).not.toBe("/tmp/other-login"); + expect(env.HOME).toBe(process.env.HOME!); + expect(env.NODE_OPTIONS).not.toBe("--require /tmp/evil.js"); + openGate(); + await provider.teardown(); + } finally { + if (saved === undefined) delete process.env.ANTHROPIC_API_KEY; + else process.env.ANTHROPIC_API_KEY = saved; + } + }); + it("a Bedrock account: mapped families go to the CLI as aliases; its catalog never reaches the shared cache", async () => { const reported: unknown[] = []; const current = { diff --git a/src/tests/provider-codex.test.ts b/src/tests/provider-codex.test.ts index b7076c1d..e214d0d6 100644 --- a/src/tests/provider-codex.test.ts +++ b/src/tests/provider-codex.test.ts @@ -118,6 +118,40 @@ async function collect(run: TurnRun): Promise { } describe("CodexProvider over fake-codex", () => { + it("extraEnv reaches the spawned codex; credential, home and codeoid-owned keys keep codeoid's value", async () => { + const saved = process.env.OPENAI_API_KEY; + process.env.OPENAI_API_KEY = "sk-daemon"; + try { + const p = new CodexProvider({ + sessionId: "sess-env", + initialBackingId: "sess-env", + command: process.execPath, + argsPrefix: [FIXTURE], + store: {} as Store, + extraEnv: { + LOOP_RUN_ID: "run-9", + OPENAI_API_KEY: "sk-evil", + CODEX_HOME: "/tmp/other-login", + HOME: "/tmp/evil-home", + PATH: "/tmp/evil-bin", + CODEOID_MEMORY_TOKEN: "forged", + }, + }); + const events = await collect(p.runTurn(turnOpts("echo-env"))); + await p.teardown(); + const env = JSON.parse((events.find((e) => e.type === "text_done") as { content: string }).content) as Record; + expect(env.LOOP_RUN_ID).toBe("run-9"); + expect(env.OPENAI_API_KEY).toBe("sk-daemon"); + expect(env.CODEX_HOME).toBeUndefined(); + expect(env.HOME).toBe(process.env.HOME!); + expect(env.PATH).toBe(process.env.PATH!); + expect(env.CODEOID_MEMORY_TOKEN).toBeUndefined(); + } finally { + if (saved === undefined) delete process.env.OPENAI_API_KEY; + else process.env.OPENAI_API_KEY = saved; + } + }); + it("C1: text turn streams thinking + text and completes with usage", async () => { const p = makeProvider(); const events = await collect(p.runTurn(turnOpts("hello"))); diff --git a/src/tests/provider-env.test.ts b/src/tests/provider-env.test.ts index 53f924b8..b2af3973 100644 --- a/src/tests/provider-env.test.ts +++ b/src/tests/provider-env.test.ts @@ -5,7 +5,7 @@ */ import { describe, expect, it } from "bun:test"; -import { buildPiEnv, buildSubprocessEnv, withGatewayCredential } from "../daemon/providers/env.js"; +import { buildPiEnv, buildSubprocessEnv, extraEnvViolations, withExtraEnv, withGatewayCredential } from "../daemon/providers/env.js"; import { buildAgentEnv } from "../daemon/providers/claude/index.js"; const DAEMON_ENV: Record = { @@ -178,3 +178,45 @@ describe("withGatewayCredential (forge#111)", () => { expect(env.ANTHROPIC_AUTH_TOKEN).toBe("zid_sk_session"); }); }); + +describe("extraEnv denylist (SessionCreateOptions.extraEnv)", () => { + // Every variable codeoid sets or scrubs for account isolation, plus + // process-loading and request-routing ones. None may be overridden. + const DENIED = [ + "HOME", "USER", "LOGNAME", "PATH", "SHELL", "XDG_CONFIG_HOME", "XDG_DATA_HOME", + "NODE_OPTIONS", "NODE_PATH", "LD_PRELOAD", "LD_LIBRARY_PATH", "DYLD_INSERT_LIBRARIES", + "HTTPS_PROXY", "https_proxy", "NODE_EXTRA_CA_CERTS", "NODE_TLS_REJECT_UNAUTHORIZED", "SSL_CERT_FILE", + "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_BASE_URL", "ANTHROPIC_CUSTOM_HEADERS", + "CLAUDE_CODE_OAUTH_TOKEN", "CLAUDE_CONFIG_DIR", "CLAUDE_CODE_USE_BEDROCK", + "AWS_BEARER_TOKEN_BEDROCK", "AWS_PROFILE", "OTEL_EXPORTER_OTLP_ENDPOINT", + "CODEX_HOME", "OPENAI_API_KEY", "OPENAI_BASE_URL", + "GEMINI_API_KEY", "GEMINI_CLI_HOME", "GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_CLOUD_PROJECT", "CODE_ASSIST_ENDPOINT", + "QWEN_HOME", "DASHSCOPE_API_KEY", "BAILIAN_API_KEY", "PI_CODING_AGENT_DIR", + "CODEOID_API_KEY", "CODEOID_AGENT_ENV_ALLOW", "CODEOID_MEMORY_TOKEN", "ZEROID_URL", "TELEGRAM_BOT_TOKEN", + "GROQ_API_KEY", "MISTRAL_API_TOKEN", "SOME_AUTH_TOKEN", "VENDOR_ACCESS_TOKEN", "vendor_api_key", + ]; + const ALLOWED = ["LOOP_RUN_ID", "DATABASE_URL", "CI", "NODE_ENV", "RUST_LOG", "GITHUB_TOKEN", "TZ", "LANG", "MY_FLAG"]; + + it("refuses every denylisted key and none of the ordinary ones", () => { + const extra = Object.fromEntries([...DENIED, ...ALLOWED].map((k) => [k, "x"])); + expect(extraEnvViolations(extra).sort()).toEqual([...DENIED].sort()); + }); + + it("refuses malformed names and values", () => { + expect(extraEnvViolations({ "BAD-NAME": "x", "1X": "x", OK: "x", NUL: "a\0b", NUM: 1 as unknown as string })).toEqual([ + "BAD-NAME", "1X", "NUL", "NUM", + ]); + }); + + it("withExtraEnv merges allowed keys over the built env and keeps codeoid's value for refused ones", () => { + const built = { PATH: "/usr/bin", HOME: "/home/u", ANTHROPIC_API_KEY: "ant-key", TZ: "UTC" }; + const merged = withExtraEnv(built, { PATH: "/evil", HOME: "/evil", ANTHROPIC_API_KEY: "evil", TZ: "Asia/Kolkata", LOOP_RUN_ID: "r1" }); + expect(merged).toEqual({ PATH: "/usr/bin", HOME: "/home/u", ANTHROPIC_API_KEY: "ant-key", TZ: "Asia/Kolkata", LOOP_RUN_ID: "r1" }); + expect(built.TZ).toBe("UTC"); // pure + expect(withExtraEnv(built, undefined)).toBe(built); + }); + + it("a refused key absent from the built env is not added either", () => { + expect(withExtraEnv({ PATH: "/usr/bin" }, { OPENAI_API_KEY: "evil" })).toEqual({ PATH: "/usr/bin" }); + }); +}); diff --git a/src/tests/provider-qwen-accounts.test.ts b/src/tests/provider-qwen-accounts.test.ts index a39f7b99..923b7e05 100644 --- a/src/tests/provider-qwen-accounts.test.ts +++ b/src/tests/provider-qwen-accounts.test.ts @@ -105,4 +105,23 @@ describe("QwenProvider on a named account", () => { void run; await provider.teardown(); }); + + test("extraEnv rides on top, but can't override a daemon key or move the Qwen home", async () => { + const before = queryCalls.length; + const provider = new QwenProvider({ + sessionId: "sess-q2", + initialBackingId: "22222222-2222-4222-8222-222222222222", + workspaceId: "ws", + store: { audit: () => {} } as never, + extraEnv: { LOOP_RUN_ID: "run-q", DASHSCOPE_API_KEY: "evil", QWEN_HOME: "/tmp/evil", OPENAI_ORG_ID: "org-evil" }, + }); + void provider.runTurn({ history: [], userMessage: "hi", workdir: dir, canUseTool: async () => ({ behavior: "allow" as const }) }); + await Bun.sleep(100); + const env = queryCalls[before]!.env!; + expect(env.LOOP_RUN_ID).toBe("run-q"); + expect(env.DASHSCOPE_API_KEY).toBe("daemon-dashscope"); + expect(env.QWEN_HOME).toBe(process.env.QWEN_HOME!); + expect(env.OPENAI_ORG_ID).toBe("org-daemon"); + await provider.teardown(); + }); }); diff --git a/src/tests/session-loop-controls.test.ts b/src/tests/session-loop-controls.test.ts new file mode 100644 index 00000000..abad13ac --- /dev/null +++ b/src/tests/session-loop-controls.test.ts @@ -0,0 +1,333 @@ +/** + * Session-level controls a loop controller drives (loop design §6, §8, §12.11), + * through a real Session over MockSessionProvider: + * + * - lastTurnInfraError: a turn failed on a usage / rate limit (with its + * reset), set on failure, cleared by the next successful turn; + * - onBudgetExhausted: "end_attempt" denies and ends the turn, settling idle, + * while the default "park" still waits for approval; + * - extraEnv: refused keys fail the create; allowed ones reach the provider init; + * - supervise(): a wall-time deadline that fires while a tool is running + * (stall watchdog paused), kills the backend after the grace period, and + * never fires once cancelled. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { randomUUID } from "node:crypto"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { Session, type SessionCreateOptions } from "../daemon/session.js"; +import { MockSessionProvider } from "../daemon/providers/mock/session-provider.js"; +import { mockResult } from "../daemon/providers/mock/index.js"; +import { ProviderRegistry, type ProviderSessionInit } from "../daemon/providers/registry.js"; +import type { ProviderEvent, TurnOpts, TurnRun } from "../daemon/providers/interface.js"; +import { supervise, type DeadlineEvent, type SupervisedSession } from "../daemon/supervisor.js"; +import { Store } from "../daemon/store.js"; +import { TranscriptStore } from "../daemon/transcript.js"; +import type { AuthContext, SessionStatus } from "../protocol/types.js"; +import type { CodeoidConfig } from "../config.js"; + +const AUTH: AuthContext = { sub: "user:loops", scopes: [], delegationDepth: 0, accountId: "acc", projectId: "proj" }; + +let tmp: string; +let store: Store; +let transcriptStore: TranscriptStore; + +beforeEach(() => { + tmp = mkdtempSync(join(tmpdir(), "codeoid-loopctl-")); + store = new Store(join(tmp, "codeoid.db")); + transcriptStore = new TranscriptStore(join(tmp, "transcripts")); +}); +afterEach(async () => { + await new Promise((r) => setTimeout(r, 50)); + try { await transcriptStore.flush(); } catch {} + try { store.close(); } catch {} + try { rmSync(tmp, { recursive: true, force: true }); } catch {} +}); + +function makeSession(extra: Partial = {}): Session { + const id = randomUUID(); + store.createSession({ + id, name: "loop", workdir: tmp, status: "idle", createdBy: AUTH.sub, + createdAt: new Date().toISOString(), attachedClients: 0, accountId: AUTH.accountId, projectId: AUTH.projectId, + }); + return new Session({ name: "loop", workdir: tmp, auth: AUTH, store, transcriptStore, existingId: id, ...extra }); +} + +async function until(cond: () => boolean, ms = 3_000): Promise { + const deadline = Date.now() + ms; + while (!cond()) { + if (Date.now() > deadline) throw new Error("condition not reached"); + await new Promise((r) => setTimeout(r, 5)); + } +} + +const done = (overrides: Parameters[0] = {}): ProviderEvent => ({ type: "turn_done", result: mockResult(overrides) }); +const bash = (n: number): ProviderEvent => ({ + type: "tool_start", toolId: `t${n}`, sdkToolUseId: `sdk-t${n}`, name: "Bash", input: { command: `echo ${n}` }, approvalId: `ap-${n}`, +}); +const bashDone = (n: number): ProviderEvent => ({ type: "tool_complete", sdkToolUseId: `sdk-t${n}`, output: `${n}`, success: true }); + +/** Config with a short stall watchdog (autoRotate off — #shouldRotate reads it on every send). */ +function stallConfig(turnStallTimeoutMs: number): CodeoidConfig { + return { + session: { turnStallTimeoutMs }, + autoRotate: { enabled: false, warnPct: 0.75, rotatePct: 0.9, hardRotatePct: 0.95, minTurnsBeforeRotate: 1, strategy: "task-anchor" }, + } as unknown as CodeoidConfig; +} + +// ── lastTurnInfraError ────────────────────────────────────────────────────── + +describe("lastTurnInfraError", () => { + test("a usage-limit turn records kind + reset; the next successful turn clears it", async () => { + const provider = new MockSessionProvider("claude", [ + [done({ isError: true, errorMessage: "You've hit your session limit · resets 3pm (UTC)" })], + [{ type: "text_done", content: "ok" }, done()], + ]); + const session = makeSession({ _testProvider: provider }); + expect(session.lastTurnInfraError).toBeNull(); + + await session.send("work", AUTH); + await until(() => session.status === "error"); + const infra = session.lastTurnInfraError; + expect(infra?.kind).toBe("usage_limit"); + expect(infra?.provider).toBe("claude"); + expect(infra?.message).toContain("session limit"); + expect(new Date(infra!.resetAt!).getUTCHours()).toBe(15); + expect(infra!.resetAt!).toBeGreaterThan(Date.now() - 60_000); + + await session.send("again", AUTH); + await until(() => session.status === "idle" && session.lastTurnInfraError === null); + }); + + test("the Claude CLI's limit notice in the turn's own assistant text is recognized", async () => { + const provider = new MockSessionProvider("claude", [ + [{ type: "text_done", content: "API Error: Repeated 529 Overloaded errors" }, done({ isError: true })], + ]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError?.kind).toBe("rate_limit"); + }); + + test("a provider error event, and an ordinary failure afterwards resets it to null", async () => { + const provider = new MockSessionProvider("codex", [ + [{ type: "error", message: "codex: You've hit your usage limit. Try again in 2h." }], + [{ type: "error", message: "codex exited unexpectedly (code=1 signal=null)" }], + ]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + const infra = session.lastTurnInfraError; + expect(infra?.kind).toBe("usage_limit"); + expect(infra!.resetAt! - infra!.at).toBeGreaterThan(2 * 3600_000 - 5_000); + + await session.send("again", AUTH); + await until(() => session.status === "error" && session.lastTurnInfraError === null); + }); + + test("Claude's rate_limit_event supplies the exact reset when the error text has none", async () => { + const resetsAt = Date.now() + 4 * 3600_000; + const provider = new MockSessionProvider("claude", [ + [{ type: "rate_limit", status: "rejected", resetsAt, limitType: "five_hour" }, done({ isError: true, errorMessage: "Claude AI usage limit reached" })], + ]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).toMatchObject({ kind: "usage_limit", resetAt: resetsAt }); + }); + + test("an ordinary failed turn records nothing", async () => { + const provider = new MockSessionProvider("claude", [[done({ isError: true, errorMessage: "Prompt is too long" })]]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).toBeNull(); + }); +}); + +// ── onBudgetExhausted ────────────────────────────────────────────────────── + +describe("onBudgetExhausted", () => { + // Budget 1: the first Bash spends it, the second finds it spent. `stall` + // keeps the turn open afterwards, so only an interrupt can end it. + const script = () => [[bash(1), bashDone(1), bash(2)]]; + + test("default park: the call waits for approval, the session wedges at waiting_approval", async () => { + const provider = new MockSessionProvider("claude", script(), { stall: true }); + const session = makeSession({ _testProvider: provider, initialMode: { mode: "autonomous", maxTurns: 1 } }); + await session.send("work", AUTH); + await until(() => session.pendingToolApprovals().length === 1); + expect(session.status).toBe("waiting_approval"); + expect(session.pendingToolApprovals()[0]!.toolName).toBe("Bash"); + expect(provider.canUseToolResults.map((r) => r.behavior)).toEqual(["allow"]); + expect(session.budgetExhausted).toBe(false); + expect(session.mode).toBe("guarded"); + await session.interrupt(AUTH); + }); + + test("end_attempt: the call is denied, the turn interrupted, the session idle with budgetExhausted", async () => { + const provider = new MockSessionProvider("claude", script(), { stall: true }); + const session = makeSession({ + _testProvider: provider, + initialMode: { mode: "autonomous", maxTurns: 1 }, + onBudgetExhausted: "end_attempt", + }); + const seen: SessionStatus[] = []; + session.attach({ id: "c", auth: AUTH, send: (m) => { if (m.type === "session.status_change") seen.push(m.status); } }); + await session.send("work", AUTH); + await until(() => provider.canUseToolResults.length === 2); + expect(provider.canUseToolResults[1]!.behavior).toBe("deny"); + expect(provider.canUseToolResults[1]!.message).toMatch(/budget/i); + await until(() => session.status === "idle" && session.turnInterrupted); + expect(seen).not.toContain("waiting_approval"); + expect(session.budgetExhausted).toBe(true); + expect(session.mode).toBe("guarded"); + expect(session.pendingToolApprovals()).toEqual([]); + + // A new autonomous budget is a new attempt. + session.setMode("autonomous", 3); + expect(session.budgetExhausted).toBe(false); + }); + + test("end_attempt with budget to spare behaves exactly like park", async () => { + const provider = new MockSessionProvider("claude", [[bash(1), bashDone(1), done()]]); + const session = makeSession({ + _testProvider: provider, + initialMode: { mode: "autonomous", maxTurns: 5 }, + onBudgetExhausted: "end_attempt", + }); + await session.send("work", AUTH); + await until(() => session.status === "idle" && provider.canUseToolResults.length === 1); + expect(provider.canUseToolResults[0]!.behavior).toBe("allow"); + expect(session.turnsRemaining).toBe(4); + expect(session.budgetExhausted).toBe(false); + }); +}); + +// ── extraEnv ─────────────────────────────────────────────────────────────── + +describe("extraEnv", () => { + test("refused keys fail the create, naming them", () => { + expect(() => + makeSession({ _testProvider: new MockSessionProvider("claude"), extraEnv: { LOOP_RUN_ID: "r", ANTHROPIC_API_KEY: "k", HOME: "/x" } }), + ).toThrow(/ANTHROPIC_API_KEY, HOME/); + }); + + test("allowed keys reach the provider through ProviderSessionInit", () => { + let captured: ProviderSessionInit | undefined; + const providers = new ProviderRegistry("cap"); + providers.register({ + id: "cap", + displayName: "Capture", + create: (init) => { + captured = init; + return new MockSessionProvider("cap"); + }, + }); + makeSession({ providers, providerId: "cap", extraEnv: { LOOP_RUN_ID: "run-1", DATABASE_URL: "postgres://t" } }); + expect(captured?.extraEnv).toEqual({ LOOP_RUN_ID: "run-1", DATABASE_URL: "postgres://t" }); + }); +}); + +// ── supervise() ──────────────────────────────────────────────────────────── + +/** A backend whose interrupt never returns — the wedge that makes the kill necessary. */ +class WedgedProvider extends MockSessionProvider { + override runTurn(opts: TurnOpts): TurnRun { + const run = super.runTurn(opts); + return { ...run, interrupt: () => new Promise(() => {}) }; + } +} + +describe("supervise()", () => { + test("fires while a tool is running (watchdog paused) and kills the backend after the grace period", async () => { + // The tool starts and never completes: tool_running, the stall watchdog + // (30ms here) paused for the whole time. + const provider = new WedgedProvider("claude", [[bash(1)]], { stall: true }); + const session = makeSession({ _testProvider: provider, config: stallConfig(30), initialMode: { mode: "autonomous" } }); + await session.send("long tool", AUTH); + await until(() => session.status === "tool_running"); + await new Promise((r) => setTimeout(r, 80)); // well past the stall window + expect(session.status).toBe("tool_running"); + expect(provider.teardownCount).toBe(0); + + const events: DeadlineEvent[] = []; + const deadline = supervise(session, { wallMs: 50, graceMs: 60, onEvent: (e) => events.push(e) }); + expect(await deadline.expired).toBe(true); + expect(events.map((e) => e.type)).toEqual(["expired", "terminated"]); + expect(provider.teardownCount).toBe(1); + expect(session.status).toBe("idle"); + expect(session.turnInterrupted).toBe(true); + }); + + test("a session the interrupt settles is not killed", async () => { + const provider = new MockSessionProvider("claude", [[bash(1)]], { stall: true }); + const session = makeSession({ _testProvider: provider, initialMode: { mode: "autonomous" } }); + await session.send("long tool", AUTH); + await until(() => session.status === "tool_running"); + const events: DeadlineEvent[] = []; + const deadline = supervise(session, { wallMs: 20, graceMs: 2_000, onEvent: (e) => events.push(e) }); + expect(await deadline.expired).toBe(true); + expect(events.map((e) => e.type)).toEqual(["expired", "settled"]); + expect(provider.teardownCount).toBe(0); + }); +}); + +describe("supervise() — timers", () => { + function fake(status: SessionStatus = "tool_running") { + const calls: string[] = []; + const s: SupervisedSession & { status: SessionStatus } = { + id: "s1", + accountId: "a", + projectId: "p", + status, + preparingTurn: false, + interrupt: async (sender) => { calls.push(`interrupt:${sender.sub}`); }, + terminateBackend: async (sender) => { calls.push(`terminate:${sender.sub}`); s.status = "idle"; }, + }; + return { s, calls }; + } + + test("cancel before expiry: nothing fires, expired resolves false", async () => { + const { s, calls } = fake(); + const d = supervise(s, { wallMs: 30, graceMs: 10 }); + d.cancel(); + expect(await d.expired).toBe(false); + await new Promise((r) => setTimeout(r, 80)); + expect(calls).toEqual([]); + d.cancel(); // idempotent + }); + + test("interrupt, then kill when still busy after grace; audited as the supervisor", async () => { + const { s, calls } = fake(); + const d = supervise(s, { wallMs: 10, graceMs: 20 }); + expect(await d.expired).toBe(true); + expect(calls).toEqual(["interrupt:system:supervisor", "terminate:system:supervisor"]); + }); + + test("cancel during the grace period abandons the kill", async () => { + const { s, calls } = fake(); + const d = supervise(s, { wallMs: 10, graceMs: 200 }); + await until(() => calls.length === 1); + d.cancel(); + expect(await d.expired).toBe(true); + await new Promise((r) => setTimeout(r, 250)); + expect(calls).toEqual(["interrupt:system:supervisor"]); + }); + + test("a queued send counts as busy", async () => { + const { s, calls } = fake("idle"); + (s as { preparingTurn: boolean }).preparingTurn = true; + const d = supervise(s, { wallMs: 5, graceMs: 15 }); + expect(await d.expired).toBe(true); + expect(calls.at(-1)).toBe("terminate:system:supervisor"); + }); + + test("rejects a non-positive wall time or a negative grace", () => { + const { s } = fake(); + expect(() => supervise(s, { wallMs: 0, graceMs: 0 })).toThrow(RangeError); + expect(() => supervise(s, { wallMs: 10, graceMs: -1 })).toThrow(RangeError); + }); +}); diff --git a/src/tests/usage-limit.test.ts b/src/tests/usage-limit.test.ts new file mode 100644 index 00000000..fdbac079 --- /dev/null +++ b/src/tests/usage-limit.test.ts @@ -0,0 +1,191 @@ +/** + * classifyUsageLimit — the real message shapes each backend emits for a usage + * or rate limit (sources cited in usage-limit.ts), and the ordinary errors that + * must NOT be mistaken for one. + */ + +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { CLAUDE_USAGE_LIMIT_PREFIXES, classifyUsageLimit } from "../daemon/providers/usage-limit.js"; + +// 2026-10-25 12:00 UTC — 05:00 in Los Angeles (PDT, UTC-7: DST ends Nov 1). +const NOW = Date.UTC(2026, 9, 25, 12, 0); +const at = (input: unknown) => classifyUsageLimit(input, { now: NOW }); + +describe("classifyUsageLimit — usage limits", () => { + test("Claude legacy `Claude AI usage limit reached|`", () => { + const resetSec = Math.floor(NOW / 1000) + 3600; + expect(at(`Claude AI usage limit reached|${resetSec}`)).toEqual({ kind: "usage_limit", resetAt: resetSec * 1000, provider: "claude" }); + }); + + test("Claude Code: You've hit your session limit · resets 3pm (America/Los_Angeles)", () => { + expect(at("You've hit your session limit · resets 3pm (America/Los_Angeles)")).toEqual({ + kind: "usage_limit", + resetAt: Date.UTC(2026, 9, 25, 22, 0), + }); + }); + + test("Claude Code: weekly limit with a date and a minute, typographic apostrophe", () => { + expect(at("You’ve hit your weekly limit · resets Oct 28, 9:30am (UTC)")?.resetAt).toBe(Date.UTC(2026, 9, 28, 9, 30)); + }); + + test("a bare time already past today resets tomorrow", () => { + // 4am UTC is 8 hours before NOW. + expect(at("You've hit your Opus limit · resets 4am (UTC)")?.resetAt).toBe(Date.UTC(2026, 9, 26, 4, 0)); + }); + + test("older CLI form: 5-hour limit reached ∙ resets 3pm (host time zone)", () => { + const r = at("5-hour limit reached ∙ resets 3pm"); + expect(r?.kind).toBe("usage_limit"); + expect(new Date(r!.resetAt!).getHours()).toBe(15); + }); + + test("Claude org / seat usage messages", () => { + expect(at("Your org is out of usage · add funds to continue")?.kind).toBe("usage_limit"); + expect(at("You're out of extra usage")?.kind).toBe("usage_limit"); + expect(at("You've hit your monthly spend limit.")?.kind).toBe("usage_limit"); + }); + + test("Codex: You've hit your usage limit … try again at 3:30 PM (local time)", () => { + const r = at( + "codex: You've hit your usage limit. Upgrade to Plus to continue using Codex (https://chatgpt.com/explore/plus), or try again at 3:30 PM.", + ); + expect(r?.kind).toBe("usage_limit"); + expect(r?.provider).toBe("codex"); + const d = new Date(r!.resetAt!); + expect([d.getHours(), d.getMinutes()]).toEqual([15, 30]); + }); + + test("Codex: a reset on another day", () => { + const r = at("You've hit your usage limit. To get more access now, send a request to your admin or try again at Oct 28th, 2026 3:30 PM."); + expect(r?.kind).toBe("usage_limit"); + expect(r?.resetAt).toBe(new Date(2026, 9, 28, 15, 30).getTime()); + }); + + test("Codex structured error info", () => { + expect(at({ message: "turn failed", codexErrorInfo: "usageLimitExceeded" })).toEqual({ kind: "usage_limit", provider: "codex" }); + }); + + test("gemini-cli TerminalQuotaError (message, class name, retry delay)", () => { + expect(at("gemini-cli: You have exhausted your daily quota on this model.")).toEqual({ kind: "usage_limit", provider: "gemini" }); + const err = Object.assign(new Error("quota"), { name: "TerminalQuotaError", retryDelayMs: 3_600_000 }); + expect(at(err)).toEqual({ kind: "usage_limit", resetAt: NOW + 3_600_000, provider: "gemini" }); + }); + + test("Google daily quota metric", () => { + expect(at("Quota exceeded for quota metric 'Gemini 2.5 Pro Requests' and limit 'Gemini 2.5 Pro Requests per day per user'")?.kind).toBe( + "usage_limit", + ); + }); + + test("OpenAI billing quota", () => { + expect(at("429 You exceeded your current quota, please check your plan and billing details.")?.kind).toBe("usage_limit"); + expect(at({ status: 429, error: { code: "insufficient_quota", message: "quota" } })?.kind).toBe("usage_limit"); + }); + + test("Qwen / DashScope allocation quota", () => { + expect(at("[API Error: Throttling.AllocationQuota: free allocated quota exceeded]")).toEqual({ kind: "usage_limit", provider: "qwen" }); + expect(at("Qwen OAuth quota exceeded: Your free daily quota has been reached.")?.kind).toBe("usage_limit"); + }); + + test("the caller's provider wins over the one a shape implies", () => { + expect(classifyUsageLimit("You've hit your usage limit.", { now: NOW, provider: "claude" })?.provider).toBe("claude"); + }); + + test("an implausible reset (in the past, or months out) is dropped, the kind kept", () => { + expect(at("Claude AI usage limit reached|1000")).toEqual({ kind: "usage_limit", provider: "claude" }); + expect(at(`Claude AI usage limit reached|${Math.floor(NOW / 1000) + 90 * 86_400}`)?.resetAt).toBeUndefined(); + }); +}); + +describe("classifyUsageLimit — rate limits", () => { + test("Anthropic API errors: 429 rate_limit_error and 529 overloaded_error", () => { + expect(at('429 {"type":"error","error":{"type":"rate_limit_error","message":"Number of request tokens has exceeded your per-minute rate limit"}}')).toEqual({ + kind: "rate_limit", + provider: "claude", + }); + expect(at('API Error: 529 {"type":"error","error":{"type":"overloaded_error","message":"Overloaded"}}')?.kind).toBe("rate_limit"); + }); + + test("Claude CLI after its own retries", () => { + expect(at("API Error: Repeated 529 Overloaded errors")?.kind).toBe("rate_limit"); + expect(at("API Error: Request rejected (429) · this may be a temporary capacity issue.")?.kind).toBe("rate_limit"); + // Says "usage limit" — but explicitly NOT one. + expect(at("API Error: Server is temporarily limiting requests (not your usage limit) · Rate limited")?.kind).toBe("rate_limit"); + }); + + test("Gemini free tier: RESOURCE_EXHAUSTED with a short retry delay", () => { + const body = JSON.stringify({ + error: { + code: 429, + message: "You exceeded your current quota, please check your plan and billing details. Please retry in 41.5s.", + status: "RESOURCE_EXHAUSTED", + }, + }); + expect(at(`GeminiProvider: ${body}`)).toEqual({ kind: "rate_limit", resetAt: NOW + 41_500 }); + }); + + test("qwen-code formatted 429", () => { + expect( + at("[API Error: Requests rate limit exceeded, please try again later. (Status: RESOURCE_EXHAUSTED)]\nPossible quota limitations in place or slow response times detected. Please wait and try again later.")?.kind, + ).toBe("rate_limit"); + }); + + test("OpenAI rate limit with its retry hint", () => { + expect( + at("429 Rate limit reached for gpt-4.1 in organization org-x on tokens per min (TPM): Limit 30000, Used 29000, Requested 2000. Please try again in 2.5s."), + ).toEqual({ kind: "rate_limit", resetAt: NOW + 2500 }); + }); + + test("codex retry exhaustion and a bare structured 429", () => { + expect(at("codex: exceeded retry limit, last status: 429 Too Many Requests")?.kind).toBe("rate_limit"); + expect(at({ status: 429, message: "request failed" })?.kind).toBe("rate_limit"); + expect(at({ message: "slow down", cause: { statusCode: 529 } })?.kind).toBe("rate_limit"); + }); + + test("durations in several units", () => { + expect(at("Rate limit exceeded. Try again in 1h 30m")?.resetAt).toBe(NOW + 90 * 60_000); + expect(at("Too Many Requests\nRetry-After: 30")?.resetAt).toBe(NOW + 30_000); + }); +}); + +describe("classifyUsageLimit — ordinary errors are not limits", () => { + const negatives: unknown[] = [ + "Error: 401 Invalid API key · Please run /login", + "Prompt is too long: 210000 tokens > 200000 maximum", + "Concurrent session limit reached (max 3)", + "Context limit reached", + "Budget limit reached ($5.00) — raise maxBudgetUsd to continue", + "codex exited unexpectedly (code=1 signal=null)", + "Build a rate limiter for the API", + "You've hit your stride — keep going", + "ECONNRESET socket hang up", + "Turn ended with an error", + "TypeError: cannot read properties of undefined (line 429)", + "Qwen OAuth free tier has been discontinued as of 2026-04-15.", + "Subagent nesting limit reached (depth 3)", + { status: 500, message: "Internal server error" }, + "", + null, + undefined, + {}, + ]; + for (const input of negatives) { + test(`null for ${JSON.stringify(input)}`, () => { + expect(at(input)).toBeNull(); + }); + } +}); + +describe("CLAUDE_USAGE_LIMIT_PREFIXES", () => { + test("matches the installed Agent SDK's USAGE_LIMIT_ERROR_PREFIXES", () => { + // Read from the SDK's declarations, not imported: another suite mocks the + // module, and the export is @alpha (the copy exists so a bump can't break us). + const dts = readFileSync(join(import.meta.dir, "..", "..", "node_modules", "@anthropic-ai", "claude-agent-sdk", "sdk.d.ts"), "utf8"); + const decl = /USAGE_LIMIT_ERROR_PREFIXES: readonly \[([^\]]*)\]/.exec(dts); + expect(decl).not.toBeNull(); + const sdk = [...decl![1]!.matchAll(/"([^"]*)"|'([^']*)'/g)].map((m) => m[1] ?? m[2]); + expect([...CLAUDE_USAGE_LIMIT_PREFIXES]).toEqual(sdk); + }); +}); From 57eb960c0d4446662a658166ce7de3ecb01bdb7d Mon Sep 17 00:00:00 2001 From: Yash Datta Date: Sun, 25 Oct 2026 08:19:12 +0800 Subject: [PATCH 2/4] feat: exact usage-limit resets from codex's structured error, and gemini-cli's "Access resets at" - codexErrorDetail() lifts the v2 TurnError codexErrorInfo variant (usageLimitExceeded, rateLimitExceeded, httpConnectionFailed's status) and its resets_at / resetsAt onto the error event's new `detail`; the classifier reads the structured reset ahead of any prose time. describeCodexError's display text is unchanged. - gemini-cli's "Access resets at 3:30 PM PDT." (getResetTimeMessage) is parsed in host time; the short zone name is not a zone id and is ignored. - extraEnv's doc states it stays in memory and the loop controller re-supplies it when it re-arms a worker after a restart. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/daemon/providers/codex/index.ts | 37 ++++++++++++++++++++++++- src/daemon/providers/interface.ts | 7 ++++- src/daemon/providers/usage-limit.ts | 33 ++++++++++++++++++---- src/daemon/session.ts | 9 ++++-- src/tests/provider-codex.test.ts | 27 +++++++++++++++++- src/tests/session-loop-controls.test.ts | 16 +++++++++++ src/tests/usage-limit.test.ts | 24 ++++++++++++++++ 7 files changed, 141 insertions(+), 12 deletions(-) diff --git a/src/daemon/providers/codex/index.ts b/src/daemon/providers/codex/index.ts index a53a089c..c21a4ecb 100644 --- a/src/daemon/providers/codex/index.ts +++ b/src/daemon/providers/codex/index.ts @@ -267,6 +267,40 @@ export function describeCodexError(params: Record): string { } } +/** + * The structured half of a codex `error` notification, for the usage-limit + * classifier: the v2 `TurnError.codexErrorInfo` variant (`usageLimitExceeded`, + * `rateLimitExceeded`, `httpConnectionFailed { httpStatusCode }`, … — codex-rs + * 0.154.0 app-server protocol) and the limit's reset instant when it carries + * one (`resets_at` / `resetsAt`, epoch seconds or ms). The info is a bare + * string for a unit variant and `{ variant: { …fields } }` otherwise. + * Undefined when the notification has no error info. describeCodexError keeps + * producing the display text; this never reaches the user. + */ +export function codexErrorDetail(params: Record): Record | undefined { + const err = params.error && typeof params.error === "object" ? (params.error as Record) : undefined; + const info = err?.codexErrorInfo ?? params.codexErrorInfo; + let variant: string | undefined; + let fields: Record = {}; + if (typeof info === "string") variant = info; + else if (info && typeof info === "object") { + const [name, value] = Object.entries(info as Record)[0] ?? []; + variant = name; + if (value && typeof value === "object") fields = value as Record; + } + if (!variant) return undefined; + const data = err?.data && typeof err.data === "object" ? (err.data as Record) : {}; + const reset = [fields.resetsAt, fields.resets_at, err?.resetsAt, err?.resets_at, data.resetsAt, data.resets_at].find( + (v): v is number => typeof v === "number" && v > 0, + ); + const status = fields.httpStatusCode; + return { + codexErrorInfo: variant, + ...(reset !== undefined ? { resetsAt: reset } : {}), + ...(typeof status === "number" ? { status } : {}), + }; +} + export class CodexProvider implements SessionProvider { /** * codex's approval replies carry only a decision (`approved` / `accept`) — @@ -1176,7 +1210,8 @@ export class CodexProvider implements SessionProvider { // the literal "[object Object]" — the cause destroyed at the one place // it was supposed to be reported. Observed live: a role-child failed // twice with `Error: [object Object]` and nothing anywhere said why. - this.#push({ type: "error", message: `codex: ${describeCodexError(params)}` }); + const detail = codexErrorDetail(params); + this.#push({ type: "error", message: `codex: ${describeCodexError(params)}`, ...(detail ? { detail } : {}) }); break; } default: diff --git a/src/daemon/providers/interface.ts b/src/daemon/providers/interface.ts index 8a128891..174358f6 100644 --- a/src/daemon/providers/interface.ts +++ b/src/daemon/providers/interface.ts @@ -368,7 +368,12 @@ export type ProviderEvent = * denied. Distinct from a tool approval, whose SDK turn is still live. */ | { type: "approval_pending"; command: string } - | { type: "error"; message: string }; + /** + * The turn failed. `detail` is the backend's STRUCTURED error, when it gave + * one (codex's `codexErrorInfo` + reset instant — codexErrorDetail): read by + * the usage-limit classifier, never shown. `message` stays the display text. + */ + | { type: "error"; message: string; detail?: Record }; /** * True when a text/thinking ProviderEvent was produced by a subagent diff --git a/src/daemon/providers/usage-limit.ts b/src/daemon/providers/usage-limit.ts index 9d94b201..496d2eb8 100644 --- a/src/daemon/providers/usage-limit.ts +++ b/src/daemon/providers/usage-limit.ts @@ -153,6 +153,8 @@ const RATE_RULES: readonly Rule[] = [ re(/\b(?:status|code|HTTP)\s*[:=]?\s*429\b/i, "rate_limit"), re(/"code"\s*:\s*429\b/, "rate_limit"), re(/(?:^|[:\]\n]\s*)429\s+(?:\w+\s+){0,2}(?:rate|limit|quota|too many)/im, "rate_limit"), + // codex's v2 `codexErrorInfo: "rateLimitExceeded"` (codex-rs 0.154.0). + re(/\brateLimitExceeded\b/, "rate_limit", "codex"), // DashScope throttling codes other than the allocation quota (e.g. Throttling.RateQuota). re(/\bThrottling\.(?!AllocationQuota)\w+/, "rate_limit", "qwen"), ]; @@ -161,6 +163,8 @@ interface Signals { texts: string[]; status?: number; retryDelayMs?: number; + /** A reset instant the error carried as data (codex `resets_at`), epoch ms. */ + resetAtMs?: number; } function collect(input: unknown, out: Signals, depth: number): void { @@ -187,10 +191,20 @@ function collect(input: unknown, out: Signals, depth: number): void { if (out.retryDelayMs === undefined && typeof o.retryDelayMs === "number" && o.retryDelayMs > 0) { out.retryDelayMs = o.retryDelayMs; } - // codex: { codexErrorInfo: "usageLimitExceeded" } or { usageLimitExceeded: {...} }. + // A structured reset: codex's `resets_at` / `resetsAt` (seconds or ms). + if (out.resetAtMs === undefined) { + const reset = [o.resetsAt, o.resets_at, o.resetAt].find((v): v is number => typeof v === "number" && v > 0); + if (reset !== undefined) out.resetAtMs = toEpochMs(reset); + } + // codex: { codexErrorInfo: "usageLimitExceeded" } or { codexErrorInfo: { usageLimitExceeded: {...} } }. const info = o.codexErrorInfo; if (typeof info === "string") out.texts.push(info); - else if (info && typeof info === "object") out.texts.push(Object.keys(info).join(" ")); + else if (info && typeof info === "object") { + for (const [variant, fields] of Object.entries(info as Record)) { + out.texts.push(variant); + if (fields && typeof fields === "object") collect(fields, out, depth + 1); + } + } if (o.error !== undefined && o.error !== input) collect(o.error, out, depth + 1); if (o.cause !== undefined && o.cause !== input) collect(o.cause, out, depth + 1); } @@ -261,7 +275,7 @@ export function classifyUsageLimit( kind = "usage_limit"; } - const resetAt = parseResetAt(text, now, signals.retryDelayMs); + const resetAt = parseResetAt(text, now, signals); const resolvedProvider = opts.provider ?? provider; return { kind, @@ -312,7 +326,12 @@ const MONTHS: Record = { * "resets Oct 28, 9am (UTC)" / "resets Oct 28, 2027, 9:30am" (its `Yc` * formatter: en-US time, lower-cased am/pm, the zone in parentheses) and * Codex's "Try again at 3:30 PM." / "try again at Oct 28th, 2026 3:30 PM." - * (codex-rs `%-I:%M %p` / `%b %-d, %Y %-I:%M %p`, local time). + * (codex-rs `%-I:%M %p` / `%b %-d, %Y %-I:%M %p`, local time), and + * gemini-cli's "Access resets at 3:30 PM PDT." (getResetTimeMessage, gemini-cli + * 0.50.0 interactiveCli bundle: en-US `hour: numeric, minute: 2-digit, + * timeZoneName: short` in the host zone). A short zone name ("PDT") is not a + * zone id and is ignored: these CLIs run on the daemon's host, so host time is + * the zone they printed in. Only a parenthesized IANA id (Claude's) is applied. */ const CLOCK = /(?:resets?(?:\s+at)?|try again at)\s+(?:(jan|feb|mar|apr|may|jun|jul|aug|sep|oct|nov|dec)[a-z]*\.?\s+(\d{1,2})(?:st|nd|rd|th)?,?\s+(?:(\d{4}),?\s+)?)?(\d{1,2})(?::(\d{2}))?\s*([ap])\.?\s?m\.?(?:\s*\(([^)]+)\))?/i; @@ -394,8 +413,10 @@ function wallToEpoch(year: number, month: number, day: number, hour: number, min return at; } -function parseResetAt(text: string, now: number, retryDelayMs: number | undefined): number | undefined { - const candidates: Array = []; +function parseResetAt(text: string, now: number, signals: Signals): number | undefined { + const { retryDelayMs } = signals; + // Data beats prose: an instant the error carried as a field is exact. + const candidates: Array = [signals.resetAtMs]; // Claude's legacy `Claude AI usage limit reached|1730000000`. const legacy = /usage limit reached\|(\d{9,13})/i.exec(text); if (legacy) candidates.push(toEpochMs(Number(legacy[1]))); diff --git a/src/daemon/session.ts b/src/daemon/session.ts index 66c88a86..63cf89c2 100644 --- a/src/daemon/session.ts +++ b/src/daemon/session.ts @@ -502,8 +502,10 @@ export interface SessionCreateOptions { * account-isolation denylist (`extraEnvViolations`, providers/env.ts) is * refused: the constructor throws, so a misconfigured caller learns at once. * No effect on in-process backends (openai, gemini, llama.cpp, endpoints). - * Held in memory only — not persisted, so a session restored after a daemon - * restart runs without it unless its owner supplies it again. + * Held in memory only, by design — never persisted (it can carry a run's + * secrets-adjacent settings, and a stale copy must not outlive the run). A + * session restored after a daemon restart has none: the loop controller + * re-supplies it when it re-arms the worker during restart reconciliation. */ extraEnv?: Readonly>; /** @@ -7106,7 +7108,8 @@ export class Session { case "error": { console.error(`[codeoid/session ${this.id}] provider error:`, event.message); this.#lastTurnError = event.message; // before the status change (see above) - this.#recordTurnFailure([event.message]); + // The structured detail (codex's error info + reset) rides with the text. + this.#recordTurnFailure([event.detail ? { message: event.message, error: event.detail } : event.message]); this.#setStatus("error"); const errorMsg = this.#makeMessage("system", `Error: ${event.message}`, SYSTEM_IDENTITY, undefined, undefined, { event: "agent_error" }); this.#persistAndBuffer(errorMsg); diff --git a/src/tests/provider-codex.test.ts b/src/tests/provider-codex.test.ts index e214d0d6..0fbce963 100644 --- a/src/tests/provider-codex.test.ts +++ b/src/tests/provider-codex.test.ts @@ -27,7 +27,7 @@ import { chmodSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:f import { tmpdir } from "node:os"; import { join } from "node:path"; import { CodexRpcProcess } from "../daemon/providers/codex/rpc.js"; -import { CodexProvider, describeCodexError } from "../daemon/providers/codex/index.js"; +import { CodexProvider, codexErrorDetail, describeCodexError } from "../daemon/providers/codex/index.js"; import { MemoryMcpHttp, MEMORY_MCP_SERVER_NAME } from "../daemon/memory/mcp-http.js"; import { McpRegistry } from "../daemon/mcp/registry.js"; import { MCP_PROXY_PATH_PREFIX, MCP_PROXY_TOKEN_ENV, McpProxy } from "../daemon/mcp/proxy.js"; @@ -1330,6 +1330,31 @@ describe("describeCodexError", () => { }); }); +describe("codexErrorDetail", () => { + it("extracts a unit variant and a reset instant from the error object", () => { + expect(codexErrorDetail({ error: { message: "x", codexErrorInfo: "usageLimitExceeded", resets_at: 1_800_000_000 } })).toEqual({ + codexErrorInfo: "usageLimitExceeded", + resetsAt: 1_800_000_000, + }); + }); + + it("extracts a struct variant's fields (reset, HTTP status)", () => { + expect(codexErrorDetail({ error: { message: "x", codexErrorInfo: { usageLimitExceeded: { resetsAt: 1_800_000_000_000 } } } })).toEqual({ + codexErrorInfo: "usageLimitExceeded", + resetsAt: 1_800_000_000_000, + }); + expect(codexErrorDetail({ error: { message: "x", codexErrorInfo: { httpConnectionFailed: { httpStatusCode: 429 } } } })).toEqual({ + codexErrorInfo: "httpConnectionFailed", + status: 429, + }); + }); + + it("undefined without error info", () => { + expect(codexErrorDetail({ error: { message: "x" } })).toBeUndefined(); + expect(codexErrorDetail({})).toBeUndefined(); + }); +}); + describe("CodexProvider – thinking effort (#417)", () => { const effortSeen = async (effort?: EffortLevel): Promise => { const p = makeProvider(); diff --git a/src/tests/session-loop-controls.test.ts b/src/tests/session-loop-controls.test.ts index abad13ac..5a7358ac 100644 --- a/src/tests/session-loop-controls.test.ts +++ b/src/tests/session-loop-controls.test.ts @@ -127,6 +127,22 @@ describe("lastTurnInfraError", () => { await until(() => session.status === "error" && session.lastTurnInfraError === null); }); + test("a provider error's structured detail (codex) gives the exact reset", async () => { + const resetSec = Math.floor(Date.now() / 1000) + 3 * 3600; + const provider = new MockSessionProvider("codex", [ + [{ type: "error", message: "codex: You've hit your usage limit.", detail: { codexErrorInfo: "usageLimitExceeded", resetsAt: resetSec } }], + ]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).toMatchObject({ + kind: "usage_limit", + resetAt: resetSec * 1000, + message: "codex: You've hit your usage limit.", + provider: "codex", + }); + }); + test("Claude's rate_limit_event supplies the exact reset when the error text has none", async () => { const resetsAt = Date.now() + 4 * 3600_000; const provider = new MockSessionProvider("claude", [ diff --git a/src/tests/usage-limit.test.ts b/src/tests/usage-limit.test.ts index fdbac079..560bdc29 100644 --- a/src/tests/usage-limit.test.ts +++ b/src/tests/usage-limit.test.ts @@ -67,6 +67,30 @@ describe("classifyUsageLimit — usage limits", () => { expect(at({ message: "turn failed", codexErrorInfo: "usageLimitExceeded" })).toEqual({ kind: "usage_limit", provider: "codex" }); }); + test("Codex structured error info carries the exact reset (seconds or ms, unit or struct variant)", () => { + const resetSec = Math.floor(NOW / 1000) + 7200; + expect(at({ message: "codex: You've hit your usage limit.", error: { codexErrorInfo: "usageLimitExceeded", resetsAt: resetSec } })).toEqual({ + kind: "usage_limit", + resetAt: resetSec * 1000, + provider: "codex", + }); + // The struct field beats the prose time. + expect( + at({ message: "codex: You've hit your usage limit. Try again at 3:30 PM.", error: { codexErrorInfo: { usageLimitExceeded: { resets_at: resetSec * 1000 } } } }) + ?.resetAt, + ).toBe(resetSec * 1000); + expect(at({ message: "codex: slow down", error: { codexErrorInfo: "rateLimitExceeded" } })).toEqual({ kind: "rate_limit", provider: "codex" }); + expect(at({ message: "codex: request failed", error: { codexErrorInfo: "httpConnectionFailed", status: 429 } })?.kind).toBe("rate_limit"); + expect(at({ message: "codex: boom", error: { codexErrorInfo: "contextWindowExceeded" } })).toBeNull(); + }); + + test("gemini-cli: Usage limit reached … Access resets at 3:30 PM PDT (host time; the short zone name is ignored)", () => { + const r = at("Usage limit reached for gemini-2.5-pro.\nAccess resets at 3:30 PM PDT.\n/stats model for usage details\n/model to switch models."); + expect(r?.kind).toBe("usage_limit"); + const d = new Date(r!.resetAt!); + expect([d.getHours(), d.getMinutes()]).toEqual([15, 30]); + }); + test("gemini-cli TerminalQuotaError (message, class name, retry delay)", () => { expect(at("gemini-cli: You have exhausted your daily quota on this model.")).toEqual({ kind: "usage_limit", provider: "gemini" }); const err = Object.assign(new Error("quota"), { name: "TerminalQuotaError", retryDelayMs: 3_600_000 }); From 86ddee26306360a36042d560c714df17b15e5e30 Mon Sep 17 00:00:00 2001 From: Yash Datta Date: Sun, 25 Oct 2026 08:41:04 +0800 Subject: [PATCH 3/4] fix: round-1 audit of the unattended-worker controls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. A Claude rate_limit "rejected" that overage is carrying no longer counts, and the rate-limit-event fallback never relabels a failure another recognizer explains (prompt too long, the spend ceiling). 2. Claude's result text on a success-subtype is_error turn is carried as NormalizedTurnResult.resultText; it and the turn's assistant text are read only with CLI-shaped rules (classifyCliNotice: the SDK's USAGE_LIMIT_ERROR_PREFIXES or a leading "API Error:"). The generic openers now need the limit noun within four words, so model replies such as "You've reached your goal: the rate limit test passes" stay null. 3. lastTurnInfraError is cleared when a turn starts. 4. Under end_attempt, a spent budget keeps denying every tool call — later prompts and backend-started turns included — until a new autonomous budget; ending the attempt also stops background tasks where the backend can (Claude: stopTask). 5. #sendInner resets per-turn controls only when a turn actually starts, not for a message that joins the running one. 6. extraEnv names must be ^[A-Z][A-Z0-9_]{0,63}$; the denylist adds tool-config, pager/editor, shell-startup, temp-dir, TZ, RUST_* knobs and the GIT_/NPM_CONFIG_/PYTHON/RUBY/PERL/JAVA/JDK/CLOUDSDK_/LC_ namespaces. Documented as daemon-controlled only. 7. SessionProvider.enforcesToolBudget (false for codex, whose autonomous mode is approvalPolicy "never") and Session.toolBudgetEnforced; end_attempt on such a backend logs a warning. 8. interrupt() leaves alone a run that took over while it awaited the backend; hard-stop teardowns are bounded (10s default, overridable on terminateBackend and supervise). 9. terminateBackend snapshots the process tree under the backend's own pids before the teardown and SIGKILLs what outlives it (process-tree.ts). Measured on the real Claude CLI: tool shells are reaped on abort and interrupt, survive a SIGKILL of the CLI (each runs in its own process group), and self-detached commands survive everything. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/daemon/process-tree.ts | 124 +++++++++++++ src/daemon/providers/acp/index.ts | 5 + src/daemon/providers/claude/index.ts | 42 ++++- src/daemon/providers/codex/index.ts | 11 ++ src/daemon/providers/env.ts | 33 +++- src/daemon/providers/interface.ts | 32 +++- src/daemon/providers/jsonrpc-stdio.ts | 5 + src/daemon/providers/pi/index.ts | 5 + src/daemon/providers/pi/rpc.ts | 5 + src/daemon/providers/usage-limit.ts | 51 +++++- src/daemon/session.ts | 227 ++++++++++++++++++------ src/daemon/supervisor.ts | 10 +- src/tests/process-tree.test.ts | 133 ++++++++++++++ src/tests/provider-claude.test.ts | 28 +++ src/tests/provider-env.test.ts | 39 ++-- src/tests/session-loop-controls.test.ts | 207 +++++++++++++++++++++ src/tests/usage-limit.test.ts | 47 ++++- 17 files changed, 922 insertions(+), 82 deletions(-) create mode 100644 src/daemon/process-tree.ts create mode 100644 src/tests/process-tree.test.ts diff --git a/src/daemon/process-tree.ts b/src/daemon/process-tree.ts new file mode 100644 index 00000000..54eedcb1 --- /dev/null +++ b/src/daemon/process-tree.ts @@ -0,0 +1,124 @@ +/** + * Killing what a backend left running (Session.terminateBackend, supervisor.ts). + * + * Tearing a backend down kills ITS process; what that process had started — + * the shell a tool runs in, and whatever that shell runs — does not always go + * with it. Measured against the Claude Code CLI (SDK 0.3.281, a fake Messages + * API driving a real `Bash: sleep` call): + * + * - abort (SIGTERM via the SDK) and Query.interrupt(): the CLI kills the + * tool's shell. Nothing survives. + * - the CLI SIGKILLed (the SDK's own escalation 5s after a SIGTERM the CLI + * didn't act on): the tool's `sleep` SURVIVES, re-parented. Each Bash + * call runs in its OWN process group (pgid = the shell's pid), so killing + * the CLI's process group would not reach it either. + * - a command that detaches itself (`setsid …`, `nohup … &`): survives every + * path — it left the tree before anything here looks at it. + * + * codex and gemini-cli are stopped with a bare SIGKILL (jsonrpc-stdio.ts), so + * their running commands are in the second case on every teardown. + * + * So the stop is a TREE kill, not a group kill: snapshot every descendant of + * the backend's own process(es) before the teardown, and after it SIGKILL any + * of them still alive. A survivor is matched by pid AND command line, so a pid + * the OS reused in between is never touched. What this cannot reach, and does + * not claim to: anything that detached itself before the snapshot (only a + * cgroup or sandbox boundary bounds that), and a backend whose own process + * ids are unknown (qwen-code: @qwen-code/sdk spawns its CLI internally). + * + * Uses `ps` (Linux, macOS). Unavailable `ps` or Windows: empty snapshots, so + * terminateBackend degrades to the plain teardown. + */ + +import { execFile } from "node:child_process"; + +export interface ProcessEntry { + pid: number; + ppid: number; + args: string; +} + +const PS_TIMEOUT_MS = 3_000; + +/** Every process visible to this user: pid, parent pid, command line. */ +export function listProcesses(): Promise { + if (process.platform === "win32") return Promise.resolve([]); + return new Promise((resolve) => { + execFile( + "ps", + ["-A", "-o", "pid=,ppid=,args="], + { timeout: PS_TIMEOUT_MS, maxBuffer: 32 * 1024 * 1024 }, + (err, stdout) => { + if (err) { + resolve([]); + return; + } + resolve(parsePs(stdout)); + }, + ); + }); +} + +/** Parse `ps -o pid=,ppid=,args=` output. Exported for tests. */ +export function parsePs(stdout: string): ProcessEntry[] { + const out: ProcessEntry[] = []; + for (const line of stdout.split("\n")) { + const m = /^\s*(\d+)\s+(\d+)\s+(.*)$/.exec(line); + if (m) out.push({ pid: Number(m[1]), ppid: Number(m[2]), args: m[3]!.trim() }); + } + return out; +} + +/** `roots` (those present) and all their descendants in `table`. Never this process. */ +export function treeOf(table: readonly ProcessEntry[], roots: readonly number[]): ProcessEntry[] { + const children = new Map(); + const byPid = new Map(); + for (const p of table) { + byPid.set(p.pid, p); + const list = children.get(p.ppid); + if (list) list.push(p); + else children.set(p.ppid, [p]); + } + const out: ProcessEntry[] = []; + const seen = new Set([process.pid]); + const queue = roots.filter((pid) => byPid.has(pid) && !seen.has(pid)); + for (const pid of queue) seen.add(pid); + while (queue.length > 0) { + const pid = queue.shift()!; + out.push(byPid.get(pid)!); + for (const child of children.get(pid) ?? []) { + if (seen.has(child.pid)) continue; + seen.add(child.pid); + queue.push(child.pid); + } + } + return out; +} + +/** The process tree under `roots` right now. */ +export async function snapshotTree(roots: readonly number[]): Promise { + if (roots.length === 0) return []; + return treeOf(await listProcesses(), roots); +} + +/** + * SIGKILL every process of `snapshot` that is still running with the same + * command line (so a reused pid is spared). Returns the pids signalled. + */ +export async function killSurvivors(snapshot: readonly ProcessEntry[]): Promise { + if (snapshot.length === 0) return []; + const now = new Map((await listProcesses()).map((p) => [p.pid, p])); + const killed: number[] = []; + for (const p of snapshot) { + if (p.pid === process.pid) continue; + const live = now.get(p.pid); + if (!live || live.args !== p.args) continue; + try { + process.kill(p.pid, "SIGKILL"); + killed.push(p.pid); + } catch { + // already gone, or not ours to signal + } + } + return killed; +} diff --git a/src/daemon/providers/acp/index.ts b/src/daemon/providers/acp/index.ts index b56cd2fe..611ded93 100644 --- a/src/daemon/providers/acp/index.ts +++ b/src/daemon/providers/acp/index.ts @@ -252,6 +252,11 @@ export class GeminiAcpProvider implements SessionProvider { await this.teardown(); } + async backendProcessIds(): Promise { + const pid = this.#proc?.pid; + return pid ? [pid] : []; + } + async teardown(): Promise { this.#turnQueue?.close(); this.#turnQueue = null; diff --git a/src/daemon/providers/claude/index.ts b/src/daemon/providers/claude/index.ts index f161c74a..96369a08 100644 --- a/src/daemon/providers/claude/index.ts +++ b/src/daemon/providers/claude/index.ts @@ -67,6 +67,7 @@ import { BUDGET_STOP_MESSAGE, isBackgroundLifecycleEvent } from "../interface.js import { isUngrantableSkillCommand, redactCommand } from "../../skill-command.js"; import { renderHistorySeed, type CanonicalTurn, type HistorySeedResult } from "../canonical.js"; import { buildSubprocessEnv, withExtraEnv, withGatewayCredential } from "../env.js"; +import { listProcesses } from "../../process-tree.js"; import type { LLMCallUsage } from "../../context-math.js"; import type { PackSubagent } from "../../pipeline/subagents.js"; @@ -237,6 +238,8 @@ export class ClaudeProvider implements SessionProvider { // Claude-specific backing-session state #claudeCodeSessionId: string; + /** The backing id the live query's CLI was spawned with (backendProcessIds). */ + #spawnedWithId: string | null = null; #hasQueried = false; #backingRecoveryAttempted = false; #lastPushedContent: string | null = null; @@ -646,6 +649,19 @@ export class ClaudeProvider implements SessionProvider { await this.teardown(); } + /** + * The Claude Code CLI this provider's query runs in. The SDK spawns it + * (spawnClaudeCodeProcess is not used: replacing the SDK's spawn would drop + * its stderr diagnostics), so it is found as a direct child of this process + * whose argv names this provider's backing session (`--resume=` / + * `--session-id ` — ids are UUIDs, unique per session). + */ + async backendProcessIds(): Promise { + if (!this.#query) return []; + const id = this.#spawnedWithId ?? this.#claudeCodeSessionId; + return (await listProcesses()).filter((p) => p.ppid === process.pid && p.args.includes(id)).map((p) => p.pid); + } + async teardown(): Promise { this.#inputQueue?.close(); this.#abortController?.abort(); @@ -831,6 +847,8 @@ export class ClaudeProvider implements SessionProvider { const sessionOpts = this.#hasQueried ? { resume: this.#claudeCodeSessionId } : { sessionId: this.#claudeCodeSessionId }; + // What this query's CLI was started with — its argv names it (backendProcessIds). + this.#spawnedWithId = this.#claudeCodeSessionId; const merged = this.#mcpServerSet(opts, oauthMcp); const mcpServers = Object.keys(merged).length > 0 ? merged : undefined; @@ -1751,6 +1769,11 @@ export function translateSDKMessage( // Only a turn that completed is a point to fork at: an errored one // (a context overflow) would carry the failure into the fork, and an // interrupted one can end mid tool call. + // A turn the CLI ends "successfully" but flagged as an error carries + // its notice (a usage limit, "API Error: …") as the result text. + ...(r.is_error && r.subtype === "success" && typeof r.result === "string" && r.result + ? { resultText: r.result } + : {}), ...(sessionId && chainEnd && !zeroTurn && r.subtype === "success" && !r.is_error ? { nativeRef: { backingId: sessionId, ref: chainEnd, ...(state.cwd ? { dir: state.cwd } : {}) } } : {}), @@ -1890,12 +1913,29 @@ export function translateSDKMessage( case "rate_limit_event": { // SDKRateLimitInfo.resetsAt is epoch SECONDS (the CLI renders it with // `new Date(resetsAt * 1000)`); ProviderEvent carries milliseconds. - const info = (msg as { rate_limit_info?: { status?: string; resetsAt?: number; rateLimitType?: string } }).rate_limit_info; + const info = (msg as { + rate_limit_info?: { + status?: string; + resetsAt?: number; + rateLimitType?: string; + isUsingOverage?: boolean; + overageInUse?: boolean; + overageStatus?: string; + }; + }).rate_limit_info; const status = info?.status; if (status !== "allowed" && status !== "allowed_warning" && status !== "rejected") break; + // The plan window is spent but extra usage (overage) is carrying the + // requests: not a limit that stops the turn. + const overage = + info?.isUsingOverage === true || + info?.overageInUse === true || + info?.overageStatus === "allowed" || + info?.overageStatus === "allowed_warning"; emit({ type: "rate_limit", status, + ...(overage ? { overage: true } : {}), ...(typeof info?.resetsAt === "number" && info.resetsAt > 0 ? { resetsAt: info.resetsAt * 1000 } : {}), ...(info?.rateLimitType ? { limitType: info.rateLimitType } : {}), }); diff --git a/src/daemon/providers/codex/index.ts b/src/daemon/providers/codex/index.ts index c21a4ecb..7c350fcd 100644 --- a/src/daemon/providers/codex/index.ts +++ b/src/daemon/providers/codex/index.ts @@ -307,6 +307,12 @@ export class CodexProvider implements SessionProvider { * codex runs the input IT proposed, so a rewrite can't apply (#367). */ readonly appliesToolInputRewrite = false; + /** + * Autonomous mode is `approvalPolicy: "never"` (approvalPolicyForMode): codex + * runs commands and patches without an approval request, so codeoid's tool + * budget never sees them. + */ + readonly enforcesToolBudget = false; readonly id = "codex"; readonly displayName = "Codex (OpenAI)"; @@ -743,6 +749,11 @@ export class CodexProvider implements SessionProvider { await this.teardown(); } + async backendProcessIds(): Promise { + const pid = this.#proc?.pid; + return pid ? [pid] : []; + } + async teardown(): Promise { this.#pendingMcpCalls.clear(); this.#turnQueue?.close(); diff --git a/src/daemon/providers/env.ts b/src/daemon/providers/env.ts index d6ad7e62..da05f079 100644 --- a/src/daemon/providers/env.ts +++ b/src/daemon/providers/env.ts @@ -91,9 +91,10 @@ export function buildSubprocessEnv( * home at another login, inject code into the backend (NODE_OPTIONS, * LD_PRELOAD), or route its bearer token through a proxy. * - * Matched case-insensitively, by exact name, namespace prefix, or credential - * suffix. Deliberately broad: a controller that needs one of these has to get - * it through the account system, not an env side door. + * Matched by exact name, namespace prefix, or credential suffix, on + * upper-case names (anything else is refused outright). Deliberately broad: a + * controller that needs one of these has to get it through the account system + * or a trusted setup recipe, not an env side door. */ const EXTRA_ENV_DENY_EXACT = new Set([ // Identity and the vendor homes every account overlay lives under. @@ -106,6 +107,14 @@ const EXTRA_ENV_DENY_EXACT = new Set([ "HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "NO_PROXY", "NODE_EXTRA_CA_CERTS", "NODE_TLS_REJECT_UNAUTHORIZED", "SSL_CERT_FILE", "SSL_CERT_DIR", "CODE_ASSIST_ENDPOINT", + // Variables a tool the backend runs reads as configuration or as a command + // to execute: shell startup files, pagers and editors, ripgrep's config, + // glibc's charset-module path. + "RIPGREP_CONFIG_PATH", "BASH_ENV", "ENV", "PROMPT_COMMAND", + "PAGER", "GIT_PAGER", "LESSOPEN", "LESSCLOSE", "EDITOR", "VISUAL", "GCONV_PATH", + // Backend runtime knobs: logging that can echo requests, and where temp + // files (prompts, patches) land. + "RUST_LOG", "RUST_BACKTRACE", "TMPDIR", "TEMP", "TMP", "TZ", ]); const EXTRA_ENV_DENY_PREFIXES = [ // codeoid's own namespaces (DENY_PREFIXES above) — tokens, the escape hatch. @@ -117,17 +126,25 @@ const EXTRA_ENV_DENY_PREFIXES = [ "AWS_", "AZURE_", "VERTEX_", "OTEL_", // Dynamic-loader injection. "LD_", "DYLD_", + // Toolchains whose env is configuration that can run code or swap + // credentials: git (hooks via GIT_CONFIG_*, GIT_SSH_COMMAND), npm, Python, + // Ruby, Perl, the JVM, gcloud. Locale categories (LC_*) too. + "GIT_", "NPM_CONFIG_", "PYTHON", "RUBY", "PERL5", "PERL_", "JAVA_", "_JAVA_", "JDK_", "CLOUDSDK_", "LC_", ] as const; const EXTRA_ENV_DENY_SUFFIXES = ["_API_KEY", "_API_TOKEN", "_AUTH_TOKEN", "_ACCESS_TOKEN", "_REFRESH_TOKEN", "_OAUTH_TOKEN"] as const; -const ENV_NAME = /^[A-Za-z_][A-Za-z0-9_]*$/; +/** + * Upper-case names only: lower-case twins (`http_proxy`, `npm_config_*`) are + * read by tools as the same setting, and refusing the shape is simpler and + * safer than enumerating every twin. + */ +const ENV_NAME = /^[A-Z][A-Z0-9_]{0,63}$/; function extraEnvRefused(name: string, value: unknown): boolean { if (!ENV_NAME.test(name) || typeof value !== "string" || value.includes("\0")) return true; - const upper = name.toUpperCase(); return ( - EXTRA_ENV_DENY_EXACT.has(upper) || - EXTRA_ENV_DENY_PREFIXES.some((p) => upper.startsWith(p)) || - EXTRA_ENV_DENY_SUFFIXES.some((s) => upper.endsWith(s)) + EXTRA_ENV_DENY_EXACT.has(name) || + EXTRA_ENV_DENY_PREFIXES.some((p) => name.startsWith(p)) || + EXTRA_ENV_DENY_SUFFIXES.some((s) => name.endsWith(s)) ); } diff --git a/src/daemon/providers/interface.ts b/src/daemon/providers/interface.ts index 174358f6..027ae450 100644 --- a/src/daemon/providers/interface.ts +++ b/src/daemon/providers/interface.ts @@ -203,6 +203,13 @@ export interface NormalizedTurnResult { stopReason?: string; isError?: boolean; errorMessage?: string; + /** + * The backend's own result text for a turn it flagged as an error without + * an error subtype (Claude: `subtype: "success"`, `is_error: true`) — where + * the CLI puts a usage-limit or "API Error: …" notice. Text the CLI wrote, + * possibly the model's: read only with CLI-shaped rules (classifyCliNotice). + */ + resultText?: string; /** * Where this turn ends in the backend's OWN record of the conversation, for * a backend that can later fork or resume from that point natively (#387). @@ -342,7 +349,14 @@ export type ProviderEvent = * `resetsAt` (epoch ms) — the exact reset a usage-limit error's text often * gives only as a wall-clock time. Observational: nothing retries on it. */ - | { type: "rate_limit"; status: "allowed" | "allowed_warning" | "rejected"; resetsAt?: number; limitType?: string } + | { + type: "rate_limit"; + status: "allowed" | "allowed_warning" | "rejected"; + resetsAt?: number; + limitType?: string; + /** Extra usage (overage) is carrying requests past the spent window — nothing is refused. */ + overage?: boolean; + } | { type: "tool_progress"; toolName?: string; elapsedSeconds?: number } /** * Provider-authored standalone message — extension output, status cards, @@ -668,6 +682,22 @@ export interface AgentProvider { * applies the new input to what it actually executes. */ readonly appliesToolInputRewrite?: boolean; + /** + * False when, in autonomous mode, the backend runs tool calls WITHOUT + * asking canUseTool — so codeoid's autonomous tool budget (and + * `onBudgetExhausted`) cannot see, count or stop them. Codex: autonomous + * maps to `approvalPolicy: "never"`. Absent = true (every call is gated). + * A loop controller falls back to wall-time limits (supervisor.ts) there. + */ + readonly enforcesToolBudget?: boolean; + /** + * Pids of the processes this backend spawned itself (its CLI / app-server) + * that are alive now. Session.terminateBackend snapshots their whole tree + * before the teardown and kills what outlives it (process-tree.ts). Absent + * or empty = unknown (in-process backends; qwen-code, whose SDK spawns + * internally): only the teardown runs. + */ + backendProcessIds?(): Promise; dispose(): Promise; } diff --git a/src/daemon/providers/jsonrpc-stdio.ts b/src/daemon/providers/jsonrpc-stdio.ts index 7162e0d5..94613453 100644 --- a/src/daemon/providers/jsonrpc-stdio.ts +++ b/src/daemon/providers/jsonrpc-stdio.ts @@ -84,6 +84,11 @@ export class StdioJsonRpcProcess { return !this.#exited; } + /** The child's pid while it runs (process-tree kill on terminateBackend). */ + get pid(): number | undefined { + return this.#exited ? undefined : this.#proc.pid; + } + /** Client→server request; resolves with the JSON-RPC result. */ request( method: string, diff --git a/src/daemon/providers/pi/index.ts b/src/daemon/providers/pi/index.ts index 70db03a3..e64dc8a0 100644 --- a/src/daemon/providers/pi/index.ts +++ b/src/daemon/providers/pi/index.ts @@ -198,6 +198,11 @@ export class PiProvider implements SessionProvider { this.#hasQueried = value; } + async backendProcessIds(): Promise { + const pid = this.#proc?.pid; + return pid ? [pid] : []; + } + async teardown(): Promise { this.#failTurn("pi session torn down"); this.#proc?.kill(); diff --git a/src/daemon/providers/pi/rpc.ts b/src/daemon/providers/pi/rpc.ts index fdad2469..c368ad52 100644 --- a/src/daemon/providers/pi/rpc.ts +++ b/src/daemon/providers/pi/rpc.ts @@ -100,6 +100,11 @@ export class PiRpcProcess { return !this.#exited; } + /** The child's pid while it runs (process-tree kill on terminateBackend). */ + get pid(): number | undefined { + return this.#exited ? undefined : this.#proc.pid; + } + /** * Strict-LF line assembly. pi warns that Unicode-separator-splitting * readers corrupt frames — we split on `\n` only and strip a trailing diff --git a/src/daemon/providers/usage-limit.ts b/src/daemon/providers/usage-limit.ts index 496d2eb8..69471a50 100644 --- a/src/daemon/providers/usage-limit.ts +++ b/src/daemon/providers/usage-limit.ts @@ -73,9 +73,14 @@ export const CLAUDE_USAGE_LIMIT_PREFIXES: readonly string[] = [ "Your seat type doesn't include extra usage", ]; -/** Openers too generic to trust alone ("You've hit your stride"): they need a limit word after them. */ +/** + * Openers too generic to trust alone ("You've hit your stride", "You've + * reached your goal: the rate limit test passes"): the limit noun must follow + * within four words, as in every completion the CLIs compose — "session + * limit", "monthly spend limit", "team's shared budget", "usage limit". + */ const GENERIC_PREFIXES = new Set(["You've hit your", "You've reached your"]); -const LIMIT_WORD = /\b(?:limit|usage|quota|budget|credits?|spend)\b/i; +const GENERIC_TAIL = /^\s+(?:[\w'-]+\s+){0,3}(?:limit|budget)\b/i; /** * gemini-cli treats a quota error whose retry delay is at most this as @@ -221,8 +226,7 @@ function prefixRule(text: string): boolean { const at = new RegExp(`(?:^|[:\\]\\n]\\s*)${escapeRegExp(prefix)}`).exec(text); if (!at) continue; if (!GENERIC_PREFIXES.has(prefix)) return true; - const rest = text.slice(at.index + at[0].length).split("\n", 1)[0] ?? ""; - if (LIMIT_WORD.test(rest)) return true; + if (GENERIC_TAIL.test(text.slice(at.index + at[0].length))) return true; } return false; } @@ -284,6 +288,45 @@ export function classifyUsageLimit( }; } +/** + * Classify text the CLI may have written AS the turn's reply — Claude's result + * text on an errored turn, or the turn's last assistant message — with the + * CLI-shaped rules only. That text can just as well be the model's own words + * ("the API returned 429 Too Many Requests twice, so I added retries"), so the + * generic rate-limit / 429 / quota rules never apply. A notice counts only + * when the text OPENS with what the CLI composes: + * + * - one of the SDK's USAGE_LIMIT_ERROR_PREFIXES (the generic openers still + * need a limit word on their first line), or the legacy + * "Claude AI usage limit reached"; + * - "API Error:" — the CLI's prefix for an API failure it reports as the + * reply — after which the full classifier reads the rest. + */ +export function classifyCliNotice( + text: string | null | undefined, + opts: ClassifyUsageLimitOptions = {}, +): UsageLimitClassification | null { + if (!text) return null; + const t = normalize(text).trimStart(); + if (/^API Error:/.test(t)) return classifyUsageLimit(t, opts); + if (/^Claude AI usage limit reached/i.test(t) || startsWithLimitPrefix(t)) { + const now = opts.now ?? Date.now(); + const resetAt = parseResetAt(t, now, { texts: [t] }); + const provider = opts.provider ?? (t.startsWith("Claude AI") ? "claude" : undefined); + return { kind: "usage_limit", ...(resetAt !== undefined ? { resetAt } : {}), ...(provider ? { provider } : {}) }; + } + return null; +} + +function startsWithLimitPrefix(t: string): boolean { + for (const prefix of CLAUDE_USAGE_LIMIT_PREFIXES) { + if (!t.startsWith(prefix)) continue; + if (!GENERIC_PREFIXES.has(prefix)) return true; + if (GENERIC_TAIL.test(t.slice(prefix.length))) return true; + } + return false; +} + // ── Reset time ──────────────────────────────────────────────────────────────── const UNIT_MS: Record = { diff --git a/src/daemon/session.ts b/src/daemon/session.ts index 63cf89c2..f81110ac 100644 --- a/src/daemon/session.ts +++ b/src/daemon/session.ts @@ -20,8 +20,9 @@ import { type HistorySeedResult, } from "./providers/canonical.js"; import { targetContextWindow, seedBudgetChars, isPromptTooLongError } from "./providers/context-windows.js"; -import { classifyUsageLimit, type UsageLimitKind } from "./providers/usage-limit.js"; +import { classifyCliNotice, classifyUsageLimit, type UsageLimitKind } from "./providers/usage-limit.js"; import { extraEnvViolations } from "./providers/env.js"; +import { killSurvivors, snapshotTree, type ProcessEntry } from "./process-tree.js"; import { CONDUCTOR_SYSTEM_PROMPT_APPEND, isFleetSendTool } from "./fleet.js"; import type { McpSdkServerConfigWithInstance } from "@anthropic-ai/claude-agent-sdk"; import { @@ -41,6 +42,7 @@ import { type TurnOpts, type CatalogEntry, isPlaceholderModel, + BUDGET_STOP_MESSAGE, } from "./providers/interface.js"; import { createDefaultProviderRegistry, type ProviderRegistry } from "./providers/registry.js"; import { selectContextStrategy, renderSessionMap, renderRotationSeed, type ContextStrategy } from "./providers/context-strategy.js"; @@ -111,6 +113,9 @@ const FORK_SETUP_TIMEOUT_MS = 600_000; * the overflow is surfaced as an error. */ const SEED_OVERFLOW_MAX_RETRIES = 3; + +/** How long a hard stop (stall recovery, terminateBackend) waits for the backend's teardown. */ +const HARD_STOP_TEARDOWN_MS = 10_000; import { authToIdentity, CAPABILITIES, @@ -490,17 +495,27 @@ export interface SessionCreateOptions { * session wedge at waiting_approval. * - `"end_attempt"`: for an unattended worker whose controller owns the * next step. The call that finds the budget spent is DENIED with a clear - * message, the turn is interrupted, the session settles idle (never - * waiting_approval for that call), the mode reverts to guarded, and - * `budgetExhausted` reads true until a new autonomous budget is set. + * message, the turn is interrupted, background tasks are stopped where the + * backend can, the session settles idle (never waiting_approval for that + * call), the mode reverts to guarded, and `budgetExhausted` reads true. + * Until a new autonomous budget is set (setMode), EVERY tool call is + * denied and ends its turn — including later prompts and turns the + * backend starts on its own. Only enforceable where `toolBudgetEnforced` + * (not codex); elsewhere a warning is logged and wall time must bound it. */ onBudgetExhausted?: "park" | "end_attempt"; /** * Extra environment for the backend's child process — every process-backed * provider (Claude Code CLI, codex, gemini-cli, pi, qwen-code). Merged after * codeoid builds the backend's env, but a key in codeoid's credential / - * account-isolation denylist (`extraEnvViolations`, providers/env.ts) is + * account-isolation / code-loading denylist (`extraEnvViolations`, + * providers/env.ts), or any name that isn't `^[A-Z][A-Z0-9_]{0,63}$`, is * refused: the constructor throws, so a misconfigured caller learns at once. + * + * DAEMON-CONTROLLED ONLY. The denylist is a backstop, not an authorization: + * a name a user (or a contract, or an agent) supplies must first pass an + * explicit allowlist the daemon owns — a later slice binds it to the trusted + * setup recipe's declared `resources.env`. Never forward client input here. * No effect on in-process backends (openai, gemini, llama.cpp, endpoints). * Held in memory only, by design — never persisted (it can carry a run's * secrets-adjacent settings, and a stale copy must not outlive the run). A @@ -1050,11 +1065,14 @@ export class Session { #turnsRemaining: number | undefined = undefined; /** See SessionCreateOptions.onBudgetExhausted. */ #onBudgetExhausted: "park" | "end_attempt" = "park"; - /** end_attempt only: the autonomous budget ran out; reset by a new autonomous budget (setMode). */ + /** + * end_attempt only: the autonomous budget ran out. While set, EVERY tool + * call is denied — in the turn it ran out in, a later prompted turn, or a + * turn the backend starts on its own — until a new autonomous budget is set + * (setMode). Only an explicit re-arm clears it, never a turn boundary. + */ #budgetExhausted = false; - /** end_attempt only: the turn the budget ran out in is being ended — deny every further call in it. */ - #endingAttempt = false; - /** The ending turn's interrupt has been scheduled (once per ending). */ + /** This turn's end-attempt interrupt has been scheduled (once per turn). */ #endAttemptInterruptQueued = false; /** See SessionCreateOptions.extraEnv (validated in the constructor). */ #extraEnv: Readonly> | undefined; @@ -1426,6 +1444,7 @@ export class Session { : (opts.config?.session.defaultEffort ?? null); this.#provider = opts._testProvider ?? this.#createProvider(opts.providerId); + this.#warnIfBudgetUnenforced(); this.#appliedAccount = this.#accountState(this.#provider.id); this.#provider.onSessionEvent = (e) => this.#onSessionScopedEvent(e); @@ -1927,6 +1946,7 @@ export class Session { } this.#provider = this.#createProvider(requested); + this.#warnIfBudgetUnenforced(); this.#effortRestartPending = false; // the new backend starts with the session's effort this.#appliedAccount = this.#accountState(requested); this.#resetMcpState({ backendChanged: true }); @@ -2032,7 +2052,7 @@ export class Session { /** * The most recent finished turn's failure, when the backend blamed a usage or * rate limit (with its reset time when it gave one); null when that turn - * failed for another reason, and cleared by the next successful turn. A loop + * failed for another reason, succeeded, or a new turn has started. A loop * controller reads it to record an infrastructure error that spends no * attempt. Retry behavior is unchanged — this only observes. */ @@ -2040,7 +2060,8 @@ export class Session { /** * True once the autonomous tool budget ran out on a session created with * `onBudgetExhausted: "end_attempt"` — the attempt was ended, not parked. - * Stays true until a new autonomous budget is set (setMode). + * Stays true, and every tool call is denied, until a new autonomous budget + * is set (setMode). */ get budgetExhausted(): boolean { return this.#budgetExhausted; } /** @@ -2488,6 +2509,22 @@ export class Session { get agentUri(): string | undefined { return this.#agentIdentity.sub; } get mode(): SessionMode { return this.#mode; } get turnsRemaining(): number | undefined { return this.#turnsRemaining; } + /** + * Whether this session's backend asks codeoid before every tool call in + * autonomous mode, so the tool budget (and `onBudgetExhausted`) actually + * bounds it. False on codex (SessionProvider.enforcesToolBudget): there a + * controller must bound the attempt by wall time (supervisor.ts) instead. + * Read it again after a backend switch. + */ + get toolBudgetEnforced(): boolean { return this.#provider.enforcesToolBudget !== false; } + + /** end_attempt on a backend that can't count tool calls: not refused, but said. */ + #warnIfBudgetUnenforced(): void { + if (this.#onBudgetExhausted !== "end_attempt" || this.toolBudgetEnforced) return; + console.warn( + `[codeoid/session ${this.id}] onBudgetExhausted "end_attempt" requested, but the ${this.#provider.id} backend runs autonomous tool calls without asking codeoid — the tool budget can't be enforced; bound this attempt by wall time (supervise) instead`, + ); + } get pinnedFiles(): readonly string[] { return this.#pinnedFiles; } get model(): string | null { return this.#model; } get fallbackModel(): string | null { return this.#fallbackModel; } @@ -3095,25 +3132,33 @@ export class Session { /** Per-turn loop controls, reset when a turn starts. */ #resetTurnControls(): void { - this.#endingAttempt = false; this.#endAttemptInterruptQueued = false; this.#rateLimitRejection = null; + // It describes the last FINISHED turn; a running one has not failed yet. + this.#lastTurnInfraError = null; } /** * A turn failed: record whether a usage or rate limit was the cause - * (lastTurnInfraError). `sources` are what the turn reported, most - * authoritative first — the error, then (when this turn wrote it) the - * assistant text the Claude CLI puts its limit message in. Any failure - * overwrites the previous record, so it always describes the latest turn. - * Observation only: nothing here retries or changes status. - */ - #recordTurnFailure(sources: readonly unknown[]): void { + * (lastTurnInfraError). Any failure overwrites the previous record, so it + * always describes the latest turn. Observation only: nothing here retries + * or changes status. + * + * - `errors`: what the backend reported AS the failure (error text, an + * Error, a structured error) — read with the full classifier. + * - `cliTexts`: text the turn's reply may hold a CLI notice in (Claude's + * result text on an is_error turn, the turn's own assistant text). It + * can be the model's words, so only CLI-shaped rules apply (classifyCliNotice). + * - `explained`: another recognizer already knows the cause (prompt too + * long, the spend ceiling) — then a rate_limit_event alone never makes it + * a usage limit. + */ + #recordTurnFailure( + errors: readonly unknown[], + opts: { cliTexts?: readonly (string | undefined)[]; explained?: boolean } = {}, + ): void { const provider = this.#provider.id; - for (const source of sources) { - if (source == null || source === "") continue; - const hit = classifyUsageLimit(source, { provider }); - if (!hit) continue; + const record = (hit: { kind: UsageLimitKind; resetAt?: number }, source: unknown) => { // The text often says only "resets 3pm"; Claude's rate_limit_event in // the same turn carries the exact instant. const resetAt = hit.resetAt ?? this.#rateLimitRejection?.resetAt; @@ -3124,13 +3169,28 @@ export class Session { provider, at: Date.now(), }; - return; + }; + for (const source of errors) { + if (source == null || source === "") continue; + const hit = classifyUsageLimit(source, { provider }); + if (hit) { + record(hit, source); + return; + } } - // No text said so, but the backend reported this turn's request rejected - // by a plan limit (Claude's rate_limit_event, status "rejected"). + for (const text of opts.cliTexts ?? []) { + const hit = classifyCliNotice(text, { provider }); + if (hit) { + record(hit, text); + return; + } + } + // Nothing said so, but the backend reported this turn's request refused + // by a plan limit (Claude's rate_limit_event "rejected", not carried by + // overage) — and no other recognizer explains the failure. const rejection = this.#rateLimitRejection; - if (rejection) { - const first = sources.find((s) => s != null && s !== ""); + if (rejection && !opts.explained) { + const first = errors.find((e) => e != null && e !== ""); this.#lastTurnInfraError = { kind: "usage_limit", ...(rejection.resetAt !== undefined ? { resetAt: rejection.resetAt } : {}), @@ -3182,7 +3242,6 @@ export class Session { // A new turn is starting — clear any interrupt from the previous one. this.#turnInterrupted = false; this.#lastTurnError = null; - this.#resetTurnControls(); // Any inbound message invalidates a repeat run: the guard only claims // "N identical calls with nothing else happening", and this is something // else happening. Applies to system principals too (a background-task @@ -3258,6 +3317,9 @@ export class Session { const joinsRunningTurn = wasWorking && !!this.#activeRun?.pushMidTurn && this.#currentTurnId !== null; const previousTurnId = this.#currentTurnId; let turnId = joinsRunningTurn ? (previousTurnId as string) : this.#beginTurn(); + // Per-turn controls belong to a turn that STARTS here, not to one this + // message only joins (its rate-limit record and interrupt are still live). + if (!joinsRunningTurn) this.#resetTurnControls(); const turnAt = new Date().toISOString(); try { const userMsg = this.#makeMessage( @@ -4082,6 +4144,11 @@ export class Session { } catch { // fall through — the provider's own fallback is a hard abort } + // While we awaited, the turn may have ended and another started (a + // queued send, a backend-adopted turn, a terminateBackend). Its + // approvals and status are not this interrupt's to settle; an ended + // run's consumer already settled its own. + if (this.#activeRun !== run) return; // Tool cards are NOT closed here: the interrupted turn is still // draining, and a result that lands after interrupt() resolves must // still reach its card, the transcript and memory. The consumer's turn @@ -6028,17 +6095,19 @@ export class Session { * * The interrupt is scheduled, not awaited: the deny must reach the backend * first, and interrupt() awaits the backend, which is waiting on this gate. - * Only the first denial interrupts; later calls in the draining turn are - * simply refused. + * Only a turn's first denial interrupts it; later calls in the draining + * turn are simply refused. Background tasks are stopped too where the + * backend can (Claude: SDK `stopTask`); on a backend without + * stopBackgroundTasks they keep running, but every tool call they make is + * still denied here. */ #endAttemptOnBudget(approvalId: string, toolName: string, sender: AuthContext): { behavior: "deny"; message: string } { - if (!this.#endingAttempt) { + if (!this.#budgetExhausted) { // Same reversion as park (reads frictionless, writes ask again if a // person takes over); flagged after setMode, which clears the flag only // for a NEW autonomous budget. this.setMode("guarded"); this.#budgetExhausted = true; - this.#endingAttempt = true; } const reason = "The autonomous tool budget for this attempt is spent, so the attempt has ended. Do not retry; the controller decides what happens next."; this.#resolveToolCallMessage(approvalId, { phase: "cancelled", reason: "denied", message: reason }); @@ -6065,7 +6134,15 @@ export class Session { }; setTimeout(() => { if (this.#destroyed) return; - this.interrupt(system).catch((e) => { + void (async () => { + await this.interrupt(system); + // interrupt() spares background agents while a turn runs; an ended + // attempt must not leave them working on the spent budget. + if (this.#backgroundTasks.size > 0 && this.#provider.stopBackgroundTasks) { + await this.#provider.stopBackgroundTasks([...this.#backgroundTasks.keys()]); + this.#clearBackgroundTasks(); + } + })().catch((e) => { console.error(`[codeoid/session ${this.id}] ending the attempt on a spent budget failed: ${e instanceof Error ? e.message : String(e)}`); }); }, 0); @@ -6090,8 +6167,8 @@ export class Session { if (!this.#turnActive()) await this.#backgroundEventChain; // The attempt already ended on a spent budget (end_attempt): nothing - // else runs in it — not even a read — while the interrupt lands. - if (this.#endingAttempt) return this.#endAttemptOnBudget(approvalId, toolName, sender); + // runs — not even a read — until a new budget re-arms the session. + if (this.#budgetExhausted) return this.#endAttemptOnBudget(approvalId, toolName, sender); // Hook gate — the policy layer, run BEFORE the approval gate. A hook // block is a policy deny that never prompts the user (and never burns @@ -6581,10 +6658,19 @@ export class Session { * ran out and the grace period passed). Whatever is in flight — a tool * without its own timeout, a wedged turn, background agents — is ended by * tearing the backend down (its child process is killed), exactly as stall - * recovery does. The session survives and settles idle; the next send starts - * a fresh backend and resumes the conversation. Idempotent. - */ - async terminateBackend(sender: AuthContext, reason: string): Promise { + * recovery does — and then killing whatever that process had started and + * left running (process-tree.ts: the tree under the backend's own pids is + * snapshotted first, survivors are SIGKILLed after the bounded teardown). + * Self-detached commands and backends with unknown pids (qwen) are outside + * that reach; see process-tree.ts for exactly what survives. The session + * survives and settles idle; the next send starts a fresh backend and + * resumes the conversation. Idempotent. + */ + async terminateBackend( + sender: AuthContext, + reason: string, + opts: { teardownTimeoutMs?: number } = {}, + ): Promise { this.#store.audit(sender.sub, "session.terminate_backend", this.id, reason); // Stop sends still preparing their turn too, as interrupt() does. this.#turnInterrupted = true; @@ -6592,10 +6678,34 @@ export class Session { this.#cancelAllUiRequests("interrupted"); this.#earlyApprovals.clear(); this.#approvalPatchKeys.clear(); - await this.#hardStop("backend terminated", { - text: `⏹ Backend stopped by ${sender.sub}: ${reason}. Send a new message to continue.`, - meta: { event: "session.backend_terminated", sub: sender.sub }, - }); + // Before the teardown: once the backend is gone its children re-parent + // and can no longer be told apart from anyone else's. + let tree: ProcessEntry[] = []; + try { + const roots = (await withTimeout(this.#provider.backendProcessIds?.() ?? Promise.resolve([]), 3_000)) ?? []; + tree = await withTimeout(snapshotTree(roots), 3_000); + } catch (e) { + console.error(`[codeoid/session ${this.id}] could not snapshot the backend's processes: ${e instanceof Error ? e.message : String(e)}`); + } + await this.#hardStop( + "backend terminated", + { + text: `⏹ Backend stopped by ${sender.sub}: ${reason}. Send a new message to continue.`, + meta: { event: "session.backend_terminated", sub: sender.sub }, + }, + opts.teardownTimeoutMs, + ); + // The teardown had its graceful chance (and the backend its own + // escalation); what is still running from the snapshot is left over. + try { + const killed = await killSurvivors(tree); + if (killed.length > 0) { + this.#store.audit(sender.sub, "session.terminate_backend.reaped", this.id, `pids=${killed.join(",")}`); + console.warn(`[codeoid/session ${this.id}] killed ${killed.length} process(es) the backend left running: ${killed.join(", ")}`); + } + } catch (e) { + console.error(`[codeoid/session ${this.id}] killing the backend's leftover processes failed: ${e instanceof Error ? e.message : String(e)}`); + } } /** @@ -6607,7 +6717,11 @@ export class Session { * abandoned consumer (if any) unblocks when teardown closes the turn queue * and its finally no-ops via the run guard. */ - async #hardStop(label: string, notice: { text: string; meta: Record }): Promise { + async #hardStop( + label: string, + notice: { text: string; meta: Record }, + teardownTimeoutMs = HARD_STOP_TEARDOWN_MS, + ): Promise { // Stop any spinning UI and release waiters BEFORE we tear down. The // teardown below kills the CLI and every background agent with it, so // nothing is kept: the whole background set is dead. @@ -6631,8 +6745,11 @@ export class Session { // Hard teardown: abort the controller and reap the (presumed hung) // subprocess. Safe to await here — provider.teardown() awaits the // PROVIDER's own pump, never this session's consumer. + // Bounded: a backend that won't acknowledge its teardown must not hang + // the caller (a supervisor's `expired` waits on this). The run slots are + // already cleared, so the session is usable either way. try { - await this.#provider.teardown(); + await withTimeout(this.#provider.teardown(), teardownTimeoutMs); } catch (e) { console.error( `[codeoid/session ${this.id}] provider teardown during ${label} failed: ${e instanceof Error ? e.message : String(e)}`, @@ -6771,7 +6888,7 @@ export class Session { secretPathVerdict(event.name, event.input, { workdir: this.workdir, configDir: getConfigDir() }) === null && this.#peekAutoApprove(event.name); // end_attempt: the gate will refuse this call outright — no approval card. - const budgetDenied = this.#endingAttempt || this.#budgetSpentFor(event.name); + const budgetDenied = this.#budgetExhausted || this.#budgetSpentFor(event.name); const toolMsg = this.#makeMessage( "tool_call", `${event.name}(${Object.keys(event.input).join(", ")})`, @@ -6999,8 +7116,9 @@ export class Session { case "rate_limit": { // Kept for #recordTurnFailure: if this turn fails, the reset is exact. // A later "allowed" in the same turn means the rejection was retried past. + // Overage carrying the requests means nothing was refused. this.#rateLimitRejection = - event.status === "rejected" ? (event.resetsAt !== undefined ? { resetAt: event.resetsAt } : {}) : null; + event.status === "rejected" && !event.overage ? (event.resetsAt !== undefined ? { resetAt: event.resetsAt } : {}) : null; break; } @@ -7058,12 +7176,13 @@ export class Session { } this.#lastTurnError = errText; // Before the status change below, like lastTurnError: an error - // waiter reads it. The CLI puts a limit message in the turn's - // assistant text (only if THIS turn committed it, as above). - // Short only: the CLI's limit notice is the whole reply, and a long - // reply that happens to discuss rate limits is the model's work. - const notice = last?.role === "assistant" && last.content.length <= 500 ? last.content : undefined; - this.#recordTurnFailure([event.result.errorMessage, notice, errText]); + // waiter reads it. The CLI puts a limit notice in the result text + // or the turn's own assistant text (only if THIS turn committed it, + // as above) — CLI-shaped rules only there, never the generic ones. + this.#recordTurnFailure([event.result.errorMessage], { + cliTexts: [event.result.resultText, last?.role === "assistant" ? last.content : undefined], + explained: tooLong || event.result.errorMessage === BUDGET_STOP_MESSAGE, + }); const hint = tooLong ? " — this session's context no longer fits the model. Send /rotate to continue in a fresh context (memory recall keeps the history reachable)." : ""; diff --git a/src/daemon/supervisor.ts b/src/daemon/supervisor.ts index 13719f68..5456f31b 100644 --- a/src/daemon/supervisor.ts +++ b/src/daemon/supervisor.ts @@ -29,7 +29,7 @@ export interface SupervisedSession { /** A queued send is about to start a turn — not settled even when idle. */ readonly preparingTurn: boolean; interrupt(sender: AuthContext): Promise; - terminateBackend(sender: AuthContext, reason: string): Promise; + terminateBackend(sender: AuthContext, reason: string, opts?: { teardownTimeoutMs?: number }): Promise; } export interface DeadlineOptions { @@ -39,6 +39,8 @@ export interface DeadlineOptions { graceMs: number; /** Who the interrupt and termination are audited as. Default `system:supervisor` in the session's tenant. */ sender?: AuthContext; + /** Bound on the backend's teardown when it is killed (Session default 10s), so `expired` cannot hang. */ + teardownTimeoutMs?: number; /** How often the grace period re-checks for a settled session. Default 100ms (bounded by graceMs). */ pollMs?: number; /** Observes each step (logging, tests). Exceptions are swallowed. */ @@ -114,7 +116,11 @@ export function supervise(session: SupervisedSession, opts: DeadlineOptions): De if (done) return; note({ type: "terminated", sessionId: session.id }); session - .terminateBackend(sender, `attempt wall time of ${Math.round(wallMs / 1000)}s ran out and it did not stop within ${Math.round(graceMs / 1000)}s`) + .terminateBackend( + sender, + `attempt wall time of ${Math.round(wallMs / 1000)}s ran out and it did not stop within ${Math.round(graceMs / 1000)}s`, + opts.teardownTimeoutMs !== undefined ? { teardownTimeoutMs: opts.teardownTimeoutMs } : {}, + ) .catch((e) => { console.error(`[codeoid/supervisor] terminating session ${session.id} failed: ${e instanceof Error ? e.message : String(e)}`); }) diff --git a/src/tests/process-tree.test.ts b/src/tests/process-tree.test.ts new file mode 100644 index 00000000..869745d4 --- /dev/null +++ b/src/tests/process-tree.test.ts @@ -0,0 +1,133 @@ +/** + * Tree kill after a backend teardown (process-tree.ts, Session.terminateBackend). + * + * The integration case is the one measured against the Claude CLI and the one + * codex / gemini-cli hit on every teardown: the backend process is SIGKILLed, + * and the command its tool was running is left behind, re-parented. The + * snapshot taken before the teardown is what still identifies it. + */ + +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { spawn, type ChildProcess } from "node:child_process"; +import { randomUUID } from "node:crypto"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { killSurvivors, listProcesses, parsePs, snapshotTree, treeOf } from "../daemon/process-tree.js"; +import { MockSessionProvider } from "../daemon/providers/mock/session-provider.js"; +import { Session } from "../daemon/session.js"; +import { Store } from "../daemon/store.js"; +import { TranscriptStore } from "../daemon/transcript.js"; +import type { AuthContext } from "../protocol/types.js"; + +const AUTH: AuthContext = { sub: "user:tree", scopes: [], delegationDepth: 0, accountId: "acc", projectId: "proj" }; + +describe("parsePs / treeOf", () => { + const table = parsePs([" 10 1 /sbin/init", " 20 10 backend --serve", " 30 20 /bin/sh -c sleep 99", " 31 30 sleep 99", " 40 10 other", "garbage"].join("\n")); + + test("parses pid, ppid and the full command line", () => { + expect(table).toHaveLength(5); + expect(table[2]).toEqual({ pid: 30, ppid: 20, args: "/bin/sh -c sleep 99" }); + }); + + test("the tree under a root, root included, nothing else", () => { + expect(treeOf(table, [20]).map((p) => p.pid)).toEqual([20, 30, 31]); + expect(treeOf(table, [99])).toEqual([]); + }); + + test("never includes this process", () => { + const self = parsePs(`${process.pid} 1 bun test\n77 ${process.pid} child`); + expect(treeOf(self, [process.pid]).map((p) => p.pid)).toEqual([]); + }); +}); + +const alive = (pid: number): boolean => { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +}; + +async function until(cond: () => boolean | Promise, ms = 5_000): Promise { + const deadline = Date.now() + ms; + while (!(await cond())) { + if (Date.now() > deadline) throw new Error("condition not reached"); + await new Promise((r) => setTimeout(r, 20)); + } +} + +/** A "backend": a shell running a long command in the background, like a tool call. */ +function spawnBackend(mark: string): Promise<{ proc: ChildProcess; grandchild: number }> { + const proc = spawn("sh", ["-c", `sleep ${mark} & echo $!; wait`], { stdio: ["ignore", "pipe", "ignore"] }); + return new Promise((resolve, reject) => { + proc.stdout!.once("data", (d: Buffer) => resolve({ proc, grandchild: Number(String(d).trim()) })); + proc.once("error", reject); + }); +} + +/** A backend whose teardown SIGKILLs its process, as codex / gemini-cli do. */ +class ProcessBackend extends MockSessionProvider { + proc: ChildProcess | null = null; + async backendProcessIds(): Promise { + return this.proc?.pid ? [this.proc.pid] : []; + } + override async teardown(): Promise { + this.teardownCount++; + this.proc?.kill("SIGKILL"); + } +} + +describe.skipIf(process.platform === "win32")("killing what a backend left running", () => { + let tmp: string; + let store: Store; + let transcriptStore: TranscriptStore; + const leftovers: number[] = []; + beforeEach(() => { + tmp = mkdtempSync(join(tmpdir(), "codeoid-tree-")); + store = new Store(join(tmp, "codeoid.db")); + transcriptStore = new TranscriptStore(join(tmp, "transcripts")); + }); + afterEach(async () => { + for (const pid of leftovers.splice(0)) try { process.kill(pid, "SIGKILL"); } catch {} + await new Promise((r) => setTimeout(r, 50)); + try { await transcriptStore.flush(); } catch {} + try { store.close(); } catch {} + try { rmSync(tmp, { recursive: true, force: true }); } catch {} + }); + + test("killSurvivors: the re-parented grandchild is killed; a pid whose command changed is spared", async () => { + const mark = String(800000 + Math.floor(Math.random() * 99999)); + const { proc, grandchild } = await spawnBackend(mark); + leftovers.push(grandchild); + const snap = await snapshotTree([proc.pid!]); + expect(snap.map((p) => p.pid)).toContain(grandchild); + proc.kill("SIGKILL"); + await until(async () => !(await listProcesses()).some((p) => p.pid === proc.pid && !p.args.includes("defunct") && p.args.startsWith("sh"))); + expect(alive(grandchild)).toBe(true); // the teardown alone leaves it + const forged = snap.map((p) => (p.pid === grandchild ? p : { ...p, args: `${p.args} (other)` })); + expect(await killSurvivors(forged)).toEqual([grandchild]); + await until(() => !alive(grandchild)); + }); + + test("Session.terminateBackend snapshots before the teardown and kills the leftover", async () => { + const mark = String(800000 + Math.floor(Math.random() * 99999)); + const backend = new ProcessBackend("codex", [[{ type: "tool_start", toolId: "t1", sdkToolUseId: "s1", name: "Bash", input: { command: "x" }, approvalId: "a1" }]], { stall: true }); + const { proc, grandchild } = await spawnBackend(mark); + leftovers.push(grandchild); + backend.proc = proc; + const id = randomUUID(); + store.createSession({ + id, name: "tree", workdir: tmp, status: "idle", createdBy: AUTH.sub, + createdAt: new Date().toISOString(), attachedClients: 0, accountId: AUTH.accountId, projectId: AUTH.projectId, + }); + const session = new Session({ name: "tree", workdir: tmp, auth: AUTH, store, transcriptStore, existingId: id, _testProvider: backend, initialMode: { mode: "autonomous" } }); + await session.send("work", AUTH); + await until(() => session.status === "tool_running"); + await session.terminateBackend(AUTH, "test"); + expect(backend.teardownCount).toBe(1); + await until(() => !alive(grandchild)); + expect(session.status).toBe("idle"); + }); +}); diff --git a/src/tests/provider-claude.test.ts b/src/tests/provider-claude.test.ts index 4fff0a7a..2220a380 100644 --- a/src/tests/provider-claude.test.ts +++ b/src/tests/provider-claude.test.ts @@ -327,6 +327,25 @@ describe("translateSDKMessage – result", () => { }); }); + it("carries the result text of a success-subtype is_error turn as resultText (where the CLI puts a limit notice)", () => { + const events = collectEmits({ + type: "result", + subtype: "success", + is_error: true, + result: "You've hit your session limit · resets 3pm (UTC)", + usage: { input_tokens: 0, output_tokens: 0 }, + modelUsage: {}, + }); + const result = (events[0] as { result: { resultText?: string; errorMessage?: string } }).result; + expect(result.resultText).toBe("You've hit your session limit · resets 3pm (UTC)"); + expect(result.errorMessage).toBeUndefined(); + }); + + it("no resultText on a turn that did not fail", () => { + const events = collectEmits({ type: "result", subtype: "success", result: "all done", usage: {}, modelUsage: {} }); + expect((events[0] as { result: { resultText?: string } }).result.resultText).toBeUndefined(); + }); + it("sets errorMessage when subtype is not 'success' and result is set", () => { const events = collectEmits({ type: "result", subtype: "error_during_execution", result: "tool failed" }); expect(events[0]).toMatchObject({ type: "turn_done", result: { errorMessage: "tool failed" } }); @@ -1338,6 +1357,15 @@ describe("translateSDKMessage – rate_limit_event", () => { expect(events).toEqual([{ type: "rate_limit", status: "rejected", resetsAt: 1_800_000_000_000, limitType: "five_hour" }]); }); + it("flags a rejection that overage is carrying", () => { + for (const info of [{ isUsingOverage: true }, { overageInUse: true }, { overageStatus: "allowed" }, { overageStatus: "allowed_warning" }]) { + const events = collectEmits({ type: "rate_limit_event", rate_limit_info: { status: "rejected", ...info } }); + expect(events[0]).toMatchObject({ type: "rate_limit", status: "rejected", overage: true }); + } + const plain = collectEmits({ type: "rate_limit_event", rate_limit_info: { status: "rejected", overageStatus: "rejected" } }); + expect((plain[0] as { overage?: boolean }).overage).toBeUndefined(); + }); + it("emits nothing for a status it doesn't know", () => { expect(collectEmits({ type: "rate_limit_event", rate_limit_info: { status: "mystery" } })).toHaveLength(0); }); diff --git a/src/tests/provider-env.test.ts b/src/tests/provider-env.test.ts index b2af3973..4c38b890 100644 --- a/src/tests/provider-env.test.ts +++ b/src/tests/provider-env.test.ts @@ -185,7 +185,7 @@ describe("extraEnv denylist (SessionCreateOptions.extraEnv)", () => { const DENIED = [ "HOME", "USER", "LOGNAME", "PATH", "SHELL", "XDG_CONFIG_HOME", "XDG_DATA_HOME", "NODE_OPTIONS", "NODE_PATH", "LD_PRELOAD", "LD_LIBRARY_PATH", "DYLD_INSERT_LIBRARIES", - "HTTPS_PROXY", "https_proxy", "NODE_EXTRA_CA_CERTS", "NODE_TLS_REJECT_UNAUTHORIZED", "SSL_CERT_FILE", + "HTTPS_PROXY", "NODE_EXTRA_CA_CERTS", "NODE_TLS_REJECT_UNAUTHORIZED", "SSL_CERT_FILE", "ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "ANTHROPIC_BASE_URL", "ANTHROPIC_CUSTOM_HEADERS", "CLAUDE_CODE_OAUTH_TOKEN", "CLAUDE_CONFIG_DIR", "CLAUDE_CODE_USE_BEDROCK", "AWS_BEARER_TOKEN_BEDROCK", "AWS_PROFILE", "OTEL_EXPORTER_OTLP_ENDPOINT", @@ -193,26 +193,43 @@ describe("extraEnv denylist (SessionCreateOptions.extraEnv)", () => { "GEMINI_API_KEY", "GEMINI_CLI_HOME", "GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_CLOUD_PROJECT", "CODE_ASSIST_ENDPOINT", "QWEN_HOME", "DASHSCOPE_API_KEY", "BAILIAN_API_KEY", "PI_CODING_AGENT_DIR", "CODEOID_API_KEY", "CODEOID_AGENT_ENV_ALLOW", "CODEOID_MEMORY_TOKEN", "ZEROID_URL", "TELEGRAM_BOT_TOKEN", - "GROQ_API_KEY", "MISTRAL_API_TOKEN", "SOME_AUTH_TOKEN", "VENDOR_ACCESS_TOKEN", "vendor_api_key", + "GROQ_API_KEY", "MISTRAL_API_TOKEN", "SOME_AUTH_TOKEN", "VENDOR_ACCESS_TOKEN", + // Tool configuration that can run code, and runtime knobs. + "RIPGREP_CONFIG_PATH", "BASH_ENV", "ENV", "PROMPT_COMMAND", "PAGER", "GIT_PAGER", "LESSOPEN", "LESSCLOSE", + "EDITOR", "VISUAL", "GCONV_PATH", "RUST_LOG", "RUST_BACKTRACE", "TMPDIR", "TEMP", "TMP", "TZ", + "GIT_CONFIG_GLOBAL", "GIT_SSH_COMMAND", "NPM_CONFIG_USERCONFIG", "PYTHONPATH", "PYTHONSTARTUP", "RUBYOPT", "RUBYLIB", + "PERL5LIB", "PERL5OPT", "PERL_MB_OPT", "JAVA_TOOL_OPTIONS", "_JAVA_OPTIONS", "JDK_JAVA_OPTIONS", "CLOUDSDK_CONFIG", "LC_ALL", ]; - const ALLOWED = ["LOOP_RUN_ID", "DATABASE_URL", "CI", "NODE_ENV", "RUST_LOG", "GITHUB_TOKEN", "TZ", "LANG", "MY_FLAG"]; + const ALLOWED = ["LOOP_RUN_ID", "DATABASE_URL", "CI", "NODE_ENV", "GITHUB_TOKEN", "LANG", "MY_FLAG", "X"]; it("refuses every denylisted key and none of the ordinary ones", () => { const extra = Object.fromEntries([...DENIED, ...ALLOWED].map((k) => [k, "x"])); expect(extraEnvViolations(extra).sort()).toEqual([...DENIED].sort()); }); - it("refuses malformed names and values", () => { - expect(extraEnvViolations({ "BAD-NAME": "x", "1X": "x", OK: "x", NUL: "a\0b", NUM: 1 as unknown as string })).toEqual([ - "BAD-NAME", "1X", "NUL", "NUM", - ]); + it("refuses malformed names and values, and every lower-case or over-long name", () => { + expect( + extraEnvViolations({ + "BAD-NAME": "x", + "1X": "x", + OK: "x", + NUL: "a\0b", + NUM: 1 as unknown as string, + http_proxy: "x", + npm_config_registry: "x", + Mixed_Case: "x", + _LEADING: "x", + ["A".repeat(65)]: "x", + ["A".repeat(64)]: "x", + }), + ).toEqual(["BAD-NAME", "1X", "NUL", "NUM", "http_proxy", "npm_config_registry", "Mixed_Case", "_LEADING", "A".repeat(65)]); }); it("withExtraEnv merges allowed keys over the built env and keeps codeoid's value for refused ones", () => { - const built = { PATH: "/usr/bin", HOME: "/home/u", ANTHROPIC_API_KEY: "ant-key", TZ: "UTC" }; - const merged = withExtraEnv(built, { PATH: "/evil", HOME: "/evil", ANTHROPIC_API_KEY: "evil", TZ: "Asia/Kolkata", LOOP_RUN_ID: "r1" }); - expect(merged).toEqual({ PATH: "/usr/bin", HOME: "/home/u", ANTHROPIC_API_KEY: "ant-key", TZ: "Asia/Kolkata", LOOP_RUN_ID: "r1" }); - expect(built.TZ).toBe("UTC"); // pure + const built = { PATH: "/usr/bin", HOME: "/home/u", ANTHROPIC_API_KEY: "ant-key", TZ: "UTC", LOOP_FLAG: "a" }; + const merged = withExtraEnv(built, { PATH: "/evil", HOME: "/evil", ANTHROPIC_API_KEY: "evil", TZ: "Asia/Kolkata", LOOP_FLAG: "b", LOOP_RUN_ID: "r1" }); + expect(merged).toEqual({ PATH: "/usr/bin", HOME: "/home/u", ANTHROPIC_API_KEY: "ant-key", TZ: "UTC", LOOP_FLAG: "b", LOOP_RUN_ID: "r1" }); + expect(built.LOOP_FLAG).toBe("a"); // pure expect(withExtraEnv(built, undefined)).toBe(built); }); diff --git a/src/tests/session-loop-controls.test.ts b/src/tests/session-loop-controls.test.ts index 5a7358ac..655b1f1b 100644 --- a/src/tests/session-loop-controls.test.ts +++ b/src/tests/session-loop-controls.test.ts @@ -347,3 +347,210 @@ describe("supervise() — timers", () => { expect(() => supervise(s, { wallMs: 10, graceMs: -1 })).toThrow(RangeError); }); }); + +// ── Round-1 audit fixes ────────────────────────────────────────────────────── + +const rejected = (extra: Partial> = {}): ProviderEvent => ({ + type: "rate_limit", + status: "rejected", + resetsAt: Date.now() + 3600_000, + ...extra, +}); + +describe("rate_limit fallback (audit #1)", () => { + test("a rejection that overage is carrying is not a limit", async () => { + const provider = new MockSessionProvider("claude", [[rejected({ overage: true }), done({ isError: true, errorMessage: "tool crashed" })]]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).toBeNull(); + }); + + for (const [label, errorMessage] of [ + ["prompt too long", "Prompt is too long"], + ["the spend ceiling", "Stopped at the spend ceiling for this run (maxBudgetUsd)."], + ] as const) { + test(`a rejection does not relabel a failure already explained by ${label}`, async () => { + const provider = new MockSessionProvider("claude", [[rejected(), done({ isError: true, errorMessage })]]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).toBeNull(); + }); + } + + test("an unexplained failure after a plain rejection is still a usage limit", async () => { + const r = rejected(); + const provider = new MockSessionProvider("claude", [[r, done({ isError: true })]]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).toMatchObject({ kind: "usage_limit", resetAt: (r as { resetsAt: number }).resetsAt }); + }); +}); + +describe("CLI notice in result / assistant text (audit #2)", () => { + test("Claude's result text on an is_error turn is classified", async () => { + const provider = new MockSessionProvider("claude", [ + [done({ isError: true, resultText: "You've hit your weekly limit · resets Oct 28, 9am (UTC)" })], + ]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError?.kind).toBe("usage_limit"); + }); + + for (const reply of [ + "The upstream API returned 429 Too Many Requests twice, so I added retries.", + "HTTP 429 handler added; tests pass", + "You've reached your goal: the rate limit test passes", + ]) { + test(`a model reply in a failed turn is not a limit: ${JSON.stringify(reply)}`, async () => { + const provider = new MockSessionProvider("claude", [[{ type: "text_done", content: reply }, done({ isError: true, resultText: reply })]]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).toBeNull(); + }); + } +}); + +describe("turn-start clearing (audit #3, #5)", () => { + test("lastTurnInfraError is cleared when the next turn starts, not when it ends", async () => { + const provider = new MockSessionProvider("claude", [ + [done({ isError: true, errorMessage: "You've hit your session limit · resets 3pm (UTC)" })], + [bash(1)], + ]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).not.toBeNull(); + await session.send("again", AUTH); + await until(() => session.pendingToolApprovals().length === 1); // turn 2 running, parked on approval + expect(session.lastTurnInfraError).toBeNull(); + await session.interrupt(AUTH); + }); + + test("a message that joins the running turn keeps that turn's rate-limit record", async () => { + const r = rejected(); + const provider = new MockSessionProvider("claude", [[r, { type: "text_delta", content: "working" }]], { stall: true, midTurn: true }); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "thinking"); + await new Promise((res) => setTimeout(res, 20)); + await session.send("also this", AUTH); // joins (mid-turn push), no new turn + expect(provider.midTurnPushes.length).toBe(1); + // The interrupted partial turn's boundary is absorbed; the continuation fails. + provider.emitLive(done()); + provider.emitLive(done({ isError: true })); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError).toMatchObject({ kind: "usage_limit", resetAt: (r as { resetsAt: number }).resetsAt }); + }); +}); + +describe("end_attempt keeps the attempt ended (audit #4)", () => { + test("later prompted and backend-started turns are denied too, and background tasks are stopped", async () => { + const provider = new MockSessionProvider("claude", [[bash(1), bashDone(1), bash(2)], [bash(3)]], { stall: true }); + const session = makeSession({ + _testProvider: provider, + initialMode: { mode: "autonomous", maxTurns: 1 }, + onBudgetExhausted: "end_attempt", + }); + provider.onSessionEvent?.({ type: "background_tasks", tasks: [{ id: "bg-1", kind: "subagent", description: "bg", status: "running" }] }); + await session.send("work", AUTH); + await until(() => provider.canUseToolResults.length === 2 && session.status === "idle"); + await until(() => provider.stoppedTasks.includes("bg-1")); + expect(session.budgetExhausted).toBe(true); + + // A later prompt: still ended. + await session.send("try again", AUTH); + await until(() => provider.canUseToolResults.length === 3); + expect(provider.canUseToolResults[2]!.behavior).toBe("deny"); + await until(() => session.status === "idle"); + expect(session.budgetExhausted).toBe(true); + + // A turn the backend starts on its own: still ended, flag kept. + provider.startOwnTurn([bash(4)]); + await until(() => provider.boundGates.length === 1); + await new Promise((res) => setTimeout(res, 20)); + const verdict = await provider.boundGates[0]!.canUseTool("t4", "ap-4", "Bash", { command: "echo 4" }); + expect(verdict.behavior).toBe("deny"); + expect(session.budgetExhausted).toBe(true); + + session.setMode("autonomous", 2); + expect(session.budgetExhausted).toBe(false); + }); +}); + +class UnbudgetedProvider extends MockSessionProvider { + readonly enforcesToolBudget = false; +} + +describe("toolBudgetEnforced (audit #7)", () => { + test("true on a gated backend; false, with a warning, on one that skips approvals", () => { + expect(makeSession({ _testProvider: new MockSessionProvider("claude") }).toolBudgetEnforced).toBe(true); + const warn = console.warn; + const warnings: string[] = []; + console.warn = (m: unknown) => { warnings.push(String(m)); }; + try { + const s = makeSession({ _testProvider: new UnbudgetedProvider("codex"), onBudgetExhausted: "end_attempt" }); + expect(s.toolBudgetEnforced).toBe(false); + expect(warnings.some((w) => w.includes("can't be enforced"))).toBe(true); + warnings.length = 0; + makeSession({ _testProvider: new UnbudgetedProvider("codex") }); // park: nothing to warn about + expect(warnings).toEqual([]); + } finally { + console.warn = warn; + } + }); +}); + +/** A backend whose interrupt waits for the test to release it. */ +class SlowInterruptProvider extends MockSessionProvider { + release!: () => void; + override runTurn(opts: TurnOpts): TurnRun { + const run = super.runTurn(opts); + const gate = new Promise((r) => { this.release = r; }); + return { ...run, interrupt: () => gate }; + } +} + +/** A backend whose teardown never completes. */ +class StuckTeardownProvider extends MockSessionProvider { + override async teardown(): Promise { + this.teardownCount++; + await new Promise(() => {}); + } +} + +describe("interrupt and terminate races (audit #8)", () => { + test("an interrupt that returns after another turn took over leaves that turn alone", async () => { + const provider = new SlowInterruptProvider("claude", [[bash(1)], [bash(2)]], { stall: true }); + const session = makeSession({ _testProvider: provider }); + await session.send("first", AUTH); + await until(() => session.pendingToolApprovals().length === 1); + const releaseFirst = provider.release; // turn 1's gate (turn 2 gets its own) + const pending = session.interrupt(AUTH); + await session.terminateBackend(AUTH, "test"); + expect(session.status).toBe("idle"); + await session.send("second", AUTH); + await until(() => session.pendingToolApprovals().some((a) => a.approvalId === "ap-2")); + releaseFirst(); + await pending; + expect(session.status).toBe("waiting_approval"); + expect(session.pendingToolApprovals().map((a) => a.approvalId)).toEqual(["ap-2"]); + await session.terminateBackend(AUTH, "cleanup"); + }); + + test("terminateBackend does not hang on a teardown that never finishes", async () => { + const provider = new StuckTeardownProvider("claude", [[bash(1)]], { stall: true }); + const session = makeSession({ _testProvider: provider, initialMode: { mode: "autonomous" } }); + await session.send("work", AUTH); + await until(() => session.status === "tool_running"); + const t0 = Date.now(); + await session.terminateBackend(AUTH, "test", { teardownTimeoutMs: 50 }); + expect(Date.now() - t0).toBeLessThan(2_000); + expect(session.status).toBe("idle"); + expect(provider.teardownCount).toBe(1); + }); +}); diff --git a/src/tests/usage-limit.test.ts b/src/tests/usage-limit.test.ts index 560bdc29..087200e7 100644 --- a/src/tests/usage-limit.test.ts +++ b/src/tests/usage-limit.test.ts @@ -7,7 +7,7 @@ import { describe, expect, test } from "bun:test"; import { readFileSync } from "node:fs"; import { join } from "node:path"; -import { CLAUDE_USAGE_LIMIT_PREFIXES, classifyUsageLimit } from "../daemon/providers/usage-limit.js"; +import { CLAUDE_USAGE_LIMIT_PREFIXES, classifyCliNotice, classifyUsageLimit } from "../daemon/providers/usage-limit.js"; // 2026-10-25 12:00 UTC — 05:00 in Los Angeles (PDT, UTC-7: DST ends Nov 1). const NOW = Date.UTC(2026, 9, 25, 12, 0); @@ -202,6 +202,51 @@ describe("classifyUsageLimit — ordinary errors are not limits", () => { } }); +// Replies a model might write that discuss limits — the audit's false +// positives when assistant text was read with the generic rules. +const MODEL_REPLIES = [ + "The upstream API returned 429 Too Many Requests twice, so I added retries.", + "HTTP 429 handler added; tests pass", + "You've reached your goal: the rate limit test passes", + "Rate limit exceeded errors are now retried with backoff (RESOURCE_EXHAUSTED included).", + "I hit the usage limit reached branch in the parser and fixed it.", + "You've hit your target — the overloaded_error path is covered.", +]; + +describe("classifyCliNotice — CLI-shaped rules only", () => { + test("a notice the CLI composes, at the start of the text", () => { + expect(classifyCliNotice("You've hit your session limit · resets 3pm (UTC)", { now: NOW })).toEqual({ + kind: "usage_limit", + resetAt: Date.UTC(2026, 9, 25, 15, 0), + }); + expect(classifyCliNotice("You've reached your Fable limit.", { now: NOW })?.kind).toBe("usage_limit"); + expect(classifyCliNotice("You've hit your team's shared budget. Switch to another model to continue.", { now: NOW })?.kind).toBe("usage_limit"); + expect(classifyCliNotice(`Claude AI usage limit reached|${Math.floor(NOW / 1000) + 60}`, { now: NOW })?.resetAt).toBe(NOW + 60_000); + expect(classifyCliNotice("API Error: Repeated 529 Overloaded errors", { now: NOW })?.kind).toBe("rate_limit"); + expect(classifyCliNotice("API Error: Request rejected (429) · this may be a temporary capacity issue.", { now: NOW })?.kind).toBe("rate_limit"); + }); + + test("not when it is not the opening of the text", () => { + expect(classifyCliNotice("Done. You've hit your session limit · resets 3pm (UTC)", { now: NOW })).toBeNull(); + expect(classifyCliNotice("The log said API Error: Repeated 529 Overloaded errors", { now: NOW })).toBeNull(); + }); + + test("an API Error that is not a limit is not one", () => { + expect(classifyCliNotice("API Error: 401 Invalid API key · Please run /login", { now: NOW })).toBeNull(); + }); + + for (const reply of MODEL_REPLIES) { + test(`a model reply is not a notice: ${JSON.stringify(reply)}`, () => { + expect(classifyCliNotice(reply, { now: NOW })).toBeNull(); + }); + } + + test("the generic openers need the limit noun right after them in the full classifier too", () => { + expect(at("You've reached your goal: the rate limit test passes")).toBeNull(); + expect(at("codex: You've hit your usage limit.")?.kind).toBe("usage_limit"); + }); +}); + describe("CLAUDE_USAGE_LIMIT_PREFIXES", () => { test("matches the installed Agent SDK's USAGE_LIMIT_ERROR_PREFIXES", () => { // Read from the SDK's declarations, not imported: another suite mocks the From 077190527a58c3abb99db10a6f5e9248c6c58c94 Mon Sep 17 00:00:00 2001 From: Yash Datta Date: Sun, 25 Oct 2026 09:01:51 +0800 Subject: [PATCH 4/4] fix: round-2 audit of the unattended-worker controls MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. A person can take over a spent end_attempt worker: any explicit setMode from a non-system sender (checked before the same-mode early return) or a new turn a person starts lifts budgetExhausted. Session info carries budgetExhausted, and the denial says how to re-arm. 2. killSurvivors kills, from one fresh ps table, every confirmed survivor and its whole current tree — including children spawned after the snapshot. 3. Process identity adds the start time (Linux /proc//stat field 22, macOS ps lstart); ps runs with -ww and without COLUMNS. 4. The Claude CLI pid is found by exact argv tokens (--resume=, --session-id=, and their two-token spelling), not a substring. 5. The scheduled end-attempt interrupt is bound to the turn and background tasks at denial time and no-ops if the turn moved on; tasks whose stop failed stay in the background set. 6. classifyCliNotice also reads the reply's last block, and the session passes the accumulator's last text block. 7. extraEnv denylist adds RUSTC_WRAPPER, RUSTFLAGS, CARGO_*, GOFLAGS, REQUESTS_CA_BUNDLE, CURL_CA_BUNDLE, SSH_AUTH_SOCK, SHELLOPTS, BASHOPTS, PS4. Co-Authored-By: Claude Opus 5.5 (1M context) --- packages/protocol/src/types.ts | 6 ++ src/daemon/process-tree.ts | 93 ++++++++++++++++++++----- src/daemon/providers/claude/index.ts | 22 +++++- src/daemon/providers/env.ts | 7 +- src/daemon/providers/usage-limit.ts | 12 +++- src/daemon/session.ts | 51 +++++++++++--- src/tests/process-tree.test.ts | 57 ++++++++++++++- src/tests/provider-env.test.ts | 2 + src/tests/session-loop-controls.test.ts | 60 +++++++++++++++- src/tests/usage-limit.test.ts | 7 ++ 10 files changed, 280 insertions(+), 37 deletions(-) diff --git a/packages/protocol/src/types.ts b/packages/protocol/src/types.ts index db8b51ad..13e0538c 100644 --- a/packages/protocol/src/types.ts +++ b/packages/protocol/src/types.ts @@ -440,6 +440,12 @@ export interface SessionInfo { turnQueued?: true; /** Remaining turns budget for autonomous mode (undefined = unbounded, 0 = exhausted). */ turnsRemaining?: number; + /** + * An unattended worker (onBudgetExhausted "end_attempt") spent its + * autonomous tool budget: its tool calls are refused until a person sets a + * mode or sends a message, or its controller sets a new budget. Absent = not. + */ + budgetExhausted?: true; /** Files pinned to the session — prepended to every turn's prompt. */ pinnedFiles?: string[]; /** SPIFFE/WIMSE URI of the primary session agent (falls back to anonymous:session:). */ diff --git a/src/daemon/process-tree.ts b/src/daemon/process-tree.ts index 54eedcb1..db4ca61c 100644 --- a/src/daemon/process-tree.ts +++ b/src/daemon/process-tree.ts @@ -20,8 +20,9 @@ * * So the stop is a TREE kill, not a group kill: snapshot every descendant of * the backend's own process(es) before the teardown, and after it SIGKILL any - * of them still alive. A survivor is matched by pid AND command line, so a pid - * the OS reused in between is never touched. What this cannot reach, and does + * of them still alive — with everything under them, including children they + * spawned after the snapshot. A survivor is matched by pid, command line AND + * start time, so a pid the OS reused in between is never touched. What this cannot reach, and does * not claim to: anything that detached itself before the snapshot (only a * cgroup or sandbox boundary bounds that), and a backend whose own process * ids are unknown (qwen-code: @qwen-code/sdk spawns its CLI internally). @@ -31,44 +32,88 @@ */ import { execFile } from "node:child_process"; +import { readFileSync } from "node:fs"; export interface ProcessEntry { pid: number; ppid: number; args: string; + /** + * When the process started, as an opaque comparable token — Linux: + * /proc//stat field 22 (clock ticks since boot); macOS: ps `lstart`. + * With pid and command line, the identity a survivor must still match. + * Absent when it could not be read (the process is gone, or no /proc). + */ + start?: string; } const PS_TIMEOUT_MS = 3_000; +const DARWIN = process.platform === "darwin"; -/** Every process visible to this user: pid, parent pid, command line. */ +/** `ps` with no COLUMNS (it would truncate `args` even with -ww on some systems). */ +function psEnv(): Record { + const env: Record = {}; + for (const [k, v] of Object.entries(process.env)) if (v !== undefined && k !== "COLUMNS") env[k] = v; + return env; +} + +/** Every process visible to this user: pid, parent pid, full command line (and start on macOS). */ export function listProcesses(): Promise { if (process.platform === "win32") return Promise.resolve([]); + const columns = DARWIN ? "pid=,ppid=,lstart=,args=" : "pid=,ppid=,args="; return new Promise((resolve) => { execFile( "ps", - ["-A", "-o", "pid=,ppid=,args="], - { timeout: PS_TIMEOUT_MS, maxBuffer: 32 * 1024 * 1024 }, + ["-A", "-ww", "-o", columns], + { timeout: PS_TIMEOUT_MS, maxBuffer: 32 * 1024 * 1024, env: psEnv() }, (err, stdout) => { if (err) { resolve([]); return; } - resolve(parsePs(stdout)); + resolve(parsePs(stdout, { lstart: DARWIN })); }, ); }); } -/** Parse `ps -o pid=,ppid=,args=` output. Exported for tests. */ -export function parsePs(stdout: string): ProcessEntry[] { +/** + * Parse `ps -o pid=,ppid=[,lstart=],args=` output. `lstart` is five fields + * ("Sat Oct 24 10:08:43 2026") kept verbatim as the start token. Exported for tests. + */ +export function parsePs(stdout: string, opts: { lstart?: boolean } = {}): ProcessEntry[] { const out: ProcessEntry[] = []; - for (const line of stdout.split("\n")) { - const m = /^\s*(\d+)\s+(\d+)\s+(.*)$/.exec(line); - if (m) out.push({ pid: Number(m[1]), ppid: Number(m[2]), args: m[3]!.trim() }); + const line = opts.lstart ? /^\s*(\d+)\s+(\d+)\s+(\S+\s+\S+\s+\S+\s+\S+\s+\S+)\s+(.*)$/ : /^\s*(\d+)\s+(\d+)\s+(.*)$/; + for (const text of stdout.split("\n")) { + const m = line.exec(text); + if (!m) continue; + if (opts.lstart) out.push({ pid: Number(m[1]), ppid: Number(m[2]), start: m[3]!.replace(/\s+/g, " "), args: m[4]!.trim() }); + else out.push({ pid: Number(m[1]), ppid: Number(m[2]), args: m[3]!.trim() }); } return out; } +/** Linux: /proc//stat field 22 (starttime). The comm field may hold spaces and parens, so count from the last ")". */ +export function procStartTime(pid: number): string | undefined { + try { + const stat = readFileSync(`/proc/${pid}/stat`, "utf8"); + const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" "); + // fields[0] is field 3 (state); field 22 is fields[19]. + return fields[19]; + } catch { + return undefined; + } +} + +/** Fill `start` where ps didn't (Linux) — only for the entries that matter. */ +function withStart(entries: ProcessEntry[]): ProcessEntry[] { + if (process.platform !== "linux") return entries; + return entries.map((p) => (p.start !== undefined ? p : { ...p, ...optional(procStartTime(p.pid)) })); +} +function optional(start: string | undefined): { start?: string } { + return start === undefined ? {} : { start }; +} + /** `roots` (those present) and all their descendants in `table`. Never this process. */ export function treeOf(table: readonly ProcessEntry[], roots: readonly number[]): ProcessEntry[] { const children = new Map(); @@ -95,24 +140,34 @@ export function treeOf(table: readonly ProcessEntry[], roots: readonly number[]) return out; } -/** The process tree under `roots` right now. */ +/** The process tree under `roots` right now, each entry with its start time. */ export async function snapshotTree(roots: readonly number[]): Promise { if (roots.length === 0) return []; - return treeOf(await listProcesses(), roots); + return withStart(treeOf(await listProcesses(), roots)); +} + +/** Same process: pid, command line, and (when both are known) start time. */ +function sameProcess(then: ProcessEntry, now: ProcessEntry): boolean { + if (then.pid !== now.pid || then.args !== now.args) return false; + return then.start === undefined || now.start === undefined || then.start === now.start; } /** - * SIGKILL every process of `snapshot` that is still running with the same - * command line (so a reused pid is spared). Returns the pids signalled. + * SIGKILL what is left of `snapshot`: every entry still running as the SAME + * process (pid + command line + start time — a reused pid is spared), and + * every descendant those have now, including children spawned after the + * snapshot was taken. One `ps` call. Returns the pids signalled. */ export async function killSurvivors(snapshot: readonly ProcessEntry[]): Promise { if (snapshot.length === 0) return []; - const now = new Map((await listProcesses()).map((p) => [p.pid, p])); + const table = await listProcesses(); + const now = new Map(table.map((p) => [p.pid, p])); + const confirmed = withStart(snapshot.map((p) => now.get(p.pid)).filter((p): p is ProcessEntry => p !== undefined)) + .filter((live) => sameProcess(snapshot.find((p) => p.pid === live.pid)!, live)) + .map((p) => p.pid); const killed: number[] = []; - for (const p of snapshot) { + for (const p of treeOf(table, confirmed)) { if (p.pid === process.pid) continue; - const live = now.get(p.pid); - if (!live || live.args !== p.args) continue; try { process.kill(p.pid, "SIGKILL"); killed.push(p.pid); diff --git a/src/daemon/providers/claude/index.ts b/src/daemon/providers/claude/index.ts index 96369a08..1f7b7158 100644 --- a/src/daemon/providers/claude/index.ts +++ b/src/daemon/providers/claude/index.ts @@ -653,13 +653,13 @@ export class ClaudeProvider implements SessionProvider { * The Claude Code CLI this provider's query runs in. The SDK spawns it * (spawnClaudeCodeProcess is not used: replacing the SDK's spawn would drop * its stderr diagnostics), so it is found as a direct child of this process - * whose argv names this provider's backing session (`--resume=` / - * `--session-id ` — ids are UUIDs, unique per session). + * whose argv names this provider's backing session as an exact token + * (argvNamesSession — ids are UUIDs, unique per session). */ async backendProcessIds(): Promise { if (!this.#query) return []; const id = this.#spawnedWithId ?? this.#claudeCodeSessionId; - return (await listProcesses()).filter((p) => p.ppid === process.pid && p.args.includes(id)).map((p) => p.pid); + return (await listProcesses()).filter((p) => p.ppid === process.pid && argvNamesSession(p.args, id)).map((p) => p.pid); } async teardown(): Promise { @@ -1591,6 +1591,22 @@ export interface TranslateState { cwd?: string; } +/** + * Whether a Claude CLI command line names backing session `id` as an EXACT + * argv token — the forms the SDK emits (`--resume=`, `--session-id=`, + * SDK 0.3.281) and their two-token spelling. A substring match would also hit + * a process that merely mentions the id (a prompt, a log path). Exported for tests. + */ +export function argvNamesSession(args: string, id: string): boolean { + const argv = args.split(/\s+/); + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === `--resume=${id}` || a === `--session-id=${id}`) return true; + if ((a === "--resume" || a === "--session-id") && argv[i + 1] === id) return true; + } + return false; +} + /** * Translate one SDKMessage from the Claude Agent SDK into zero or more * ProviderEvents. Pure function — no I/O, no side-effects beyond calling diff --git a/src/daemon/providers/env.ts b/src/daemon/providers/env.ts index da05f079..7ddd13fa 100644 --- a/src/daemon/providers/env.ts +++ b/src/daemon/providers/env.ts @@ -115,6 +115,11 @@ const EXTRA_ENV_DENY_EXACT = new Set([ // Backend runtime knobs: logging that can echo requests, and where temp // files (prompts, patches) land. "RUST_LOG", "RUST_BACKTRACE", "TMPDIR", "TEMP", "TMP", "TZ", + // Build tools that run a wrapper or take flags (a rustc wrapper is a + // program), CA bundles that let a proxy read requests, the user's SSH + // agent, and bash options / xtrace prompt (PS4 is expanded, so it runs code). + "RUSTC_WRAPPER", "RUSTFLAGS", "GOFLAGS", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE", "SSH_AUTH_SOCK", + "SHELLOPTS", "BASHOPTS", "PS4", ]); const EXTRA_ENV_DENY_PREFIXES = [ // codeoid's own namespaces (DENY_PREFIXES above) — tokens, the escape hatch. @@ -129,7 +134,7 @@ const EXTRA_ENV_DENY_PREFIXES = [ // Toolchains whose env is configuration that can run code or swap // credentials: git (hooks via GIT_CONFIG_*, GIT_SSH_COMMAND), npm, Python, // Ruby, Perl, the JVM, gcloud. Locale categories (LC_*) too. - "GIT_", "NPM_CONFIG_", "PYTHON", "RUBY", "PERL5", "PERL_", "JAVA_", "_JAVA_", "JDK_", "CLOUDSDK_", "LC_", + "GIT_", "NPM_CONFIG_", "PYTHON", "RUBY", "PERL5", "PERL_", "JAVA_", "_JAVA_", "JDK_", "CLOUDSDK_", "LC_", "CARGO_", ] as const; const EXTRA_ENV_DENY_SUFFIXES = ["_API_KEY", "_API_TOKEN", "_AUTH_TOKEN", "_ACCESS_TOKEN", "_REFRESH_TOKEN", "_OAUTH_TOKEN"] as const; /** diff --git a/src/daemon/providers/usage-limit.ts b/src/daemon/providers/usage-limit.ts index 69471a50..bf66da61 100644 --- a/src/daemon/providers/usage-limit.ts +++ b/src/daemon/providers/usage-limit.ts @@ -294,7 +294,7 @@ export function classifyUsageLimit( * CLI-shaped rules only. That text can just as well be the model's own words * ("the API returned 429 Too Many Requests twice, so I added retries"), so the * generic rate-limit / 429 / quota rules never apply. A notice counts only - * when the text OPENS with what the CLI composes: + * when the text — or its last block — OPENS with what the CLI composes: * * - one of the SDK's USAGE_LIMIT_ERROR_PREFIXES (the generic openers still * need a limit word on their first line), or the legacy @@ -307,7 +307,15 @@ export function classifyCliNotice( opts: ClassifyUsageLimitOptions = {}, ): UsageLimitClassification | null { if (!text) return null; - const t = normalize(text).trimStart(); + const whole = cliNotice(normalize(text).trimStart(), opts); + if (whole) return whole; + // A turn that wrote something before the limit hit ends with the notice as + // its own last block: "I'll start with the tests.\n\nYou've hit your…". + const blocks = normalize(text).split(/\n\s*\n/).map((b) => b.trim()).filter(Boolean); + return blocks.length > 1 ? cliNotice(blocks[blocks.length - 1]!, opts) : null; +} + +function cliNotice(t: string, opts: ClassifyUsageLimitOptions): UsageLimitClassification | null { if (/^API Error:/.test(t)) return classifyUsageLimit(t, opts); if (/^Claude AI usage limit reached/i.test(t) || startsWithLimitPrefix(t)) { const now = opts.now ?? Date.now(); diff --git a/src/daemon/session.ts b/src/daemon/session.ts index f81110ac..c95f2e79 100644 --- a/src/daemon/session.ts +++ b/src/daemon/session.ts @@ -217,6 +217,11 @@ const CANCEL_REASON: Record { - if (this.#destroyed) return; + if (this.#destroyed || this.#currentTurnId !== turnAtDenial) return; void (async () => { await this.interrupt(system); // interrupt() spares background agents while a turn runs; an ended - // attempt must not leave them working on the spent budget. - if (this.#backgroundTasks.size > 0 && this.#provider.stopBackgroundTasks) { - await this.#provider.stopBackgroundTasks([...this.#backgroundTasks.keys()]); - this.#clearBackgroundTasks(); - } + // attempt must not leave them working on the spent budget. Each + // stopped task settles itself out of the set; one whose stop failed + // stays listed — it is still running. + const live = tasksAtDenial.filter((id) => this.#backgroundTasks.has(id)); + if (live.length > 0 && this.#provider.stopBackgroundTasks) await this.#provider.stopBackgroundTasks(live); })().catch((e) => { console.error(`[codeoid/session ${this.id}] ending the attempt on a spent budget failed: ${e instanceof Error ? e.message : String(e)}`); }); @@ -7180,7 +7208,12 @@ export class Session { // or the turn's own assistant text (only if THIS turn committed it, // as above) — CLI-shaped rules only there, never the generic ones. this.#recordTurnFailure([event.result.errorMessage], { - cliTexts: [event.result.resultText, last?.role === "assistant" ? last.content : undefined], + cliTexts: [ + event.result.resultText, + last?.role === "assistant" ? last.content : undefined, + // The reply's final text block on its own (the notice after narration). + last?.role === "assistant" ? (this.#accumulator.lastTextBlock ?? undefined) : undefined, + ], explained: tooLong || event.result.errorMessage === BUDGET_STOP_MESSAGE, }); const hint = tooLong diff --git a/src/tests/process-tree.test.ts b/src/tests/process-tree.test.ts index 869745d4..674fbc8b 100644 --- a/src/tests/process-tree.test.ts +++ b/src/tests/process-tree.test.ts @@ -13,7 +13,8 @@ import { randomUUID } from "node:crypto"; import { mkdtempSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; -import { killSurvivors, listProcesses, parsePs, snapshotTree, treeOf } from "../daemon/process-tree.js"; +import { killSurvivors, listProcesses, parsePs, procStartTime, snapshotTree, treeOf } from "../daemon/process-tree.js"; +import { argvNamesSession } from "../daemon/providers/claude/index.js"; import { MockSessionProvider } from "../daemon/providers/mock/session-provider.js"; import { Session } from "../daemon/session.js"; import { Store } from "../daemon/store.js"; @@ -35,6 +36,19 @@ describe("parsePs / treeOf", () => { expect(treeOf(table, [99])).toEqual([]); }); + test("macOS lstart columns: five fields kept as the start token", () => { + expect(parsePs(" 42 1 Sat Oct 24 10:08:43 2026 /bin/sh -c x", { lstart: true })).toEqual([ + { pid: 42, ppid: 1, start: "Sat Oct 24 10:08:43 2026", args: "/bin/sh -c x" }, + ]); + }); + + test.skipIf(process.platform !== "linux")("Linux start time from /proc//stat field 22", () => { + const t = procStartTime(process.pid); + expect(t).toMatch(/^\d+$/); + expect(procStartTime(process.pid)).toBe(t); + expect(procStartTime(2 ** 30)).toBeUndefined(); + }); + test("never includes this process", () => { const self = parsePs(`${process.pid} 1 bun test\n77 ${process.pid} child`); expect(treeOf(self, [process.pid]).map((p) => p.pid)).toEqual([]); @@ -111,6 +125,31 @@ describe.skipIf(process.platform === "win32")("killing what a backend left runni await until(() => !alive(grandchild)); }); + test.skipIf(process.platform !== "linux" && process.platform !== "darwin")("a different start time means a different process: spared", async () => { + const mark = String(800000 + Math.floor(Math.random() * 99999)); + const { proc, grandchild } = await spawnBackend(mark); + leftovers.push(grandchild, proc.pid!); + const snap = await snapshotTree([proc.pid!]); + expect(snap.every((p) => p.start !== undefined)).toBe(true); + expect(await killSurvivors(snap.map((p) => ({ ...p, start: `${p.start}0` })))).toEqual([]); + expect(alive(grandchild)).toBe(true); + }); + + test("descendants spawned AFTER the snapshot are killed with the survivor", async () => { + const mark = String(800000 + Math.floor(Math.random() * 99999)); + // The "backend" only starts its long command after we have snapshotted it. + const proc = spawn("sh", ["-c", `sleep 0.3; sleep ${mark} & echo $!; wait`], { stdio: ["ignore", "pipe", "ignore"] }); + leftovers.push(proc.pid!); + const snap = await snapshotTree([proc.pid!]); + expect(snap.some((p) => p.args === `sleep ${mark}`)).toBe(false); + const late = await new Promise((r) => proc.stdout!.once("data", (d: Buffer) => r(Number(String(d).trim())))); + leftovers.push(late); + const killed = await killSurvivors(snap); + expect(killed).toContain(proc.pid!); + expect(killed).toContain(late); + await until(() => !alive(late)); + }); + test("Session.terminateBackend snapshots before the teardown and kills the leftover", async () => { const mark = String(800000 + Math.floor(Math.random() * 99999)); const backend = new ProcessBackend("codex", [[{ type: "tool_start", toolId: "t1", sdkToolUseId: "s1", name: "Bash", input: { command: "x" }, approvalId: "a1" }]], { stall: true }); @@ -131,3 +170,19 @@ describe.skipIf(process.platform === "win32")("killing what a backend left runni expect(session.status).toBe("idle"); }); }); + +describe("argvNamesSession", () => { + const id = "0b9a3c1e-1111-4222-8333-444455556666"; + test("the forms the SDK emits, and their two-token spelling", () => { + expect(argvNamesSession(`/x/claude --output-format stream-json --resume=${id} --verbose`, id)).toBe(true); + expect(argvNamesSession(`/x/claude --session-id=${id}`, id)).toBe(true); + expect(argvNamesSession(`/x/claude --resume ${id}`, id)).toBe(true); + expect(argvNamesSession(`/x/claude --session-id ${id} -p`, id)).toBe(true); + }); + test("a mention of the id is not a match", () => { + expect(argvNamesSession(`grep ${id} /tmp/log`, id)).toBe(false); + expect(argvNamesSession(`/x/claude --resume=${id}-fork`, id)).toBe(false); + expect(argvNamesSession(`/x/claude --append-system-prompt=--resume=${id}`, id)).toBe(false); + expect(argvNamesSession(`/x/claude --resume=other ${id}`, id)).toBe(false); + }); +}); diff --git a/src/tests/provider-env.test.ts b/src/tests/provider-env.test.ts index 4c38b890..89a8393f 100644 --- a/src/tests/provider-env.test.ts +++ b/src/tests/provider-env.test.ts @@ -199,6 +199,8 @@ describe("extraEnv denylist (SessionCreateOptions.extraEnv)", () => { "EDITOR", "VISUAL", "GCONV_PATH", "RUST_LOG", "RUST_BACKTRACE", "TMPDIR", "TEMP", "TMP", "TZ", "GIT_CONFIG_GLOBAL", "GIT_SSH_COMMAND", "NPM_CONFIG_USERCONFIG", "PYTHONPATH", "PYTHONSTARTUP", "RUBYOPT", "RUBYLIB", "PERL5LIB", "PERL5OPT", "PERL_MB_OPT", "JAVA_TOOL_OPTIONS", "_JAVA_OPTIONS", "JDK_JAVA_OPTIONS", "CLOUDSDK_CONFIG", "LC_ALL", + "RUSTC_WRAPPER", "RUSTFLAGS", "CARGO_HOME", "CARGO_BUILD_RUSTC_WRAPPER", "GOFLAGS", "REQUESTS_CA_BUNDLE", "CURL_CA_BUNDLE", + "SSH_AUTH_SOCK", "SHELLOPTS", "BASHOPTS", "PS4", ]; const ALLOWED = ["LOOP_RUN_ID", "DATABASE_URL", "CI", "NODE_ENV", "GITHUB_TOKEN", "LANG", "MY_FLAG", "X"]; diff --git a/src/tests/session-loop-controls.test.ts b/src/tests/session-loop-controls.test.ts index 655b1f1b..07c8e725 100644 --- a/src/tests/session-loop-controls.test.ts +++ b/src/tests/session-loop-controls.test.ts @@ -29,6 +29,7 @@ import type { AuthContext, SessionStatus } from "../protocol/types.js"; import type { CodeoidConfig } from "../config.js"; const AUTH: AuthContext = { sub: "user:loops", scopes: [], delegationDepth: 0, accountId: "acc", projectId: "proj" }; +const SYSTEM: AuthContext = { ...AUTH, sub: "system:dispatch" }; let tmp: string; let store: Store; @@ -462,8 +463,8 @@ describe("end_attempt keeps the attempt ended (audit #4)", () => { await until(() => provider.stoppedTasks.includes("bg-1")); expect(session.budgetExhausted).toBe(true); - // A later prompt: still ended. - await session.send("try again", AUTH); + // A later prompt from a system principal (the dispatcher, a controller): still ended. + await session.send("try again", SYSTEM); await until(() => provider.canUseToolResults.length === 3); expect(provider.canUseToolResults[2]!.behavior).toBe("deny"); await until(() => session.status === "idle"); @@ -554,3 +555,58 @@ describe("interrupt and terminate races (audit #8)", () => { expect(provider.teardownCount).toBe(1); }); }); + +// ── Round-2 audit fixes ────────────────────────────────────────────────────── + +describe("a person can take over a spent end_attempt worker (audit-2 #1)", () => { + async function spentWorker() { + const provider = new MockSessionProvider("claude", [[bash(1), bashDone(1), bash(2)], [bash(3)]], { stall: true }); + const session = makeSession({ + _testProvider: provider, + initialMode: { mode: "autonomous", maxTurns: 1 }, + onBudgetExhausted: "end_attempt", + }); + const infos: Array> = []; + session.attach({ id: "c", auth: AUTH, send: (m) => { if (m.type === "session.info_update") infos.push(m as unknown as Record); } }); + await session.send("work", SYSTEM); + await until(() => provider.canUseToolResults.length === 2 && session.status === "idle"); + expect(session.budgetExhausted).toBe(true); + return { provider, session, infos }; + } + + test("session info says so, and the denial says how to re-arm", async () => { + const { provider, session, infos } = await spentWorker(); + const last = infos.at(-1) as { session?: { budgetExhausted?: boolean } } | undefined; + expect(JSON.stringify(last)).toContain('"budgetExhausted":true'); + expect(provider.canUseToolResults[1]!.message).toContain("/mode"); + void session; + }); + + test("a person's setMode lifts it — even to the mode already set; a system one does not", async () => { + const { session } = await spentWorker(); + session.setMode("guarded", undefined, SYSTEM); + expect(session.budgetExhausted).toBe(true); + session.setMode("guarded", undefined, AUTH); // already guarded + expect(session.budgetExhausted).toBe(false); + }); + + test("a person's new turn lifts it: the call is gated normally, not denied", async () => { + const { provider, session } = await spentWorker(); + await session.send("let me look", AUTH); + await until(() => session.pendingToolApprovals().length === 1); + expect(session.budgetExhausted).toBe(false); + expect(provider.canUseToolResults).toHaveLength(2); // the third call is waiting on a person, not refused + await session.interrupt(AUTH); + }); +}); + +describe("the CLI notice as the reply's last block (audit-2 #6)", () => { + test("narration, then the limit notice", async () => { + const reply = "I'll start with the failing test.\n\nYou've hit your session limit · resets 3pm (UTC)"; + const provider = new MockSessionProvider("claude", [[{ type: "text_done", content: reply }, done({ isError: true })]]); + const session = makeSession({ _testProvider: provider }); + await session.send("work", AUTH); + await until(() => session.status === "error"); + expect(session.lastTurnInfraError?.kind).toBe("usage_limit"); + }); +}); diff --git a/src/tests/usage-limit.test.ts b/src/tests/usage-limit.test.ts index 087200e7..2555e7f3 100644 --- a/src/tests/usage-limit.test.ts +++ b/src/tests/usage-limit.test.ts @@ -226,6 +226,13 @@ describe("classifyCliNotice — CLI-shaped rules only", () => { expect(classifyCliNotice("API Error: Request rejected (429) · this may be a temporary capacity issue.", { now: NOW })?.kind).toBe("rate_limit"); }); + test("the last block of a reply that narrated first", () => { + expect(classifyCliNotice("Working on it.\n\nYou've hit your weekly limit · resets Oct 28, 9am (UTC)", { now: NOW })?.resetAt).toBe( + Date.UTC(2026, 9, 28, 9, 0), + ); + expect(classifyCliNotice("Done.\n\nThe API returned 429 Too Many Requests, so I added retries.", { now: NOW })).toBeNull(); + }); + test("not when it is not the opening of the text", () => { expect(classifyCliNotice("Done. You've hit your session limit · resets 3pm (UTC)", { now: NOW })).toBeNull(); expect(classifyCliNotice("The log said API Error: Repeated 529 Overloaded errors", { now: NOW })).toBeNull();