From 2045926b19728425fb0c4200910c11772fdcf447 Mon Sep 17 00:00:00 2001 From: Daniel Ari Friedman <6911384+docxology@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:45:57 -0800 Subject: [PATCH 1/2] Fix durable state ownership and coordinator admission - Adopt the 25 individually provenance-reviewed Safety software paths; preserve corpus and scientific holds. - Publish task state only after durable writes, arbitrate state ownership, and account cumulative provider usage once. - Traverse complete internal queues, preserve never-admitted work on shutdown, and harden HTTP and environment boundaries. --- .env.example | 4 + README.md | 13 +- src/README.md | 30 + src/config/config.ts | 20 +- src/coordination/coordinator.ts | 121 +++- src/domain/types.ts | 2 + src/http/server.ts | 12 +- src/providers/governed-provider.ts | 2 +- src/providers/provider.ts | 1 + src/state/state-store.ts | 564 ++++++++++++++----- src/tasks/task-repository.ts | 162 +++--- tests/README.md | 29 + tests/fixtures/state-boundary.ts | 86 +++ tests/fixtures/state-lock-worker.ts | 56 ++ tests/integration/coordinator-commit.test.ts | 285 ++++++++++ tests/integration/http-boundaries.test.ts | 110 ++++ tests/integration/http.test.ts | 66 ++- tests/integration/queue-traversal.test.ts | 131 +++++ tests/integration/state-lock.test.ts | 313 ++++++++++ tests/integration/task-commit.test.ts | 208 +++++++ tests/integration/usage-accounting.test.ts | 197 +++++++ tests/unit/config-launch.test.ts | 66 +++ tests/unit/provider-retention.test.ts | 52 ++ tests/unit/security.test.ts | 4 + tests/unit/state.test.ts | 6 +- 25 files changed, 2234 insertions(+), 306 deletions(-) create mode 100644 tests/fixtures/state-boundary.ts create mode 100644 tests/fixtures/state-lock-worker.ts create mode 100644 tests/integration/coordinator-commit.test.ts create mode 100644 tests/integration/http-boundaries.test.ts create mode 100644 tests/integration/queue-traversal.test.ts create mode 100644 tests/integration/state-lock.test.ts create mode 100644 tests/integration/task-commit.test.ts create mode 100644 tests/integration/usage-accounting.test.ts create mode 100644 tests/unit/config-launch.test.ts create mode 100644 tests/unit/provider-retention.test.ts diff --git a/.env.example b/.env.example index 23a2fc7..58d2277 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,7 @@ +# Load explicitly with Node --env-file=.env; the runtime does not auto-load cwd .env. +# Inherited environment values override this file. +# Blank optional key/base URL/host/auth/budget values mean absent, not zero. +# Provider selection stays explicit; openai still needs a key and production needs auth. NODE_ENV=development NOORG_LLM_PROVIDER=local OPENAI_API_KEY= diff --git a/README.md b/README.md index 7d6b1d1..e943370 100644 --- a/README.md +++ b/README.md @@ -30,10 +30,19 @@ NoOrg is an active research runtime at working-copy maturity. The default develo npm install cp .env.example .env npm run validate -npm start +node --env-file=.env --import tsx src/index.ts ``` -The default development profile uses the deterministic local provider and requires no network access. Production requires `NOORG_LLM_PROVIDER=openai` and `OPENAI_API_KEY`. +Configuration reads `process.env`: a cwd `.env` is not loaded automatically by +`npm start` or `npm run start:dev`. Node's explicit `--env-file` loads it, and +inherited environment values take precedence. After a separately verified build, +the corresponding launch is `node --env-file=.env dist/index.js`. + +Keep `NOORG_LLM_PROVIDER` explicit. The local development profile needs no model +credentials; `OPENAI_API_KEY` is required only for `openai`. Production also +requires `NOORG_HTTP_AUTH_TOKEN`. Blank optional key/base URL/host/auth/budget +values mean absent, not an implicit provider or permission to omit production +authentication. Nonempty invalid values still fail validation. ## Runtime surface diff --git a/src/README.md b/src/README.md index 979e167..5951819 100644 --- a/src/README.md +++ b/src/README.md @@ -27,6 +27,36 @@ TypeScript source for the NoOrg runtime. `src/index.ts` is the composition root; - All dependencies are constructor-injected and every background resource exposes a shutdown path. - Errors are structured and actionable; secrets are never logged. +## Runtime contracts + +Public task pages remain capped at 100. Scheduling, health and metrics use +complete internal snapshots. `blockedTasks` counts the union of capability, +capacity, relationship and backoff blockers without double-counting; the +capability/capacity classifications are not a complete partition of that total. + +State mutations publish committed values, revisions and task maps only after +durable replacement. Rejected pre-rename write/fsync candidates stay invisible +and cannot become a later flush or shutdown state. Rename/directory-sync +uncertainty may leave new raw disk bytes without acknowledgement: it fences +the store and retains its namespace guard. Guards are never automatically +reclaimed. Stop writers, preserve state/owner/guard/temp physical evidence and +obtain explicit reviewed recovery; never reset or roll back blindly. + +Provider task counters are cumulative per provider instance; the coordinator +durably commits only new deltas and advances checkpoints only after success. +Fresh provider counters do not subtract persisted historical usage. Late +cancellation/timeout settlement records usage once without publishing a result. +Disabled rate limiting retains no request-time history; enabled windows retain +their existing quota, rollover, circuit and budget semantics. + +Execution timers are cleared when the agent/timeout race settles, before durable +completion, accounting or event delivery. Reservation ownership lasts through +finalization; genuine timed-out work retains its reservation until the actual +operation settles. Structured task failures retain their original code in the +health observation rather than being reclassified as an unknown error. +Shutdown settles an already pending claim without claiming or cancelling the +remaining queued backlog, which stays persisted for the next process. + ## Commands ```bash diff --git a/src/config/config.ts b/src/config/config.ts index 4cd1b0f..ab816b7 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -17,6 +17,13 @@ function integer(name: string, fallback: number, minimum: number, maximum?: numb ); } +function blankOptional(schema: T) { + return z.preprocess( + value => typeof value === 'string' && value.trim() === '' ? undefined : value, + schema.optional() + ); +} + export interface AppConfig { readonly environment: 'development' | 'test' | 'production'; readonly provider: ProviderKind; @@ -59,17 +66,17 @@ export interface AppConfig { const rawSchema = z.object({ NODE_ENV: environmentSchema.default('development'), NOORG_LLM_PROVIDER: providerSchema, - OPENAI_API_KEY: z.string().trim().min(1).optional(), + OPENAI_API_KEY: blankOptional(z.string().trim().min(1)), OPENAI_MODEL: z.string().trim().min(1).default('gpt-4o-mini'), - OPENAI_BASE_URL: z.string().url().optional(), + OPENAI_BASE_URL: blankOptional(z.string().url()), OPENAI_TIMEOUT_MS: integer('OPENAI_TIMEOUT_MS', 30000, 1000), OPENAI_MAX_RETRIES: integer('OPENAI_MAX_RETRIES', 2, 0), OPENAI_RETRY_BASE_MS: integer('OPENAI_RETRY_BASE_MS', 250, 50), OPENAI_PROMPT_COST_PER_MILLION_USD: z.coerce.number().finite().min(0).default(0), OPENAI_COMPLETION_COST_PER_MILLION_USD: z.coerce.number().finite().min(0).default(0), - NOORG_HOST: z.string().trim().min(1).optional(), + NOORG_HOST: blankOptional(z.string().trim().min(1)), NOORG_PORT: integer('NOORG_PORT', 3000, 1), - NOORG_HTTP_AUTH_TOKEN: z.string().trim().min(1).optional(), + NOORG_HTTP_AUTH_TOKEN: blankOptional(z.string().trim().min(1)), NOORG_HTTP_MAX_BODY_BYTES: integer('NOORG_HTTP_MAX_BODY_BYTES', 1_048_576, 1024), NOORG_STATE_PATH: z.string().trim().min(1).default('data/state.json'), NOORG_POLL_INTERVAL_MS: integer('NOORG_POLL_INTERVAL_MS', 100, 10), @@ -80,10 +87,7 @@ const rawSchema = z.object({ NOORG_MAX_RESULT_BYTES: integer('NOORG_MAX_RESULT_BYTES', 1_048_576, 1024), NOORG_MAX_WORKFLOW_DEPTH: integer('NOORG_MAX_WORKFLOW_DEPTH', 32, 1, 64), NOORG_MAX_WORKFLOW_CHILDREN: integer('NOORG_MAX_WORKFLOW_CHILDREN', 100, 1), - NOORG_PROVIDER_BUDGET_USD: z.preprocess( - value => (value === undefined || value === '' ? undefined : value), - z.coerce.number().finite().min(0).optional() - ), + NOORG_PROVIDER_BUDGET_USD: blankOptional(z.coerce.number().finite().min(0)), NOORG_PROVIDER_MAX_REQUESTS_PER_MINUTE: integer('NOORG_PROVIDER_MAX_REQUESTS_PER_MINUTE', 0, 0), NOORG_PROVIDER_CIRCUIT_FAILURE_THRESHOLD: integer( 'NOORG_PROVIDER_CIRCUIT_FAILURE_THRESHOLD', diff --git a/src/coordination/coordinator.ts b/src/coordination/coordinator.ts index 4040437..c205d6c 100644 --- a/src/coordination/coordinator.ts +++ b/src/coordination/coordinator.ts @@ -12,6 +12,7 @@ import type { TaskRequest, TaskStatus, } from '../domain/types'; +import { usageMetadataSchema, type UsageMetadata } from '../domain/types'; import type { Agent } from '../agents/abstract-agent'; import type { AgentAvailability, AgentRegistry } from '../agents/registry'; import type { Logger } from '../logging/logger'; @@ -70,6 +71,8 @@ export class Coordinator implements TaskScheduler { private shutdownTimedOut = false; private startPromise: Promise | undefined; private shutdownPromise: Promise | undefined; + private readonly usageCheckpoints = new Map(); + private readonly usageMutations = new Map>(); public constructor(private readonly dependencies: CoordinatorDependencies) { this.repository = @@ -163,16 +166,16 @@ export class Coordinator implements TaskScheduler { readonly registryReady: boolean; readonly shutdownTimedOut: boolean; readonly lastFailure?: FailureObservation; - readonly usage: ReturnType; + readonly usage: UsageMetadata; } { - const queued = this.repository.list('queued'); + const { tasks: queued, total: queueDepth } = this.repository.internalSnapshot('queued'); const blocked = this.blockedTaskCounts(queued); const health = { status: this.status, provider: this.dependencies.provider.kind, agents: this.dependencies.registry.descriptors().length, activeTasks: this.dependencies.registry.activeCount(), - queueDepth: queued.length, + queueDepth, blockedTasks: blocked.total, blockedCapabilityTasks: blocked.capability, blockedCapacityTasks: blocked.capacity, @@ -184,7 +187,7 @@ export class Coordinator implements TaskScheduler { ...(this.lastFailure === undefined ? {} : { lastFailure: this.lastFailure }), usage: this.dependencies.provider.getUsage(), }; - this.refreshMetrics(health.usage); + this.refreshMetrics(health.usage, queued, blocked); return health; } @@ -208,6 +211,11 @@ export class Coordinator implements TaskScheduler { try { await this.dependencies.state.load(); await this.repository.load(); + for (const task of this.repository.internalSnapshot().tasks) { + const currentUsage = this.dependencies.provider.getTaskUsage?.(task.id); + if (currentUsage !== undefined) + this.usageCheckpoints.set(task.id, usageMetadataSchema.parse(currentUsage)); + } const recovered = await this.repository.recoverInterruptedTasks(); await this.dependencies.registry.initialize({ provider: this.dependencies.provider, @@ -307,9 +315,8 @@ export class Coordinator implements TaskScheduler { 0, this.dependencies.maxConcurrentTasks - this.dependencies.registry.activeCount() ); - if (available === 0) return 0; - for (const task of this.repository.list('queued')) { - if (launched >= available) break; + for (const task of this.repository.internalSnapshot('queued').tasks) { + if (this.status !== 'running') break; const now = this.dependencies.clock.now().getTime(); if (task.deadlineAt !== undefined && Date.parse(task.deadlineAt) <= now) { const expired = await this.repository.failQueued(task.id, { @@ -346,6 +353,7 @@ export class Coordinator implements TaskScheduler { continue; } if (parentState === 'blocked') continue; + if (launched >= available) continue; const candidate = this.toAgentTask(task); const lease = this.dependencies.registry.acquire(candidate); if (!lease) continue; @@ -361,6 +369,19 @@ export class Coordinator implements TaskScheduler { }); continue; } + if (this.status !== 'running') { + try { + if (this.repository.get(started.id)?.status === 'running') + await this.cancelTask(started.id); + } finally { + lease.release(); + } + break; + } + if (this.repository.get(started.id)?.status !== 'running') { + lease.release(); + continue; + } const controller = new AbortController(); this.taskControllers.set(started.id, controller); const execution = this.execute(started, lease.agent, controller, lease.release); @@ -405,14 +426,16 @@ export class Coordinator implements TaskScheduler { }); try { - const result = await Promise.race([operation, timeout]); + let result: AgentResult; + try { + result = await Promise.race([operation, timeout]); + } finally { + clearTimeout(timer); + } const current = this.repository.get(task.id); if (this.status === 'running' && current?.status === 'running') { - const taskUsage = this.dependencies.provider.getTaskUsage?.(task.id); - const completed = await this.repository.completeWithUsage( - task.id, - agent.validateResult(result), - taskUsage + const completed = await this.commitProviderUsage(task.id, usage => + this.repository.completeWithUsage(task.id, agent.validateResult(result), usage) ); this.dependencies.metrics.increment('tasks_completed'); this.dependencies.metrics.observe( @@ -420,6 +443,8 @@ export class Coordinator implements TaskScheduler { this.dependencies.clock.now().getTime() - startedAt ); await this.publishSafely('task.completed', completed); + } else { + await this.recordProviderUsage(task.id); } } catch (error) { if (timedOut) { @@ -442,7 +467,6 @@ export class Coordinator implements TaskScheduler { } } } finally { - if (timer) clearTimeout(timer); release(); this.taskControllers.delete(task.id); } @@ -455,7 +479,7 @@ export class Coordinator implements TaskScheduler { this.dependencies.metrics.increment( failed.status === 'queued' ? 'tasks_retried' : 'tasks_failed' ); - this.recordFailure(error); + this.recordTaskFailure(error); await this.publishSafely(failed.status === 'queued' ? 'task.requeued' : 'task.failed', failed); } @@ -541,12 +565,46 @@ export class Coordinator implements TaskScheduler { } private async recordProviderUsage(taskId: string): Promise { - const usage = this.dependencies.provider.getTaskUsage?.(taskId); - if (usage === undefined || (usage.totalTokens === 0 && usage.costUsd === 0)) return; - await this.repository.recordUsage(taskId, usage).catch(error => { - if (!(error instanceof NoOrgError && error.code === 'INVALID_TASK_TRANSITION')) - this.recordFailure(error); + await this.commitProviderUsage(taskId, async usage => { + if (usage !== undefined) await this.repository.recordUsage(taskId, usage); + }).catch(error => this.recordFailure(error)); + } + + private commitProviderUsage( + taskId: string, + commit: (delta: UsageMetadata | undefined) => Promise + ): Promise { + const previous = this.usageMutations.get(taskId) ?? Promise.resolve(); + const operation = previous.then(async () => { + const observed = this.dependencies.provider.getTaskUsage?.(taskId); + const current = observed === undefined ? undefined : usageMetadataSchema.parse(observed); + const checkpoint = this.usageCheckpoints.get(taskId) ?? { + promptTokens: 0, completionTokens: 0, totalTokens: 0, costUsd: 0, + }; + let delta: UsageMetadata | undefined; + if (current !== undefined) { + if (current.promptTokens < checkpoint.promptTokens || + current.completionTokens < checkpoint.completionTokens || + current.totalTokens < checkpoint.totalTokens || current.costUsd < checkpoint.costUsd) + throw new NoOrgError('PROVIDER_USAGE_REGRESSED', 'Provider-instance usage counters decreased'); + const added = { + promptTokens: current.promptTokens - checkpoint.promptTokens, + completionTokens: current.completionTokens - checkpoint.completionTokens, + totalTokens: current.totalTokens - checkpoint.totalTokens, + costUsd: current.costUsd - checkpoint.costUsd, + }; + if (Object.values(added).some(value => value !== 0)) delta = added; + } + const result = await commit(delta); + if (current !== undefined) this.usageCheckpoints.set(taskId, current); + return result; }); + const settled = operation.then(() => undefined, () => undefined); + this.usageMutations.set(taskId, settled); + void settled.then(() => { + if (this.usageMutations.get(taskId) === settled) this.usageMutations.delete(taskId); + }); + return operation; } private async publishSafely(type: string, task: TaskRecord): Promise { @@ -566,7 +624,10 @@ export class Coordinator implements TaskScheduler { } private recordFailure(error: unknown): void { - const taskError = toTaskError(error); + this.recordTaskFailure(toTaskError(error)); + } + + private recordTaskFailure(taskError: TaskError): void { this.lastFailure = { code: taskError.code, message: taskError.message, @@ -574,9 +635,11 @@ export class Coordinator implements TaskScheduler { }; } - private refreshMetrics(usage: ReturnType): void { - const queued = this.repository.list('queued'); - const blocked = this.blockedTaskCounts(queued); + private refreshMetrics( + usage: UsageMetadata, + queued: readonly TaskRecord[] = this.repository.internalSnapshot('queued').tasks, + blocked = this.blockedTaskCounts(queued) + ): void { this.dependencies.metrics.setGauge('active_tasks', this.dependencies.registry.activeCount()); this.dependencies.metrics.setGauge('queue_depth', queued.length); this.dependencies.metrics.setGauge('blocked_tasks', blocked.total); @@ -593,13 +656,21 @@ export class Coordinator implements TaskScheduler { readonly capability: number; readonly capacity: number; } { + let total = 0; + const now = this.dependencies.clock.now().getTime(); let capability = 0; let capacity = 0; for (const task of tasks) { const availability = this.agentAvailability(task); if (availability === 'no_match') capability += 1; if (availability === 'at_capacity') capacity += 1; + if ( + availability !== 'available' || + this.dependencyState(task) !== 'ready' || + this.parentState(task) !== 'ready' || + (task.nextAttemptAt !== undefined && Date.parse(task.nextAttemptAt) > now) + ) total += 1; } - return { total: capability + capacity, capability, capacity }; + return { total, capability, capacity }; } } diff --git a/src/domain/types.ts b/src/domain/types.ts index 222be33..2054921 100644 --- a/src/domain/types.ts +++ b/src/domain/types.ts @@ -157,6 +157,8 @@ export const taskQuerySchema = z }) .strict(); +export type ParsedTaskQuery = z.infer; + export interface TaskRecord { readonly id: string; readonly name: string; diff --git a/src/http/server.ts b/src/http/server.ts index 3ec482f..9467521 100644 --- a/src/http/server.ts +++ b/src/http/server.ts @@ -23,9 +23,11 @@ const requestTimeoutMs = 30_000; const keepAliveTimeoutMs = 5_000; export function isBearerAuthorized(authorization: string | undefined, token: string): boolean { - const expected = `Bearer ${token}`; - if (authorization === undefined || authorization.length !== expected.length) return false; - return timingSafeEqual(Buffer.from(authorization), Buffer.from(expected)); + if (authorization === undefined) return false; + const supplied = Buffer.from(authorization, 'utf8'); + const expected = Buffer.from(`Bearer ${token}`, 'utf8'); + if (supplied.length !== expected.length) return false; + return timingSafeEqual(supplied, expected); } interface AuthEntry { @@ -344,7 +346,9 @@ function mapHttpError(error: unknown): { ? 503 : error.code.startsWith('INVALID_') || error.code === 'TASK_DEADLINE_INVALID' || - error.code.endsWith('_TOO_LARGE') + error.code.endsWith('_TOO_LARGE') || + error.code === 'TASK_WORKFLOW_BREADTH_EXCEEDED' || + error.code === 'TASK_WORKFLOW_DEPTH_EXCEEDED' ? 400 : error.code.startsWith('PROVIDER_') ? 502 diff --git a/src/providers/governed-provider.ts b/src/providers/governed-provider.ts index 1e070d1..9f5a61c 100644 --- a/src/providers/governed-provider.ts +++ b/src/providers/governed-provider.ts @@ -76,7 +76,7 @@ export class GovernedProvider implements LLMProvider { this.assertCircuit(); this.assertRateLimit(); const reservation = this.reserveBudget(request); - this.requestTimes.push(this.clock.now().getTime()); + if (this.maxRequestsPerMinute > 0) this.requestTimes.push(this.clock.now().getTime()); try { const response = await this.inner.complete(request, schema, signal); if (request.taskId !== undefined) this.addTaskUsage(request.taskId, response.usage); diff --git a/src/providers/provider.ts b/src/providers/provider.ts index e743dc9..7a68225 100644 --- a/src/providers/provider.ts +++ b/src/providers/provider.ts @@ -16,6 +16,7 @@ export interface LLMProvider { readonly kind: ProviderKind; isOpen(): boolean; getUsage(): UsageMetadata; + /** Cumulative usage for this provider instance, not a persistence delta. */ getTaskUsage?(taskId: string): UsageMetadata; complete( request: CompletionRequest, diff --git a/src/state/state-store.ts b/src/state/state-store.ts index a92c24b..f6c57fa 100644 --- a/src/state/state-store.ts +++ b/src/state/state-store.ts @@ -1,5 +1,7 @@ -import { mkdir, open, readFile, rename, rm, stat, writeFile } from 'node:fs/promises'; -import { dirname } from 'node:path'; +import { lstat, mkdir, open, readFile, rename, rm, rmdir } from 'node:fs/promises'; +import { constants, type Stats } from 'node:fs'; +import type { FileHandle } from 'node:fs/promises'; +import { dirname, resolve } from 'node:path'; import { z } from 'zod'; import { randomUUID } from 'node:crypto'; import type { JsonValue } from '../domain/types'; @@ -43,7 +45,11 @@ function isLiveProcess(pid: number): boolean { process.kill(pid, 0); return true; } catch (error) { - return typeof error === 'object' && error !== null && 'code' in error && error.code === 'EPERM'; + if (typeof error === 'object' && error !== null && 'code' in error) { + if (error.code === 'EPERM') return true; + if (error.code === 'ESRCH') return false; + } + throw error; } } @@ -55,56 +61,157 @@ interface LockOwner { const staleLockMs = 30_000; +export interface FileStateStoreIO { + readonly lstat: typeof lstat; + readonly mkdir: typeof mkdir; + readonly open: typeof open; + readonly readFile: typeof readFile; + readonly rename: typeof rename; + readonly rm: typeof rm; + readonly rmdir: typeof rmdir; +} + +export const nodeStateStoreIO: FileStateStoreIO = { + lstat, + mkdir, + open, + readFile, + rename, + rm, + rmdir, +}; + +interface DirectoryLease { + readonly path: string; + readonly handle: FileHandle; + readonly dev: number; + readonly ino: number; +} + +interface StoreLease extends DirectoryLease { + readonly owner: LockOwner; +} + +interface ObservedLock { + readonly identity: Stats; + readonly owner: LockOwner | undefined; +} + +function sameIdentity(left: { dev: number; ino: number }, right: { dev: number; ino: number }): boolean { + return left.dev === right.dev && left.ino === right.ino; +} + +function parseOwner(raw: string): LockOwner | undefined { + try { + const parsed: unknown = JSON.parse(raw); + if ( + typeof parsed === 'object' && parsed !== null && + 'pid' in parsed && Number.isSafeInteger(parsed.pid) && Number(parsed.pid) > 0 && + 'createdAt' in parsed && typeof parsed.createdAt === 'number' && + Number.isFinite(parsed.createdAt) && parsed.createdAt >= 0 && + 'token' in parsed && typeof parsed.token === 'string' && parsed.token.length > 0 + ) return parsed as LockOwner; + } catch { + // Only the exact historical decimal-PID representation is admitted below. + } + const legacy = raw.trim(); + if (/^[1-9][0-9]*$/.test(legacy) && Number.isSafeInteger(Number(legacy))) { + return { pid: Number(legacy), createdAt: 0, token: 'legacy-pid' }; + } + return undefined; +} + export class FileStateStore implements StateStore { - private readonly values = new Map(); - private mutation = Promise.resolve(); + private values = new Map(); + private mutation: Promise = Promise.resolve(); private closed = false; - private locked = false; + private closing = false; + private loaded = false; + private dirty = false; + private failure: NoOrgError | undefined; + private retainGuard = false; + private lease: StoreLease | undefined; + private closePromise: Promise | undefined; private currentRevision = 0; + private readonly filePath: string; - public isOpen(): boolean { - return !this.closed; + public constructor(filePath: string, private readonly io: FileStateStoreIO = nodeStateStoreIO) { + this.filePath = resolve(filePath); } - public constructor(private readonly filePath: string) {} + public isOpen(): boolean { + return this.loaded && !this.closed && !this.closing && this.failure === undefined; + } public async load(): Promise { - if (this.closed) throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closed'); - await this.acquireLock(); - try { - const content = await readFile(this.filePath, 'utf8'); - const raw: unknown = JSON.parse(content); - const current = envelopeSchema.safeParse(raw); - const previous = previousEnvelopeSchema.safeParse(raw); - if (!current.success && !previous.success) { - throw new NoOrgError( - 'STATE_SCHEMA_INVALID', - `State file ${this.filePath} has an invalid schema` - ); - } - const parsed = current.success - ? current.data - : previous.success - ? { schemaVersion: 2 as const, revision: 0, state: previous.data.state } - : undefined; - if (!parsed) - throw new NoOrgError( - 'STATE_SCHEMA_INVALID', - `State file ${this.filePath} has an invalid schema` - ); - this.values.clear(); - for (const [key, value] of Object.entries(parsed.state)) this.values.set(key, clone(value)); - this.currentRevision = parsed.revision; - } catch (error) { - if (isMissingFile(error)) return; - await this.releaseLock(); - if (error instanceof NoOrgError) throw error; - throw new NoOrgError( - 'STATE_LOAD_FAILED', - `Unable to load state from ${this.filePath}`, - false - ); - } + this.assertAvailable(false); + if (this.closing) throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closing'); + await this.schedule(async () => { + this.assertAvailable(false); + if (this.loaded) return; + await this.io.mkdir(dirname(this.filePath), { recursive: true }); + // Optimistic inspection may reject a live/ambiguous owner without changing + // its namespace. Any stale observation is re-read under the guard. + const observed = await this.observeLock(); + if (observed !== undefined) this.assertReclaimable(observed); + await this.withGuard(async guard => { + const current = await this.observeLock(); + if (current !== undefined) { + this.assertReclaimable(current); + await this.assertDirectory(guard); + await this.io.rm(`${this.filePath}.lock`, { recursive: true }); + } + const lockPath = `${this.filePath}.lock`; + await this.io.mkdir(lockPath, { mode: 0o700 }); + const directory = await this.openDirectory(lockPath); + const owner: LockOwner = { pid: process.pid, createdAt: Date.now(), token: randomUUID() }; + this.lease = { ...directory, owner }; + try { + const marker = await this.io.open(`${lockPath}/owner`, 'wx', 0o600); + try { + await marker.writeFile(JSON.stringify(owner), 'utf8'); + await marker.sync(); + } finally { + await marker.close(); + } + await directory.handle.sync(); + let parsed: z.infer; + try { + const raw: unknown = JSON.parse(await this.io.readFile(this.filePath, 'utf8')); + const currentEnvelope = envelopeSchema.safeParse(raw); + const previous = previousEnvelopeSchema.safeParse(raw); + if (currentEnvelope.success) { + parsed = currentEnvelope.data; + this.dirty = false; + } else if (previous.success) { + parsed = { schemaVersion: 2, revision: 0, state: previous.data.state }; + this.dirty = true; + } else { + throw new NoOrgError('STATE_SCHEMA_INVALID', 'State file has an invalid schema'); + } + } catch (error) { + if (!isMissingFile(error)) throw error; + parsed = { schemaVersion: 2, revision: 0, state: {} }; + this.dirty = true; + } + await this.assertOwnership(guard); + this.values = new Map(Object.entries(parsed.state).map(([key, value]) => [key, clone(value)])); + this.currentRevision = parsed.revision; + this.loaded = true; + } catch (error) { + try { + await this.releaseLock(guard); + } catch (cleanupError) { + this.fence('STATE_LOCK_LOST', 'State lock ownership became ambiguous', cleanupError); + throw new AggregateError([error, cleanupError], 'State acquisition and owned cleanup failed'); + } + if (error instanceof NoOrgError) throw error; + const failure = new NoOrgError('STATE_LOAD_FAILED', `Unable to load state from ${this.filePath}`); + failure.cause = error; + throw failure; + } + }); + }); } public get(key: string): T | undefined { @@ -121,11 +228,17 @@ export class FileStateStore implements StateStore { } public async set(key: string, value: JsonValue): Promise { - if (this.closed) throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closed'); - jsonValueSchema.parse(value); - await this.enqueue(() => { - this.values.set(key, clone(value)); - this.currentRevision += 1; + this.assertAdmission(); + const admitted = clone(jsonValueSchema.parse(value)); + await this.schedule(async () => { + this.assertAvailable(); + const candidate = new Map(this.values); + candidate.set(key, admitted); + const revision = this.currentRevision + 1; + await this.withGuard(guard => this.writeCandidate(candidate, revision, guard)); + this.values = candidate; + this.currentRevision = revision; + this.dirty = false; }); } @@ -133,134 +246,268 @@ export class FileStateStore implements StateStore { key: string, updater: (current: T | undefined) => T ): Promise { - let nextValue!: T; - await this.enqueue(() => { - const current = this.values.get(key) as T | undefined; - nextValue = updater(current === undefined ? undefined : clone(current)); - jsonValueSchema.parse(nextValue); - this.values.set(key, clone(nextValue)); - this.currentRevision += 1; + this.assertAdmission(); + return this.schedule(async () => { + this.assertAvailable(); + const current = this.values.get(key); + const next = clone(jsonValueSchema.parse(updater(current === undefined ? undefined : clone(current) as T))) as T; + const candidate = new Map(this.values); + candidate.set(key, next); + const revision = this.currentRevision + 1; + await this.withGuard(guard => this.writeCandidate(candidate, revision, guard)); + this.values = candidate; + this.currentRevision = revision; + this.dirty = false; + return clone(next); }); - return clone(nextValue); } public async flush(): Promise { - await this.mutation; - if (this.closed || !this.locked) return; - await mkdir(dirname(this.filePath), { recursive: true }); + this.assertAdmission(); + await this.schedule(() => this.flushCommitted()); + } + + public async close(): Promise { + if (this.closePromise !== undefined) return this.closePromise; + if (this.closed) return; + this.closing = true; + this.closePromise = this.schedule(async () => { + let failure: unknown; + try { + if (this.failure !== undefined) throw this.failure; + if (this.lease !== undefined) { + await this.flushCommitted(); + await this.withGuard(guard => this.releaseLock(guard)); + } + } catch (error) { + failure = error; + } + this.closed = true; + if (this.lease !== undefined) { + try { + await this.lease.handle.close(); + } catch (error) { + failure = failure === undefined ? error : new AggregateError([failure, error], 'State close and handle cleanup failed'); + } + } + if (failure !== undefined) throw failure; + }); + return this.closePromise; + } + + private assertAvailable(requireLoaded = true): void { + if (this.failure !== undefined) throw this.failure; + if (this.closed) throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closed'); + if (requireLoaded && !this.loaded) throw new NoOrgError('STATE_NOT_LOADED', 'State store must be loaded before mutation'); + } + + private assertAdmission(): void { + this.assertAvailable(); + if (this.closing) throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closing'); + } + + private schedule(operation: () => Promise): Promise { + const next = this.mutation.then(operation); + this.mutation = next.then(() => undefined, () => undefined); + return next; + } + + private async flushCommitted(): Promise { + this.assertAvailable(); + await this.withGuard(async guard => { + await this.assertOwnership(guard); + if (this.dirty) await this.writeCandidate(this.values, this.currentRevision, guard); + }); + this.dirty = false; + } + + private async writeCandidate(values: ReadonlyMap, revision: number, guard: DirectoryLease): Promise { + await this.assertOwnership(guard); const temporaryPath = `${this.filePath}.${process.pid}.${randomUUID()}.tmp`; - const body = JSON.stringify( - { schemaVersion: 2, revision: this.currentRevision, state: this.snapshot() }, - null, - 2 - ); + const body = JSON.stringify({ schemaVersion: 2, revision, state: Object.fromEntries(values) }, null, 2); + const handle = await this.io.open(temporaryPath, 'wx', 0o600); + let identity: Stats | undefined; + let failure: unknown; try { - await writeFile(temporaryPath, body, { encoding: 'utf8', mode: 0o600 }); - const handle = await open(temporaryPath, 'r+'); + identity = await handle.stat(); + await handle.writeFile(body, 'utf8'); + await handle.sync(); + } catch (error) { + failure = error; + } + try { + await handle.close(); + } catch (error) { + failure = failure === undefined ? error : new AggregateError([failure, error], 'State write and close failed'); + } + if (failure === undefined) { try { - await handle.sync(); - } finally { - await handle.close(); + await this.assertOwnership(guard); + } catch (error) { + failure = error; } - await rename(temporaryPath, this.filePath); + } + if (failure === undefined) { + // A failed rename may already have replaced the destination. Never + // publish or automatically unlock an outcome that cannot be confirmed. + try { + await this.io.rename(temporaryPath, this.filePath); + await this.syncParent(); + return; + } catch (error) { + this.retainGuard = true; + throw this.fence('STATE_COMMIT_UNCERTAIN', 'State replacement durability is uncertain; store is fenced', error); + } + } + try { + await this.assertDirectory(guard); + const current = await this.io.lstat(temporaryPath); + if (identity !== undefined && sameIdentity(current, identity)) await this.io.rm(temporaryPath); + } catch (cleanupError) { + if (!isMissingFile(cleanupError)) { + throw new AggregateError([failure, cleanupError], 'State write and owned temporary cleanup failed'); + } + } + throw failure; + } + + private fence(code: string, message: string, cause: unknown): NoOrgError { + const failure = new NoOrgError(code, message); + failure.cause = cause; + this.failure = failure; + return failure; + } + + private async openDirectory(path: string): Promise { + const handle = await this.io.open(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + try { + const identity = await handle.stat(); + return { path, handle, dev: identity.dev, ino: identity.ino }; } catch (error) { - await rm(temporaryPath, { force: true }).catch(() => undefined); + await handle.close(); throw error; } } - public async close(): Promise { - if (this.closed) return; - let failure: unknown; + private async syncParent(): Promise { + const parent = await this.openDirectory(dirname(this.filePath)); try { - await this.flush(); - } catch (error) { - failure = error; + await parent.handle.sync(); } finally { - this.closed = true; - await this.releaseLock(); + await parent.handle.close(); } - if (failure !== undefined) throw failure; } - private async enqueue(operation: () => void): Promise { - if (this.closed) throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closed'); - const next = this.mutation.then(operation); - this.mutation = next.catch(() => undefined); - await next; + private async assertDirectory(lease: DirectoryLease): Promise { + let current: Stats; + try { + current = await this.io.lstat(lease.path); + } catch (error) { + throw this.fence('STATE_LOCK_LOST', 'State ownership directory is missing', error); + } + if (!current.isDirectory() || !sameIdentity(current, lease)) { + throw this.fence('STATE_LOCK_LOST', 'State ownership directory was replaced', undefined); + } } - private async acquireLock(): Promise { - if (this.locked) return; - const lockPath = `${this.filePath}.lock`; - await mkdir(dirname(lockPath), { recursive: true }); + private async withGuard(operation: (guard: DirectoryLease) => Promise): Promise { + const path = `${this.filePath}.lock.guard`; try { - await mkdir(lockPath); + await this.io.mkdir(path, { mode: 0o700 }); } catch (error) { - if ( - !isMissingFile(error) && - typeof error === 'object' && - error !== null && - 'code' in error && - error.code === 'EEXIST' - ) { - let owner: LockOwner | undefined; - try { - const raw = await readFile(`${lockPath}/owner`, 'utf8'); - try { - const parsed: unknown = JSON.parse(raw); - if ( - typeof parsed === 'object' && - parsed !== null && - typeof (parsed as { pid?: unknown }).pid === 'number' && - typeof (parsed as { createdAt?: unknown }).createdAt === 'number' && - typeof (parsed as { token?: unknown }).token === 'string' - ) { - owner = parsed as LockOwner; - } - } catch { - const pid = Number.parseInt(raw, 10); - if (Number.isInteger(pid)) owner = { pid, createdAt: 0, token: 'prior' }; - } - } catch { - owner = undefined; - } - let lockAge: number; - try { - lockAge = Date.now() - (await stat(lockPath)).mtimeMs; - } catch { - lockAge = Number.POSITIVE_INFINITY; - } - if ( - (owner !== undefined && isLiveProcess(owner.pid)) || - (owner === undefined && lockAge < staleLockMs) || - (owner !== undefined && owner.createdAt > 0 && Date.now() - owner.createdAt < staleLockMs) - ) { - throw new NoOrgError('STATE_LOCKED', `State file ${this.filePath} is already in use`); - } - await rm(lockPath, { recursive: true, force: true }); - await mkdir(lockPath); - } else { - throw error; + if (typeof error === 'object' && error !== null && 'code' in error && error.code === 'EEXIST') { + throw new NoOrgError('STATE_LOCKED', `State file ${this.filePath} is already in use or requires guard recovery`); } + throw error; } - const owner: LockOwner = { pid: process.pid, createdAt: Date.now(), token: randomUUID() }; + let guard: DirectoryLease; try { - await writeFile(`${lockPath}/owner`, JSON.stringify(owner), { - encoding: 'utf8', - mode: 0o600, - }); + guard = await this.openDirectory(path); + } catch (error) { + throw this.fence('STATE_LOCK_LOST', 'State namespace guard acquisition is incomplete', error); + } + let result!: T; + let failure: unknown; + try { + await guard.handle.sync(); + await this.syncParent(); + result = await operation(guard); + } catch (error) { + failure = error; + } + let owned = true; + try { + await this.assertDirectory(guard); + } catch (error) { + owned = false; + failure = failure === undefined ? error : new AggregateError([failure, error], 'State operation and namespace identity check failed'); + } + try { + await guard.handle.close(); + } catch (error) { + owned = false; + this.retainGuard = true; + const cleanup = this.fence('STATE_LOCK_LOST', 'State namespace guard handle could not be closed', error); + failure = failure === undefined ? cleanup : new AggregateError([failure, cleanup], 'State operation and handle cleanup failed'); + } + if (owned && !this.retainGuard) { + try { + await this.io.rmdir(path); + } catch (error) { + const cleanup = this.fence('STATE_LOCK_LOST', 'State namespace guard could not be released safely', error); + failure = failure === undefined ? cleanup : new AggregateError([failure, cleanup], 'State operation and namespace release failed'); + } + } + if (failure !== undefined) throw failure; + return result; + } + + private async observeLock(): Promise { + const path = `${this.filePath}.lock`; + let identity: Stats; + try { + identity = await this.io.lstat(path); } catch (error) { - await rm(lockPath, { recursive: true, force: true }).catch(() => undefined); + if (isMissingFile(error)) return undefined; throw error; } - this.locked = true; + if (!identity.isDirectory()) throw new NoOrgError('STATE_LOCKED', 'State lock path is not an owned directory'); + let owner: LockOwner | undefined; + try { + owner = parseOwner(await this.io.readFile(`${path}/owner`, 'utf8')); + } catch (error) { + if (!isMissingFile(error)) throw error; + } + return { identity, owner }; } - private async releaseLock(): Promise { - if (!this.locked) return; - await rm(`${this.filePath}.lock`, { recursive: true, force: true }); - this.locked = false; + private assertReclaimable(lock: ObservedLock): void { + const owner = lock.owner; + if (owner === undefined) throw new NoOrgError('STATE_LOCKED', 'State lock owner is incomplete or ambiguous'); + const age = Date.now() - (owner.createdAt === 0 ? lock.identity.mtimeMs : owner.createdAt); + if (age < staleLockMs || isLiveProcess(owner.pid)) { + throw new NoOrgError('STATE_LOCKED', `State file ${this.filePath} is already in use`); + } + } + + private async assertOwnership(guard: DirectoryLease): Promise { + await this.assertDirectory(guard); + const lease = this.lease; + if (lease === undefined) throw this.fence('STATE_LOCK_LOST', 'State store has no acquired lock', undefined); + const current = await this.observeLock(); + if (current === undefined || !sameIdentity(current.identity, lease) || + current.owner?.token !== lease.owner.token || current.owner.pid !== lease.owner.pid) { + throw this.fence('STATE_LOCK_LOST', 'State lock no longer belongs to this store', undefined); + } + } + + private async releaseLock(guard: DirectoryLease): Promise { + await this.assertOwnership(guard); + const lease = this.lease!; + await lease.handle.close(); + await this.io.rm(lease.path, { recursive: true }); + this.lease = undefined; + this.loaded = false; } } @@ -268,14 +515,16 @@ export class MemoryStateStore implements StateStore { private readonly values = new Map(); private mutation = Promise.resolve(); private closed = false; + private closing = false; + private closePromise: Promise | undefined; private currentRevision = 0; public isOpen(): boolean { - return !this.closed; + return !this.closed && !this.closing; } public async load(): Promise { - if (this.closed) throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closed'); + if (this.closed || this.closing) throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closed'); } public get(key: string): T | undefined { @@ -292,10 +541,10 @@ export class MemoryStateStore implements StateStore { } public async set(key: string, value: JsonValue): Promise { - if (this.closed) throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closed'); - jsonValueSchema.parse(value); + if (this.closed || this.closing) throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closed'); + const admitted = clone(jsonValueSchema.parse(value)); await this.enqueue(() => { - this.values.set(key, clone(value)); + this.values.set(key, admitted); this.currentRevision += 1; }); } @@ -320,12 +569,15 @@ export class MemoryStateStore implements StateStore { } public async close(): Promise { - await this.flush(); - this.closed = true; + if (this.closePromise !== undefined) return this.closePromise; + this.closing = true; + this.closePromise = this.mutation.then(() => { this.closed = true; }); + this.mutation = this.closePromise; + return this.closePromise; } private async enqueue(operation: () => void): Promise { - if (this.closed) throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closed'); + if (this.closed || this.closing) throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closed'); const next = this.mutation.then(operation); this.mutation = next.catch(() => undefined); await next; diff --git a/src/tasks/task-repository.ts b/src/tasks/task-repository.ts index 1650aae..4f55e1b 100644 --- a/src/tasks/task-repository.ts +++ b/src/tasks/task-repository.ts @@ -5,6 +5,7 @@ import { assertCondition, NoOrgError } from '../domain/errors'; import type { AgentResult, JsonValue, + ParsedTaskQuery, TaskError, TaskQuery, TaskRecord, @@ -53,7 +54,6 @@ function copy(value: T): T { export class TaskRepository { private tasks = new Map(); private mutation = Promise.resolve(); - private persistence = Promise.resolve(); public constructor( private readonly state: StateStore, @@ -67,32 +67,29 @@ export class TaskRepository { ) {} public async load(): Promise { - const persisted = this.state.get('tasks'); - if (persisted === undefined) return; - const parsed = taskListSchema.safeParse(persisted); - if (!parsed.success) - throw new NoOrgError( - 'TASK_STATE_INVALID', - 'Persisted task records do not match the task schema' - ); - this.tasks = new Map(parsed.data.map(task => [task.id, copy(task) as TaskRecord])); - for (const task of this.tasks.values()) { - if (task.parentTaskId !== undefined && !this.tasks.has(task.parentTaskId)) - throw new NoOrgError('TASK_PARENT_NOT_FOUND', 'Persisted task parent was not found'); - if (task.dependencyIds.some(dependencyId => !this.tasks.has(dependencyId))) - throw new NoOrgError( - 'TASK_DEPENDENCY_NOT_FOUND', - 'Persisted task dependency was not found' - ); - } - if (this.hasDependencyCycle() || this.hasParentCycle()) - throw new NoOrgError('TASK_DEPENDENCY_CYCLE', 'Persisted tasks contain a dependency cycle'); + await this.serialize(async () => { + const persisted = this.state.get('tasks'); + if (persisted === undefined) return; + const parsed = taskListSchema.safeParse(persisted); + if (!parsed.success) + throw new NoOrgError('TASK_STATE_INVALID', 'Persisted task records do not match the task schema'); + const candidate = new Map(parsed.data.map(task => [task.id, copy(task) as TaskRecord])); + for (const task of candidate.values()) { + if (task.parentTaskId !== undefined && !candidate.has(task.parentTaskId)) + throw new NoOrgError('TASK_PARENT_NOT_FOUND', 'Persisted task parent was not found'); + if (task.dependencyIds.some(dependencyId => !candidate.has(dependencyId))) + throw new NoOrgError('TASK_DEPENDENCY_NOT_FOUND', 'Persisted task dependency was not found'); + } + if (this.hasDependencyCycle(candidate) || this.hasParentCycle(candidate)) + throw new NoOrgError('TASK_DEPENDENCY_CYCLE', 'Persisted tasks contain a dependency cycle'); + this.tasks = candidate; + }); } public async recoverInterruptedTasks(): Promise { const recovered: TaskRecord[] = []; - await this.enqueue(() => { - for (const task of this.tasks.values()) { + await this.enqueue(candidate => { + for (const task of candidate.values()) { if (task.status !== 'running') continue; const now = this.clock.now().toISOString(); const retryable = task.attempt < task.maxAttempts; @@ -105,11 +102,11 @@ export class TaskRepository { const next: TaskRecord = retryable ? { ...restartable, status: 'queued', error, updatedAt: now } : { ...task, status: 'failed', error, completedAt: now, updatedAt: now }; - this.tasks.set(task.id, next); + candidate.set(task.id, next); recovered.push(copy(next)); } + return recovered.length > 0; }); - if (recovered.length > 0) await this.persist(); return recovered; } @@ -158,12 +155,11 @@ export class TaskRepository { createdAt: now, updatedAt: now, }; - let created = true; let result!: TaskRecord; - await this.enqueue(() => { + await this.enqueue(candidate => { if (task.idempotencyKey !== undefined) { - const existing = [...this.tasks.values()].find( - candidate => candidate.idempotencyKey === task.idempotencyKey + const existing = [...candidate.values()].find( + existingTask => existingTask.idempotencyKey === task.idempotencyKey ); if (existing !== undefined) { if (taskFingerprint(existing) !== taskFingerprint(task)) @@ -171,15 +167,14 @@ export class TaskRepository { 'IDEMPOTENCY_CONFLICT', 'Idempotency key was already used for a different task' ); - created = false; result = copy(existing); - return; + return false; } } - if (task.parentTaskId !== undefined && !this.tasks.has(task.parentTaskId)) + if (task.parentTaskId !== undefined && !candidate.has(task.parentTaskId)) throw new NoOrgError('TASK_PARENT_NOT_FOUND', 'Parent task was not found'); for (const dependencyId of task.dependencyIds) { - if (!this.tasks.has(dependencyId)) + if (!candidate.has(dependencyId)) throw new NoOrgError( 'TASK_DEPENDENCY_NOT_FOUND', `Dependency task ${dependencyId} was not found` @@ -188,8 +183,8 @@ export class TaskRepository { const parentChildren = task.parentTaskId === undefined ? 0 - : [...this.tasks.values()].filter( - candidate => candidate.parentTaskId === task.parentTaskId + : [...candidate.values()].filter( + existingTask => existingTask.parentTaskId === task.parentTaskId ).length; if (parentChildren >= (this.limits.maxWorkflowChildren ?? 100)) { throw new NoOrgError( @@ -199,7 +194,7 @@ export class TaskRepository { } const normalizedTask = { ...task, - workflowDepth: this.workflowDepth(task), + workflowDepth: this.workflowDepth(task, candidate), }; if (normalizedTask.workflowDepth > (this.limits.maxWorkflowDepth ?? 32)) { throw new NoOrgError( @@ -207,14 +202,12 @@ export class TaskRepository { 'Task relationships exceed the configured workflow depth' ); } - this.tasks.set(task.id, normalizedTask); - if (this.hasDependencyCycle() || this.hasParentCycle()) { - this.tasks.delete(task.id); + candidate.set(task.id, normalizedTask); + if (this.hasDependencyCycle(candidate) || this.hasParentCycle(candidate)) { throw new NoOrgError('TASK_DEPENDENCY_CYCLE', 'Task relationships would create a cycle'); } result = copy(normalizedTask); }); - if (created) await this.persist(); return result; } @@ -228,27 +221,32 @@ export class TaskRepository { ...query, ...(status === undefined ? {} : { status }), }); - const sorted = [...this.tasks.values()] - .filter(task => parsedQuery.status === undefined || task.status === parsedQuery.status) - .filter(task => parsedQuery.agentId === undefined || task.agentId === parsedQuery.agentId) - .filter( - task => - parsedQuery.capability === undefined || - task.requiredCapabilities.includes(parsedQuery.capability) - ) - .filter( - task => - parsedQuery.idempotencyKey === undefined || - task.idempotencyKey === parsedQuery.idempotencyKey - ) - .sort((left, right) => { - const priorityDifference = this.effectivePriority(right) - this.effectivePriority(left); - return priorityDifference || left.createdAt.localeCompare(right.createdAt); - }); + const sorted = this.select(parsedQuery); const end = parsedQuery.offset + parsedQuery.limit; return sorted.slice(parsedQuery.offset, end).map(copy); } + public internalSnapshot(status?: TaskStatus): { readonly tasks: readonly TaskRecord[]; readonly total: number } { + const tasks = this.select(status === undefined ? {} : { status }).map(copy); + return { tasks, total: tasks.length }; + } + + private select(query: Omit): TaskRecord[] { + const selected: TaskRecord[] = []; + for (const task of this.tasks.values()) { + if ( + (query.status === undefined || task.status === query.status) && + (query.agentId === undefined || task.agentId === query.agentId) && + (query.capability === undefined || task.requiredCapabilities.includes(query.capability)) && + (query.idempotencyKey === undefined || task.idempotencyKey === query.idempotencyKey) + ) selected.push(task); + } + return selected.sort((left, right) => { + const priorityDifference = this.effectivePriority(right) - this.effectivePriority(left); + return priorityDifference || left.createdAt.localeCompare(right.createdAt); + }); + } + public async start(id: string, agentId: string): Promise { return this.transition(id, task => { assertCondition( @@ -397,12 +395,7 @@ export class TaskRepository { } public async persist(): Promise { - const next = this.persistence.then(async () => { - await this.state.set('tasks', [...this.tasks.values()] as unknown as JsonValue); - await this.state.flush(); - }); - this.persistence = next.catch(() => undefined); - await next; + await this.serialize(() => this.state.flush()); } private async transition( @@ -410,61 +403,74 @@ export class TaskRepository { update: (task: TaskRecord) => TaskRecord ): Promise { let changed!: TaskRecord; - await this.enqueue(() => { - const current = this.tasks.get(id); + await this.enqueue(candidate => { + const current = candidate.get(id); if (!current) throw new NoOrgError('TASK_NOT_FOUND', `Task ${id} was not found`); changed = update(copy(current)); - this.tasks.set(id, copy(changed)); + candidate.set(id, copy(changed)); }); - await this.persist(); return copy(changed); } - private async enqueue(operation: () => void): Promise { + private async enqueue(operation: (candidate: Map) => void | boolean): Promise { + await this.serialize(async () => { + if (!this.state.isOpen()) + throw new NoOrgError('STATE_STORE_UNAVAILABLE', 'Task mutations require an open state store'); + const candidate = new Map(this.tasks); + if (operation(candidate) === false) return; + await this.state.set('tasks', [...candidate.values()] as unknown as JsonValue); + this.tasks = candidate; + }); + } + + private serialize(operation: () => Promise): Promise { const next = this.mutation.then(operation); - this.mutation = next.catch(() => undefined); - await next; + this.mutation = next.then(() => undefined, () => undefined); + return next; } - private hasDependencyCycle(): boolean { + private hasDependencyCycle(tasks: ReadonlyMap = this.tasks): boolean { const visiting = new Set(); const visited = new Set(); const visit = (id: string): boolean => { if (visiting.has(id)) return true; if (visited.has(id)) return false; visiting.add(id); - const task = this.tasks.get(id); + const task = tasks.get(id); if (task?.dependencyIds.some(visit)) return true; visiting.delete(id); visited.add(id); return false; }; - return [...this.tasks.keys()].some(visit); + return [...tasks.keys()].some(visit); } - private hasParentCycle(): boolean { + private hasParentCycle(tasks: ReadonlyMap = this.tasks): boolean { const visiting = new Set(); const visited = new Set(); const visit = (id: string): boolean => { if (visiting.has(id)) return true; if (visited.has(id)) return false; visiting.add(id); - const parentId = this.tasks.get(id)?.parentTaskId; + const parentId = tasks.get(id)?.parentTaskId; if (parentId !== undefined && visit(parentId)) return true; visiting.delete(id); visited.add(id); return false; }; - return [...this.tasks.keys()].some(visit); + return [...tasks.keys()].some(visit); } - private workflowDepth(task: Pick): number { + private workflowDepth( + task: Pick, + tasks: ReadonlyMap = this.tasks + ): number { const related = [ ...(task.parentTaskId === undefined ? [] : [task.parentTaskId]), ...task.dependencyIds, ]; if (related.length === 0) return 0; - return 1 + Math.max(...related.map(id => this.tasks.get(id)?.workflowDepth ?? 0)); + return 1 + Math.max(...related.map(id => tasks.get(id)?.workflowDepth ?? 0)); } private effectivePriority(task: TaskRecord): number { diff --git a/tests/README.md b/tests/README.md index a3f0e76..d11ab9c 100644 --- a/tests/README.md +++ b/tests/README.md @@ -9,6 +9,7 @@ Jest suites for the NoOrg runtime, written in TypeScript and run from the reposi | `unit/` | Focused tests for agents, configuration, events, the GNN conformance script, metrics, providers, security, state, tasks, and the unit corpus. | | `integration/` | End-to-end coordinator and HTTP integration tests. | | `live/` | Opt-in live OpenAI provider test; excluded from the ordinary local run. | +| `fixtures/` | Actual native file-write/fsync barriers and an independent Node IPC state-lock worker. | | `tsconfig.json` | TypeScript configuration shared by the suites. | ## Commands @@ -22,3 +23,31 @@ npm run test:live # live OpenAI suite; requires credentials ``` Coverage thresholds are 90% lines and 80% branches. `tests/live` is excluded from `npm test` and from the coverage run, so no default command reaches the network. See the [testing documentation](../docs/testing.md) for the gate arms and the live-provider profile, and the [validation gate](../docs/gate.md) for `./run.sh`. + +## Focused runtime boundaries + +`task-commit`, `state-lock` and `coordinator-commit` exercise real file-state +admission, pre-rename failure visibility, post-rename fencing, physical/token/ +guard ownership and agent reservations. The stale-owner race uses independent +Node processes and IPC barriers, not mocked filesystem outcomes. Both timer +resolve/reject cases hold real completion/accounting writes past the execution +deadline, then require exact usage/events and reservation/file-state lease release. + +`queue-traversal` exercises runnable and housekeeping work beyond the public +100-record page. `usage-accounting` covers additive deltas, restart history, +failed-commit reentry and late cancellation/timeout settlement. Provider receipts +use the existing offline transport seam, not live model calls. + +`http-boundaries` and security cases cover UTF-8 bearer lengths, throttling, +workflow admission and complete queue totals. HTTP fixtures consume responses +and dispose their owned servers even on failure. `config-launch` parses the +actual example and uses real Node `--env-file` children to prove inherited +precedence and missing-file negative behavior. `provider-retention` covers +disabled history and enabled rolling-window failures/rollover. + +These cases need the existing Node >=22/Jest/TypeScript/tsx dependency cohort, +bounded child/IPC/loopback access and native directory handles/fsync. Use owned +temporary state/output roots. Sanitized child environments must be accounted +for separately; parent-only temp/cache variables are not inherited implicitly. +Focused behavioral receipts do not establish coverage thresholds, live-provider +quality, scientific validity, GNN pin migration or full release acceptance. diff --git a/tests/fixtures/state-boundary.ts b/tests/fixtures/state-boundary.ts new file mode 100644 index 0000000..c411462 --- /dev/null +++ b/tests/fixtures/state-boundary.ts @@ -0,0 +1,86 @@ +/// +import type { FileHandle } from 'node:fs/promises'; +import { dirname } from 'node:path'; +import { nodeStateStoreIO, type FileStateStoreIO } from '../../src/state/state-store'; + +export type StateFault = 'write' | 'sync' | 'rename-before' | 'rename-after' | 'directory-sync' | 'guard-open'; +export interface CommitBarrier { + readonly entered: Promise; + release(): void; +} + +/** Faults/barriers surround real native file effects; successful effects are never simulated. */ +export class StateBoundary { + public readonly io: FileStateStoreIO; + private fault: StateFault | undefined; + private renamed = false; + private barrier: { enter(): void; readonly released: Promise } | undefined; + + public constructor(private readonly filePath: string, public readonly error = new Error('controlled state I/O failure')) { + this.io = { + ...nodeStateStoreIO, + open: async (path, flags, mode) => { + if (String(path) === `${this.filePath}.lock.guard` && this.fault === 'guard-open') { + this.fault = undefined; + throw this.error; + } + const handle = await nodeStateStoreIO.open(path, flags, mode); + const temporary = String(path).startsWith(`${this.filePath}.`) && String(path).endsWith('.tmp'); + const parent = String(path) === dirname(this.filePath); + return new Proxy(handle, { + get: (target, property) => { + if (property === 'writeFile') return async (...args: Parameters) => { + if (temporary) { + const barrier = this.barrier; + this.barrier = undefined; + if (barrier !== undefined) { + barrier.enter(); + await barrier.released; + } + if (this.fault === 'write') { + this.fault = undefined; + throw this.error; + } + } + return target.writeFile(...args); + }; + if (property === 'sync') return async () => { + if ((temporary && this.fault === 'sync') || (parent && this.renamed && this.fault === 'directory-sync')) { + this.fault = undefined; + throw this.error; + } + await target.sync(); + }; + const value: unknown = Reflect.get(target, property, target); + return typeof value === 'function' ? value.bind(target) : value; + }, + }); + }, + rename: async (from, to) => { + if (this.fault === 'rename-before') { + this.fault = undefined; + throw this.error; + } + await nodeStateStoreIO.rename(from, to); + this.renamed = true; + if (this.fault === 'rename-after') { + this.fault = undefined; + throw this.error; + } + }, + }; + } + + public failNext(fault: StateFault): void { + this.fault = fault; + this.renamed = false; + } + + public pauseNextWrite(): CommitBarrier { + if (this.barrier !== undefined) throw new Error('A state write barrier is already armed'); + const { promise: entered, resolve: enter } = Promise.withResolvers(); + const { promise: released, resolve: release } = Promise.withResolvers(); + this.barrier = { enter, released }; + return { entered, release }; + } +} diff --git a/tests/fixtures/state-lock-worker.ts b/tests/fixtures/state-lock-worker.ts new file mode 100644 index 0000000..93eb191 --- /dev/null +++ b/tests/fixtures/state-lock-worker.ts @@ -0,0 +1,56 @@ +import { on, once } from 'node:events'; +import { z } from 'zod'; +import { NoOrgError } from '../../src/domain/errors'; +import { FileStateStore, nodeStateStoreIO, type FileStateStoreIO } from '../../src/state/state-store'; + +async function main(): Promise { + const file = z.string().min(1).parse(process.argv[2]); + const mode = z.enum(['pause-stale-read', 'writer']).parse(process.argv[3]); + const send = process.send?.bind(process); + if (send === undefined) throw new Error('Lock worker requires its parent IPC channel'); + let observed = false; + // Preserve native readFile overloads: the barrier forwards arguments/results unchanged. + const readOwnedFile = (async (...args: Parameters) => { + const contents = await nodeStateStoreIO.readFile(...args); + if (mode === 'pause-stale-read' && !observed && String(args[0]) === `${file}.lock/owner`) { + observed = true; + const resumed = once(process, 'message', { signal: AbortSignal.timeout(5000) }); + send({ type: 'observed' }); + const [message] = await resumed; + z.object({ type: z.literal('resume') }).parse(message); + } + return contents; + }) as typeof nodeStateStoreIO.readFile; + const io: FileStateStoreIO = { ...nodeStateStoreIO, readFile: readOwnedFile }; + const store = new FileStateStore(file, io); + try { + try { + await store.load(); + } catch (error) { + if (!(error instanceof NoOrgError)) throw error; + send({ type: 'rejected', code: error.code }); + return; + } + if (mode === 'pause-stale-read') { + send({ type: 'unexpected-acquisition' }); + return; + } + await store.set('writer', 'A'); + send({ type: 'owned', pid: process.pid, revision: store.revision() }); + for await (const [message] of on(process, 'message', { signal: AbortSignal.timeout(10_000) })) { + const command = z.object({ type: z.enum(['flush', 'close']) }).parse(message); + if (command.type === 'close') break; + await store.flush(); + send({ type: 'flushed', revision: store.revision(), value: store.get('writer') }); + } + } finally { + await store.close(); + if (process.connected) process.disconnect(); + } +} + +void main().catch(error => { + process.stderr.write(String(error) + '\n'); + process.exitCode = 1; + if (process.connected) process.disconnect(); +}); diff --git a/tests/integration/coordinator-commit.test.ts b/tests/integration/coordinator-commit.test.ts new file mode 100644 index 0000000..f9cbf9d --- /dev/null +++ b/tests/integration/coordinator-commit.test.ts @@ -0,0 +1,285 @@ +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { z } from 'zod'; +import { AbstractAgent } from '../../src/agents/abstract-agent'; +import { AgentRegistry } from '../../src/agents/registry'; +import { Coordinator } from '../../src/coordination/coordinator'; +import { delay, SystemClock } from '../../src/domain/clock'; +import { NoOrgError } from '../../src/domain/errors'; +import { runtimeStateSchema, type AgentResult, type AgentTask, type TaskRecord } from '../../src/domain/types'; +import { EventBus } from '../../src/events/event-bus'; +import { MemoryLogger } from '../../src/logging/logger'; +import { Metrics } from '../../src/metrics/metrics'; +import { GovernedProvider } from '../../src/providers/governed-provider'; +import { DeterministicProvider, OpenAIProvider, type LLMProvider } from '../../src/providers/provider'; +import { FileStateStore } from '../../src/state/state-store'; +import { TaskRepository } from '../../src/tasks/task-repository'; +import { StateBoundary } from '../fixtures/state-boundary'; + +class EffectAgent extends AbstractAgent { + public effects = 0; + public constructor() { super({ id: 'effect', name: 'Effect', capabilities: ['effect'], maxConcurrentTasks: 1 }); } + protected override async run(_task: AgentTask): Promise { + this.effects += 1; + return { data: { executed: true }, summary: 'Executed' }; + } +} + +class CommitAgent extends AbstractAgent { + public effects = 0; + public signal: AbortSignal | undefined; + + public constructor( + private readonly outcome: 'resolve' | 'reject', + private readonly beforeSettlement: () => void + ) { + super({ id: 'effect', name: 'Effect', capabilities: ['effect'], maxConcurrentTasks: 1 }); + } + + protected override async run(task: AgentTask, signal: AbortSignal): Promise { + this.effects += 1; + this.signal = signal; + const data = await this.ask( + { operation: 'effect', input: null }, + z.object({ executed: z.boolean() }), + signal, + task.id + ); + this.beforeSettlement(); + if (this.outcome === 'reject') + throw new NoOrgError('EXECUTION_REJECTED', 'Agent execution rejected before its deadline', false); + return { data, summary: 'Executed' }; + } +} + +/** A real durable cancellation at the public repository boundary before start returns to its consumer. */ +class CancelAtStartRepository extends TaskRepository { + public override async start(...args: Parameters): Promise { + const started = await super.start(...args); + await super.cancel(started.id); + return started; + } +} + +function commitRuntime( + state: FileStateStore, + repository: TaskRepository, + agent: AbstractAgent, + provider: LLMProvider = new DeterministicProvider(), + events = new EventBus() +): { coordinator: Coordinator; registry: AgentRegistry } { + const registry = new AgentRegistry(); + registry.register(agent); + return { registry, coordinator: new Coordinator({ state, repository, registry, + provider, events, metrics: new Metrics(), logger: new MemoryLogger(), + clock: new SystemClock(), pollIntervalMs: 5, maxConcurrentTasks: 1, defaultTaskTimeoutMs: 1000 }) }; +} + +async function waitFor(condition: () => boolean): Promise { + const deadline = Date.now() + 3000; + while (!condition()) { + if (Date.now() >= deadline) throw new Error('Durable task execution did not reach the expected boundary'); + await delay(5); + } +} + +const request = { agentId: 'effect', name: 'Durable effect', description: 'Do not execute an uncommitted start', input: null }; + +describe('Coordinator durable task/lease admission', () => { + it('keeps a pending start invisible and releases the real reservation when it fails before a queued cancellation/shutdown', async () => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-dispatch-commit-')); + const file = join(directory, 'state.json'); + const boundary = new StateBoundary(file); + const state = new FileStateStore(file, boundary.io); + const repository = new TaskRepository(state, new SystemClock()); + const agent = new EffectAgent(); + const { coordinator, registry } = commitRuntime(state, repository, agent); + let release: (() => void) | undefined; + try { + await state.load(); + await repository.load(); + const task = await repository.create(request); + const barrier = boundary.pauseNextWrite(); + release = barrier.release; + await coordinator.start(); + await barrier.entered; + expect(coordinator.getTask(task.id)?.status).toBe('queued'); + expect(registry.activeCount()).toBe(1); + expect(agent.effects).toBe(0); + const cancellation = coordinator.cancelTask(task.id); + const shutdown = coordinator.shutdown(); + boundary.failNext('write'); + barrier.release(); + expect(await cancellation).toEqual(expect.objectContaining({ status: 'cancelled', attempt: 0 })); + await shutdown; + expect(registry.activeCount()).toBe(0); + expect(agent.effects).toBe(0); + expect(runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state.tasks).toEqual([ + expect.objectContaining({ id: task.id, status: 'cancelled', attempt: 0 }), + ]); + } finally { + release?.(); + await coordinator.shutdown(); + await rm(directory, { recursive: true, force: true }); + } + }); + + it('does not execute a task already durably cancelled before the start promise returns', async () => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-cancel-before-effect-')); + const state = new FileStateStore(join(directory, 'state.json')); + const repository = new CancelAtStartRepository(state, new SystemClock()); + const agent = new EffectAgent(); + const { coordinator, registry } = commitRuntime(state, repository, agent); + try { + await state.load(); + await repository.load(); + const task = await repository.create(request); + await coordinator.start(); + const deadline = Date.now() + 3000; + while (coordinator.getTask(task.id)?.status !== 'cancelled' || registry.activeCount() !== 0) { + if (Date.now() >= deadline) throw new Error('Cancellation did not settle'); + await delay(5); + } + expect(coordinator.getTask(task.id)?.attempt).toBe(1); + expect(agent.effects).toBe(0); + } finally { + await coordinator.shutdown(); + await rm(directory, { recursive: true, force: true }); + } + }); + + it('cancels an accepted pending start when shutdown begins before execution admission', async () => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-shutdown-before-effect-')); + const file = join(directory, 'state.json'); + const boundary = new StateBoundary(file); + const state = new FileStateStore(file, boundary.io); + const repository = new TaskRepository(state, new SystemClock()); + const agent = new EffectAgent(); + const { coordinator, registry } = commitRuntime(state, repository, agent); + let release: (() => void) | undefined; + try { + await state.load(); + await repository.load(); + const task = await repository.create(request); + const queued = await repository.create({ ...request, name: 'Retained queued work' }); + const barrier = boundary.pauseNextWrite(); + release = barrier.release; + await coordinator.start(); + await barrier.entered; + const shutdown = coordinator.shutdown(); + expect(coordinator.getStatus()).toBe('stopping'); + barrier.release(); + await shutdown; + expect(coordinator.getTask(task.id)).toEqual(expect.objectContaining({ status: 'cancelled', attempt: 1 })); + expect(coordinator.getTask(queued.id)).toEqual(expect.objectContaining({ status: 'queued', attempt: 0 })); + expect(runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state.tasks).toEqual([ + expect.objectContaining({ id: task.id, status: 'cancelled', attempt: 1 }), + expect.objectContaining({ id: queued.id, status: 'queued', attempt: 0 }), + ]); + expect(registry.activeCount()).toBe(0); + expect(agent.effects).toBe(0); + } finally { + release?.(); + await coordinator.shutdown(); + await rm(directory, { recursive: true, force: true }); + } + }); + + it.each(['resolve', 'reject'] as const)('clears the execution timer before a %s outcome waits for a durable commit', async outcome => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-execution-deadline-')); + const file = join(directory, 'state.json'); + const boundary = new StateBoundary(file); + const state = new FileStateStore(file, boundary.io); + const repository = new TaskRepository(state, new SystemClock()); + const usage = { promptTokens: 3, completionTokens: 2, totalTokens: 5, costUsd: 5 }; + let transportCalls = 0; + const provider = new GovernedProvider(new OpenAIProvider({ + apiKey: 'unused-offline-test-key', model: 'offline-test', timeoutMs: 1000, maxRetries: 0, retryBaseMs: 1, + promptCostPerMillionUsd: 1_000_000, completionCostPerMillionUsd: 1_000_000, + transport: { complete: async () => { + transportCalls += 1; + return { id: `offline-${transportCalls}`, model: 'offline-test', parsed: { executed: true }, usage: { + prompt_tokens: usage.promptTokens, completion_tokens: usage.completionTokens, total_tokens: usage.totalTokens, + } }; + } }, + })); + const events = new EventBus(); + let commitEntered = false; + let release: (() => void) | undefined; + const agent = new CommitAgent(outcome, () => { + const barrier = boundary.pauseNextWrite(); + release = barrier.release; + void barrier.entered.then(() => { commitEntered = true; }); + }); + const { coordinator, registry } = commitRuntime(state, repository, agent, provider, events); + try { + await state.load(); + await repository.load(); + const task = await repository.create({ ...request, timeoutMs: 100, maxAttempts: 2, retryBackoffMs: 0 }); + await coordinator.start(); + await waitFor(() => commitEntered); + expect(agent.effects).toBe(1); + expect(registry.activeCount()).toBe(1); + expect(provider.getTaskUsage(task.id)).toEqual(usage); + + // The real next write is completion on resolve, or usage accounting on rejection. + // Keep it blocked past the execution deadline without faking either the clock or commit. + await delay(task.timeoutMs + 25); + expect(agent.signal?.aborted).toBe(false); + expect(registry.activeCount()).toBe(1); + expect(coordinator.getTask(task.id)).toEqual(expect.objectContaining({ status: 'running', attempt: 1 })); + expect(coordinator.getTask(task.id)?.usage).toBeUndefined(); + expect(coordinator.getHealth().lastFailure).toBeUndefined(); + expect(events.getHistory()).toEqual([]); + expect(runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state.tasks).toEqual([ + expect.objectContaining({ id: task.id, status: 'running', attempt: 1 }), + ]); + + release?.(); + await waitFor(() => registry.activeCount() === 0); + const settled = coordinator.getTask(task.id); + expect(settled).toEqual(expect.objectContaining({ + status: outcome === 'resolve' ? 'completed' : 'failed', attempt: 1, usage, + })); + expect(agent.effects).toBe(1); + expect(transportCalls).toBe(1); + expect(provider.getTaskUsage(task.id)).toEqual(usage); + expect(agent.signal?.aborted).toBe(false); + expect(events.getHistory().map(event => event.type)).toEqual([ + outcome === 'resolve' ? 'task.completed' : 'task.failed', + ]); + const metrics = coordinator.getMetrics().renderPrometheus(); + expect(metrics).not.toContain('noorg_tasks_retried '); + expect(metrics).not.toContain('noorg_tasks_cancelled '); + if (outcome === 'resolve') { + expect(settled?.result?.data).toEqual({ executed: true }); + expect(settled?.error).toBeUndefined(); + expect(coordinator.getHealth().lastFailure).toBeUndefined(); + expect(metrics).toContain('noorg_tasks_completed 1\n'); + expect(metrics).not.toContain('noorg_tasks_failed '); + } else { + expect(settled?.result).toBeUndefined(); + expect(settled?.error?.code).toBe('EXECUTION_REJECTED'); + expect(coordinator.getHealth().lastFailure?.code).toBe('EXECUTION_REJECTED'); + expect(metrics).toContain('noorg_tasks_failed 1\n'); + expect(metrics).not.toContain('noorg_tasks_completed '); + } + await coordinator.shutdown(); + expect(registry.activeCount()).toBe(0); + const reopened = new FileStateStore(file); + try { + await reopened.load(); + const persisted = new TaskRepository(reopened, new SystemClock()); + await persisted.load(); + expect(persisted.internalSnapshot().tasks).toEqual([settled]); + } finally { + await reopened.close(); + } + } finally { + release?.(); + await coordinator.shutdown(); + await rm(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/integration/http-boundaries.test.ts b/tests/integration/http-boundaries.test.ts new file mode 100644 index 0000000..e2b9c3d --- /dev/null +++ b/tests/integration/http-boundaries.test.ts @@ -0,0 +1,110 @@ +import { once } from 'node:events'; +import { z } from 'zod'; +import { AgentRegistry } from '../../src/agents/registry'; +import { Coordinator } from '../../src/coordination/coordinator'; +import { SystemClock } from '../../src/domain/clock'; +import { taskRecordSchema } from '../../src/domain/types'; +import { EventBus } from '../../src/events/event-bus'; +import { createHttpServer } from '../../src/http/server'; +import { MemoryLogger } from '../../src/logging/logger'; +import { Metrics } from '../../src/metrics/metrics'; +import { DeterministicProvider } from '../../src/providers/provider'; +import { MemoryStateStore } from '../../src/state/state-store'; + +const errorSchema = z.object({ error: z.object({ code: z.string(), requestId: z.string().min(1) }) }); + +describe('HTTP authentication and workflow boundaries', () => { + it('returns structured 401/429 rather than 500 for equal-character unequal-byte bearer tokens', async () => { + const coordinator = new Coordinator({ + state: new MemoryStateStore(), registry: new AgentRegistry(), provider: new DeterministicProvider(), + events: new EventBus(), metrics: new Metrics(), logger: new MemoryLogger(), clock: new SystemClock(), + pollIntervalMs: 5, maxConcurrentTasks: 1, defaultTaskTimeoutMs: 1000, + }); + const server = createHttpServer({ coordinator, metricsEnabled: false, authToken: 'x', authFailureThreshold: 2 }); + const listening = once(server, 'listening'); + server.listen(0, '127.0.0.1'); + try { + await listening; + const address = server.address(); + if (address === null || typeof address === 'string') throw new Error('HTTP fixture did not bind'); + const url = `http://127.0.0.1:${address.port}/v1/tasks`; + const first = await fetch(url, { headers: { authorization: 'Bearer é' } }); + expect(first.status).toBe(401); + expect(errorSchema.parse(await first.json()).error.code).toBe('UNAUTHORIZED'); + const second = await fetch(url, { headers: { authorization: 'Bearer é' } }); + expect(second.status).toBe(429); + expect(errorSchema.parse(await second.json()).error.code).toBe('AUTH_RATE_LIMITED'); + const authorized = await fetch(url, { headers: { authorization: 'Bearer x' } }); + expect(authorized.status).toBe(200); + expect(z.array(taskRecordSchema).parse(await authorized.json())).toEqual([]); + } finally { + if (server.listening) { + const closed = once(server, 'close'); + server.close(); + await closed; + } + await coordinator.shutdown(); + } + }); + + it('accepts at-limit workflows, rejects breadth/depth as 400, and preserves 409 and the public page cap', async () => { + const coordinator = new Coordinator({ + state: new MemoryStateStore(), registry: new AgentRegistry(), provider: new DeterministicProvider(), + events: new EventBus(), metrics: new Metrics(), logger: new MemoryLogger(), clock: new SystemClock(), + pollIntervalMs: 5, maxConcurrentTasks: 1, defaultTaskTimeoutMs: 1000, + maxWorkflowDepth: 2, maxWorkflowChildren: 1, + }); + const server = createHttpServer({ coordinator, metricsEnabled: false }); + await coordinator.start(); + const listening = once(server, 'listening'); + server.listen(0, '127.0.0.1'); + try { + await listening; + const address = server.address(); + if (address === null || typeof address === 'string') throw new Error('HTTP fixture did not bind'); + const base = `http://127.0.0.1:${address.port}/v1/tasks`; + const rootRequest = { name: 'Root', description: 'Blocked workflow root', input: 'x', idempotencyKey: 'http-root', requiredCapabilities: ['unavailable'] }; + const headers = { 'content-type': 'application/json' }; + const rootResponse = await fetch(base, { method: 'POST', headers, body: JSON.stringify(rootRequest) }); + expect(rootResponse.status).toBe(202); + const root = taskRecordSchema.parse(await rootResponse.json()); + const childResponse = await fetch(base, { method: 'POST', headers, body: JSON.stringify({ ...rootRequest, idempotencyKey: 'http-child', parentTaskId: root.id }) }); + expect(childResponse.status).toBe(202); + let leaf = taskRecordSchema.parse(await childResponse.json()); + const broad = await fetch(base, { method: 'POST', headers, body: JSON.stringify({ ...rootRequest, idempotencyKey: 'http-sibling', parentTaskId: root.id }) }); + expect(broad.status).toBe(400); + expect(errorSchema.parse(await broad.json()).error.code).toBe('TASK_WORKFLOW_BREADTH_EXCEEDED'); + while (leaf.workflowDepth < 2) { + const next = await fetch(base, { method: 'POST', headers, body: JSON.stringify({ ...rootRequest, idempotencyKey: `depth-${leaf.id}`, parentTaskId: leaf.id }) }); + expect(next.status).toBe(202); + leaf = taskRecordSchema.parse(await next.json()); + } + expect(leaf.workflowDepth).toBe(2); + const before = coordinator.listTasks().map(task => task.id); + const deep = await fetch(base, { method: 'POST', headers, body: JSON.stringify({ ...rootRequest, idempotencyKey: 'too-deep', parentTaskId: leaf.id }) }); + expect(deep.status).toBe(400); + expect(errorSchema.parse(await deep.json()).error.code).toBe('TASK_WORKFLOW_DEPTH_EXCEEDED'); + expect(coordinator.listTasks().map(task => task.id)).toEqual(before); + const conflict = await fetch(base, { method: 'POST', headers, body: JSON.stringify({ ...rootRequest, input: 'changed' }) }); + expect(conflict.status).toBe(409); + expect(errorSchema.parse(await conflict.json()).error.code).toBe('IDEMPOTENCY_CONFLICT'); + const tooLarge = await fetch(`${base}?limit=101`); + expect(tooLarge.status).toBe(400); + for (let index = 0; index < 105; index += 1) { + await coordinator.submitTask({ name: `Public page ${index}`, description: 'Keep public bounds', input: 'x', requiredCapabilities: ['unavailable'] }); + } + const publicPage = await fetch(base); + expect(z.array(taskRecordSchema).parse(await publicPage.json())).toHaveLength(100); + const tail = await fetch(`${base}?offset=100&limit=100`); + expect(z.array(taskRecordSchema).parse(await tail.json()).map(task => task.id)).toEqual(coordinator.listTasks(undefined, { offset: 100, limit: 100 }).map(task => task.id)); + expect(coordinator.getHealth().queueDepth).toBe(before.length + 105); + } finally { + if (server.listening) { + const closed = once(server, 'close'); + server.close(); + await closed; + } + await coordinator.shutdown(); + } + }); +}); diff --git a/tests/integration/http.test.ts b/tests/integration/http.test.ts index 6f21e9d..f37bfd0 100644 --- a/tests/integration/http.test.ts +++ b/tests/integration/http.test.ts @@ -1,3 +1,4 @@ +/// import { createHttpServer } from '../../src/http/server'; import { Coordinator } from '../../src/coordination/coordinator'; import { EventBus } from '../../src/events/event-bus'; @@ -194,39 +195,42 @@ describe('HTTP health surface', () => { const address = server.address(); if (!address || typeof address === 'string') throw new Error('Test server did not bind'); const base = `http://127.0.0.1:${address.port}`; - expect((await fetch(`${base}/health`)).status).toBe(200); - expect(((await (await fetch(`${base}/ready`)).json()) as { ready: boolean }).ready).toBe(false); - expect((await fetch(`${base}/metrics`)).status).toBe(404); - expect((await fetch(`${base}/v1/tasks/00000000-0000-4000-8000-000000000001`)).status).toBe(404); - expect( - ( - await fetch(`${base}/v1/tasks/00000000-0000-4000-8000-000000000001/cancel`, { + const requestStatus = async (path: string, init?: RequestInit): Promise => { + const response = await fetch(`${base}${path}`, { + ...init, + signal: AbortSignal.timeout(5000), + }); + await response.arrayBuffer(); + return response.status; + }; + try { + expect(await requestStatus('/health')).toBe(200); + const ready = await fetch(`${base}/ready`, { signal: AbortSignal.timeout(5000) }); + expect(await ready.json()).toEqual(expect.objectContaining({ ready: false })); + expect(await requestStatus('/metrics')).toBe(404); + expect(await requestStatus('/v1/tasks/00000000-0000-4000-8000-000000000001')).toBe(404); + expect( + await requestStatus('/v1/tasks/00000000-0000-4000-8000-000000000001/cancel', { method: 'POST', }) - ).status - ).toBe(404); - expect((await fetch(`${base}/v1/tasks?status=unknown`)).status).toBe(400); - expect( - ( - await fetch(`${base}/v1/tasks`, { + ).toBe(404); + expect(await requestStatus('/v1/tasks?status=unknown')).toBe(400); + expect( + await requestStatus('/v1/tasks', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ name: 'Not started', description: 'Unavailable', input: null }), }) - ).status - ).toBe(503); - expect( - ( - await fetch(`${base}/v1/tasks`, { + ).toBe(503); + expect( + await requestStatus('/v1/tasks', { method: 'POST', headers: { 'content-type': 'text/plain' }, body: 'not-json', }) - ).status - ).toBe(415); - expect( - ( - await fetch(`${base}/v1/tasks`, { + ).toBe(415); + expect( + await requestStatus('/v1/tasks', { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ @@ -235,13 +239,15 @@ describe('HTTP health surface', () => { input: 'x', }), }) - ).status - ).toBe(413); - expect((await fetch(`${base}/unknown`)).status).toBe(404); - await new Promise((resolve, reject) => - server.close(error => (error ? reject(error) : resolve())) - ); - await coordinator.shutdown(); + ).toBe(413); + expect(await requestStatus('/unknown')).toBe(404); + } finally { + const closed = Promise.withResolvers(); + server.close(error => (error ? closed.reject(error) : closed.resolve())); + server.closeAllConnections(); + await closed.promise; + await coordinator.shutdown(); + } }); it('enforces request timeouts, an in-flight cap, and authentication throttling', async () => { diff --git a/tests/integration/queue-traversal.test.ts b/tests/integration/queue-traversal.test.ts new file mode 100644 index 0000000..b1191aa --- /dev/null +++ b/tests/integration/queue-traversal.test.ts @@ -0,0 +1,131 @@ +/// +import { AbstractAgent } from '../../src/agents/abstract-agent'; +import { AnalysisAgent } from '../../src/agents/builtins'; +import { AgentRegistry } from '../../src/agents/registry'; +import { Coordinator } from '../../src/coordination/coordinator'; +import { delay } from '../../src/domain/clock'; +import type { AgentResult, AgentTask } from '../../src/domain/types'; +import { EventBus } from '../../src/events/event-bus'; +import { MemoryLogger } from '../../src/logging/logger'; +import { Metrics } from '../../src/metrics/metrics'; +import { DeterministicProvider } from '../../src/providers/provider'; +import { MemoryStateStore } from '../../src/state/state-store'; +import { TaskRepository } from '../../src/tasks/task-repository'; + +class HoldingAgent extends AbstractAgent { + public started = 0; + private readonly settlement = Promise.withResolvers(); + public constructor() { super({ id: 'holding', name: 'Holding', capabilities: ['holding'], maxConcurrentTasks: 1 }); } + public release(): void { this.settlement.resolve({ data: { done: true }, summary: 'Released' }); } + protected override async run(_task: AgentTask): Promise { + this.started += 1; + return this.settlement.promise; + } +} + +async function waitFor(condition: () => boolean): Promise { + const deadline = Date.now() + 3000; + while (!condition()) { + if (Date.now() >= deadline) throw new Error('Queue condition did not become true'); + await delay(5); + } +} + +const base = { name: 'Queue fixture', description: 'Exercise complete internal traversal', input: 'A complete paragraph for local analysis.' }; + +describe('Complete internal queue traversal', () => { + it.each(['capability', 'capacity', 'backoff', 'workflow'] as const)( + 'dispatches runnable work beyond 101 higher-priority %s blockers and reports full counts', + async kind => { + const now = Date.now(); + const clock = { now: () => new Date(now) }; + const state = new MemoryStateStore(); + const repository = new TaskRepository(state, clock); + const registry = new AgentRegistry(); + const holding = new HoldingAgent(); + registry.register(holding); + registry.register(new AnalysisAgent()); + await repository.load(); + const parent = kind === 'workflow' ? await repository.create({ ...base, priority: 100, requiredCapabilities: ['missing'] }) : undefined; + for (let index = 0; index < 101; index += 1) { + if (kind === 'backoff') { + const task = await repository.create({ ...base, agentId: 'analysis', priority: 100, maxAttempts: 2, retryBackoffMs: 60_000 }); + await repository.start(task.id, 'analysis'); + await repository.fail(task.id, { code: 'TRANSIENT', message: 'Wait for retry', retryable: true }); + } else { + await repository.create({ + ...base, priority: 100, + ...(kind === 'capability' ? { requiredCapabilities: ['missing'] } : { agentId: kind === 'capacity' ? 'holding' : 'analysis' }), + ...(parent === undefined ? {} : { dependencyIds: [parent.id] }), + }); + } + } + const runnable = await repository.create({ ...base, priority: 1, agentId: 'analysis' }); + const coordinator = new Coordinator({ + state, repository, registry, clock, provider: new DeterministicProvider(), events: new EventBus(), + metrics: new Metrics(), logger: new MemoryLogger(), pollIntervalMs: 5, + maxConcurrentTasks: 2, defaultTaskTimeoutMs: 10_000, + }); + try { + await coordinator.start(); + await waitFor(() => coordinator.getTask(runnable.id)?.status === 'completed'); + const expected = kind === 'workflow' ? 102 : kind === 'capacity' ? 100 : 101; + expect(coordinator.getHealth()).toEqual(expect.objectContaining({ queueDepth: expected, blockedTasks: expected })); + expect(coordinator.listTasks('queued')).toHaveLength(100); + expect(coordinator.getMetrics().renderPrometheus().split('\n')).toContain(`noorg_queue_depth ${expected}`); + if (kind === 'capacity') { + expect(holding.started).toBe(1); + expect(coordinator.getHealth().blockedCapacityTasks).toBe(100); + } + if (kind === 'capability') expect(coordinator.getHealth().blockedCapabilityTasks).toBe(101); + } finally { + holding.release(); + await coordinator.shutdown(); + } + } + ); + + it('processes beyond-page deadlines and failed relationships even while global execution capacity is occupied', async () => { + let now = Date.now(); + const clock = { now: () => new Date(now) }; + const state = new MemoryStateStore(); + const repository = new TaskRepository(state, clock); + const registry = new AgentRegistry(); + const holding = new HoldingAgent(); + registry.register(holding); + registry.register(new AnalysisAgent()); + await repository.load(); + const occupied = await repository.create({ ...base, priority: 100, agentId: 'holding' }); + for (let index = 0; index < 101; index += 1) + await repository.create({ ...base, priority: 99, requiredCapabilities: ['missing'] }); + const parent = await repository.create({ ...base, requiredCapabilities: ['missing'] }); + await repository.cancel(parent.id); + const expired = await repository.create({ ...base, priority: 1, deadlineAt: new Date(now + 1000).toISOString(), requiredCapabilities: ['missing'] }); + const dependency = await repository.create({ ...base, priority: 1, agentId: 'analysis', dependencyIds: [parent.id] }); + const child = await repository.create({ ...base, priority: 1, agentId: 'analysis', parentTaskId: parent.id, parentCompletionPolicy: 'wait_for_success' }); + const runnable = await repository.create({ ...base, priority: 0, agentId: 'analysis' }); + now += 2000; + const coordinator = new Coordinator({ + state, repository, registry, clock, provider: new DeterministicProvider(), events: new EventBus(), + metrics: new Metrics(), logger: new MemoryLogger(), pollIntervalMs: 5, + maxConcurrentTasks: 1, defaultTaskTimeoutMs: 10_000, + }); + try { + await coordinator.start(); + await waitFor(() => coordinator.getTask(occupied.id)?.status === 'running'); + await waitFor(() => [expired.id, dependency.id, child.id].every(id => coordinator.getTask(id)?.status === 'failed')); + expect(registry.activeCount()).toBe(1); + expect(holding.started).toBe(1); + expect(coordinator.getTask(expired.id)?.error?.code).toBe('TASK_DEADLINE_EXCEEDED'); + expect(coordinator.getTask(dependency.id)?.error?.code).toBe('TASK_DEPENDENCY_FAILED'); + expect(coordinator.getTask(child.id)?.error?.code).toBe('TASK_PARENT_FAILED'); + expect(coordinator.getTask(runnable.id)?.status).toBe('queued'); + holding.release(); + await waitFor(() => coordinator.getTask(runnable.id)?.status === 'completed'); + expect(coordinator.getHealth().queueDepth).toBe(101); + } finally { + holding.release(); + await coordinator.shutdown(); + } + }); +}); diff --git a/tests/integration/state-lock.test.ts b/tests/integration/state-lock.test.ts new file mode 100644 index 0000000..176e4d5 --- /dev/null +++ b/tests/integration/state-lock.test.ts @@ -0,0 +1,313 @@ +import { fork, type ChildProcess } from 'node:child_process'; +import { once } from 'node:events'; +import { lstat, mkdir, mkdtemp, readFile, rename, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { z } from 'zod'; +import { runtimeStateSchema } from '../../src/domain/types'; +import { FileStateStore } from '../../src/state/state-store'; +import { StateBoundary } from '../fixtures/state-boundary'; + +const messageSchema = z.object({ type: z.string(), code: z.string().optional(), pid: z.number().optional(), revision: z.number().optional(), value: z.unknown().optional() }); +interface LockChild { readonly process: ChildProcess; readonly exited: Promise; readonly initial: Promise> } + +async function nextMessage(child: ChildProcess): Promise> { + const [message] = await once(child, 'message', { signal: AbortSignal.timeout(5000) }); + return messageSchema.parse(message); +} + +function contender(file: string, mode: 'pause-stale-read' | 'writer'): LockChild { + const child = fork(require.resolve('../fixtures/state-lock-worker'), [file, mode], { + execPath: process.execPath, execArgv: ['--import', require.resolve('tsx')], + env: { PATH: process.env.PATH, NODE_ENV: 'test' }, stdio: ['ignore', 'ignore', 'pipe', 'ipc'], + }); + child.stderr?.resume(); + const exited = once(child, 'exit').then(([code]) => z.number().nullable().parse(code)); + return { process: child, exited, initial: nextMessage(child) }; +} + +async function stopChildren(children: readonly LockChild[]): Promise { + for (const child of children) { + if (child.process.exitCode === null && child.process.signalCode === null) child.process.kill('SIGKILL'); + } + await Promise.all(children.map(child => child.exited)); +} + +describe('Physical state lock ownership', () => { + it.each([0, 1, 2])('rechecks stale O under arbitration after A replaces it while B is paused (%i)', async () => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-lock-race-')); + const file = join(directory, 'state.json'); + const lock = `${file}.lock`; + const children: LockChild[] = []; + const third = new FileStateStore(file); + try { + await mkdir(lock); + await writeFile(join(lock, 'owner'), JSON.stringify({ pid: 99999999, createdAt: Date.now() - 120_000, token: 'stale-O' })); + const b = contender(file, 'pause-stale-read'); + children.push(b); + expect(await b.initial).toEqual({ type: 'observed' }); + const a = contender(file, 'writer'); + children.push(a); + expect(await a.initial).toEqual(expect.objectContaining({ type: 'owned', pid: a.process.pid, revision: 1 })); + const ownerA = await readFile(join(lock, 'owner'), 'utf8'); + const inodeA = await lstat(lock); + const rejected = nextMessage(b.process); + b.process.send({ type: 'resume' }); + expect(await rejected).toEqual({ type: 'rejected', code: 'STATE_LOCKED' }); + expect(await b.exited).toBe(0); + expect(await readFile(join(lock, 'owner'), 'utf8')).toBe(ownerA); + expect(await lstat(lock)).toEqual(expect.objectContaining({ dev: inodeA.dev, ino: inodeA.ino })); + await expect(third.load()).rejects.toMatchObject({ code: 'STATE_LOCKED' }); + const flushed = nextMessage(a.process); + a.process.send({ type: 'flush' }); + expect(await flushed).toEqual({ type: 'flushed', revision: 1, value: 'A' }); + expect(runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state.writer).toBe('A'); + a.process.send({ type: 'close' }); + expect(await a.exited).toBe(0); + } finally { + await stopChildren(children); + await third.close(); + await rm(directory, { recursive: true, force: true }); + } + }, 15_000); + + it('cannot flush or close through a replacement physical owner, even when the replacement advertises the same pid', async () => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-lock-replace-')); + const file = join(directory, 'state.json'); + const state = new FileStateStore(file); + try { + await state.load(); + await state.set('accepted', 1); + const before = await readFile(file, 'utf8'); + await rename(`${file}.lock`, `${file}.lock.detached`); + await mkdir(`${file}.lock`); + const replacement = JSON.stringify({ pid: process.pid, createdAt: Date.now(), token: 'replacement' }); + await writeFile(`${file}.lock/owner`, replacement); + await expect(state.set('accepted', 99)).rejects.toMatchObject({ code: 'STATE_LOCK_LOST' }); + expect(state.get('accepted')).toBe(1); + expect(state.isOpen()).toBe(false); + await expect(state.flush()).rejects.toMatchObject({ code: 'STATE_LOCK_LOST' }); + await expect(state.close()).rejects.toMatchObject({ code: 'STATE_LOCK_LOST' }); + expect(await readFile(file, 'utf8')).toBe(before); + expect(await readFile(`${file}.lock/owner`, 'utf8')).toBe(replacement); + } finally { + await Promise.allSettled([state.close()]); + await rm(directory, { recursive: true, force: true }); + } + }); + + it('removes only its owned temporary file if the main owner is replaced during a paused write', async () => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-lock-write-replace-')); + const file = join(directory, 'state.json'); + const boundary = new StateBoundary(file); + const state = new FileStateStore(file, boundary.io); + let release: (() => void) | undefined; + try { + await state.load(); + await state.set('accepted', 1); + const before = await readFile(file, 'utf8'); + const barrier = boundary.pauseNextWrite(); + release = barrier.release; + const pending = state.set('rejected', 99); + const rejection = expect(pending).rejects.toMatchObject({ code: 'STATE_LOCK_LOST' }); + await barrier.entered; + await rename(`${file}.lock`, `${file}.lock.detached`); + await mkdir(`${file}.lock`); + const replacement = JSON.stringify({ pid: process.pid, createdAt: Date.now(), token: 'foreign-during-write' }); + await writeFile(`${file}.lock/owner`, replacement); + barrier.release(); + await rejection; + await expect(state.close()).rejects.toMatchObject({ code: 'STATE_LOCK_LOST' }); + expect(await readFile(file, 'utf8')).toBe(before); + expect(await readFile(`${file}.lock/owner`, 'utf8')).toBe(replacement); + } finally { + release?.(); + await Promise.allSettled([state.close()]); + await rm(directory, { recursive: true, force: true }); + } + }); + + it.each(['missing', 'invalid', 'numeric-junk', 'guard'] as const)('fails closed for an ambiguous %s owner without deleting evidence', async kind => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-lock-ambiguous-')); + const file = join(directory, 'state.json'); + const state = new FileStateStore(file); + const evidencePath = kind === 'guard' ? `${file}.lock.guard` : `${file}.lock`; + try { + await mkdir(evidencePath); + if (kind === 'invalid') await writeFile(join(evidencePath, 'owner'), '{not-json'); + if (kind === 'numeric-junk') await writeFile(join(evidencePath, 'owner'), '99999999#'); + const identity = await lstat(evidencePath); + await expect(state.load()).rejects.toMatchObject({ code: 'STATE_LOCKED' }); + expect(await lstat(evidencePath)).toEqual(expect.objectContaining({ dev: identity.dev, ino: identity.ino })); + expect(state.isOpen()).toBe(false); + } finally { + await state.close(); + await rm(directory, { recursive: true, force: true }); + } + }); +}); + +describe('Durable state commit failure boundaries', () => { + it.each(['write', 'sync'] as const)('keeps rejected %s candidates out of get/snapshot/revision and later flush/close/reload', async fault => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-commit-failure-')); + const file = join(directory, 'state.json'); + const boundary = new StateBoundary(file); + const state = new FileStateStore(file, boundary.io); + try { + await state.load(); + await state.set('accepted', { value: 1 }); + const revision = state.revision(); + boundary.failNext(fault); + await expect(state.set('accepted', { value: 99 })).rejects.toBe(boundary.error); + expect(state.get('accepted')).toEqual({ value: 1 }); + expect(state.snapshot()).toEqual({ accepted: { value: 1 } }); + expect(state.revision()).toBe(revision); + const updaterError = new Error('Rejected updater'); + await expect(state.update<{ value: number }>('accepted', current => { + if (current === undefined) throw new Error('Missing fixture'); + current.value = 77; + throw updaterError; + })).rejects.toBe(updaterError); + await state.flush(); + await state.close(); + const reopened = new FileStateStore(file); + try { + await reopened.load(); + expect(reopened.snapshot()).toEqual({ accepted: { value: 1 } }); + expect(reopened.revision()).toBe(revision); + } finally { await reopened.close(); } + } finally { + await state.close(); + await rm(directory, { recursive: true, force: true }); + } + }); + + it.each(['rename-before', 'rename-after', 'directory-sync'] as const)('quarantines uncertain %s outcomes without publishing or automatically unlocking them', async fault => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-commit-uncertain-')); + const file = join(directory, 'state.json'); + const boundary = new StateBoundary(file); + const state = new FileStateStore(file, boundary.io); + const follower = new FileStateStore(file); + try { + await state.load(); + await state.set('accepted', 1); + const revision = state.revision(); + boundary.failNext(fault); + await expect(state.set('accepted', 99)).rejects.toMatchObject({ code: 'STATE_COMMIT_UNCERTAIN', cause: boundary.error }); + expect(state.snapshot()).toEqual({ accepted: 1 }); + expect(state.revision()).toBe(revision); + expect(state.isOpen()).toBe(false); + expect((await lstat(`${file}.lock.guard`)).isDirectory()).toBe(true); + await expect(state.flush()).rejects.toMatchObject({ code: 'STATE_COMMIT_UNCERTAIN' }); + await expect(state.close()).rejects.toMatchObject({ code: 'STATE_COMMIT_UNCERTAIN' }); + await expect(follower.load()).rejects.toMatchObject({ code: 'STATE_LOCKED' }); + const disk = runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))); + expect(disk.state.accepted).toBe(fault === 'rename-before' ? 1 : 99); + // A post-effect file is evidence of uncertainty, not an accepted repository revision. + expect(state.revision()).toBe(revision); + } finally { + await Promise.allSettled([state.close(), follower.close()]); + await rm(directory, { recursive: true, force: true }); + } + }); + + it('does not expose a pending candidate and drains it before a concurrent close', async () => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-commit-close-')); + const file = join(directory, 'state.json'); + const boundary = new StateBoundary(file); + const state = new FileStateStore(file, boundary.io); + let release: (() => void) | undefined; + try { + await state.load(); + await state.set('accepted', 1); + const barrier = boundary.pauseNextWrite(); + release = barrier.release; + const pending = state.set('accepted', 2); + await barrier.entered; + const closing = state.close(); + expect(state.snapshot()).toEqual({ accepted: 1 }); + expect(state.revision()).toBe(1); + await expect(state.set('late', true)).rejects.toMatchObject({ code: 'STATE_STORE_CLOSED' }); + barrier.release(); + await pending; + await closing; + expect(runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state).toEqual({ accepted: 2 }); + } finally { + release?.(); + await state.close(); + await rm(directory, { recursive: true, force: true }); + } + }); +}); + +describe('Incomplete namespace acquisition', () => { + it('fences an unopenable guard without reporting readiness or deleting ownership evidence', async () => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-guard-open-')); + const file = join(directory, 'state.json'); + const boundary = new StateBoundary(file); + const state = new FileStateStore(file, boundary.io); + const follower = new FileStateStore(file); + try { + await state.load(); + await state.set('accepted', 1); + const revision = state.revision(); + boundary.failNext('guard-open'); + await expect(state.set('rejected', 99)).rejects.toMatchObject({ code: 'STATE_LOCK_LOST', cause: boundary.error }); + expect(state.snapshot()).toEqual({ accepted: 1 }); + expect(state.revision()).toBe(revision); + expect(state.isOpen()).toBe(false); + expect((await lstat(`${file}.lock.guard`)).isDirectory()).toBe(true); + await expect(state.close()).rejects.toMatchObject({ code: 'STATE_LOCK_LOST' }); + await expect(follower.load()).rejects.toMatchObject({ code: 'STATE_LOCKED' }); + expect(runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state).toEqual({ accepted: 1 }); + } finally { + await Promise.allSettled([state.close(), follower.close()]); + await rm(directory, { recursive: true, force: true }); + } + }); +}); + +describe('Retained owner identities', () => { + it.each(['token', 'guard'] as const)('does not publish or clean through a changed %s identity during a real write', async kind => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-retained-identity-')); + const file = join(directory, 'state.json'); + const boundary = new StateBoundary(file); + const state = new FileStateStore(file, boundary.io); + let release: (() => void) | undefined; + try { + await state.load(); + await state.set('accepted', 1); + const before = await readFile(file, 'utf8'); + const barrier = boundary.pauseNextWrite(); + release = barrier.release; + const pending = state.set('rejected', 99); + const rejection = kind === 'token' + ? expect(pending).rejects.toMatchObject({ code: 'STATE_LOCK_LOST' }) + : expect(pending).rejects.toBeInstanceOf(AggregateError); + await barrier.entered; + let evidencePath: string; + let evidence: string; + if (kind === 'token') { + evidencePath = `${file}.lock/owner`; + evidence = JSON.stringify({ pid: process.pid, createdAt: Date.now(), token: 'new-token-same-directory' }); + } else { + await rename(`${file}.lock.guard`, `${file}.lock.guard.detached`); + await mkdir(`${file}.lock.guard`); + evidencePath = `${file}.lock.guard/foreign.txt`; + evidence = 'Foreign guard evidence'; + } + await writeFile(evidencePath, evidence); + barrier.release(); + await rejection; + expect(state.snapshot()).toEqual({ accepted: 1 }); + expect(state.isOpen()).toBe(false); + await expect(state.close()).rejects.toMatchObject({ code: 'STATE_LOCK_LOST' }); + expect(await readFile(file, 'utf8')).toBe(before); + expect(await readFile(evidencePath, 'utf8')).toBe(evidence); + } finally { + release?.(); + await Promise.allSettled([state.close()]); + await rm(directory, { recursive: true, force: true }); + } + }); +}); diff --git a/tests/integration/task-commit.test.ts b/tests/integration/task-commit.test.ts new file mode 100644 index 0000000..a888a51 --- /dev/null +++ b/tests/integration/task-commit.test.ts @@ -0,0 +1,208 @@ +import { randomUUID } from 'node:crypto'; +import { mkdtemp, readFile, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { SystemClock } from '../../src/domain/clock'; +import { NoOrgError } from '../../src/domain/errors'; +import { jsonValueSchema, runtimeStateSchema } from '../../src/domain/types'; +import { FileStateStore, MemoryStateStore } from '../../src/state/state-store'; +import { TaskRepository } from '../../src/tasks/task-repository'; +import { StateBoundary } from '../fixtures/state-boundary'; + +const request = { name: 'Durable task', description: 'Publish only committed work', input: 'x', agentId: 'analysis', maxAttempts: 2 }; +const two = { promptTokens: 1, completionTokens: 1, totalTokens: 2, costUsd: 0.02 }; +const five = { promptTokens: 3, completionTokens: 2, totalTokens: 5, costUsd: 0.05 }; + +describe('TaskRepository durable publication', () => { + it('does not publish a rejected create or reserve its idempotency key for a concurrent caller', async () => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-create-commit-')); + const file = join(directory, 'state.json'); + const boundary = new StateBoundary(file); + const state = new FileStateStore(file, boundary.io); + const repository = new TaskRepository(state, new SystemClock()); + const barrier = boundary.pauseNextWrite(); + try { + await state.load(); + await repository.load(); + const first = repository.create({ ...request, idempotencyKey: 'same-key' }); + const rejection = expect(first).rejects.toBe(boundary.error); + await barrier.entered; + expect(repository.list()).toEqual([]); + expect(state.get('tasks')).toBeUndefined(); + expect(state.revision()).toBe(0); + let duplicateSettled = false; + const duplicate = repository.create({ ...request, idempotencyKey: 'same-key' }).then(task => { + duplicateSettled = true; + return task; + }); + await Promise.resolve(); + expect(duplicateSettled).toBe(false); + boundary.failNext('write'); + barrier.release(); + await rejection; + const accepted = await duplicate; + expect(repository.list().map(task => task.id)).toEqual([accepted.id]); + expect((await repository.create({ ...request, idempotencyKey: 'same-key' })).id).toBe(accepted.id); + await repository.persist(); + await state.close(); + const reopened = new FileStateStore(file); + try { + await reopened.load(); + const recovered = new TaskRepository(reopened, new SystemClock()); + await recovered.load(); + expect(recovered.list().map(task => task.id)).toEqual([accepted.id]); + expect(recovered.get(accepted.id)?.attempt).toBe(0); + } finally { await reopened.close(); } + } finally { + barrier.release(); + await state.close(); + await rm(directory, { recursive: true, force: true }); + } + }); + + it('keeps an unresolved/rejected start invisible and serializes cancellation after it', async () => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-start-commit-')); + const file = join(directory, 'state.json'); + const boundary = new StateBoundary(file); + const state = new FileStateStore(file, boundary.io); + let release: (() => void) | undefined; + try { + await state.load(); + const repository = new TaskRepository(state, new SystemClock()); + await repository.load(); + const task = await repository.create(request); + const barrier = boundary.pauseNextWrite(); + release = barrier.release; + const start = repository.start(task.id, 'analysis'); + const rejection = expect(start).rejects.toBe(boundary.error); + await barrier.entered; + expect(repository.get(task.id)).toEqual(task); + expect(repository.list('queued').map(item => item.id)).toEqual([task.id]); + const cancellation = repository.cancel(task.id); + boundary.failNext('sync'); + barrier.release(); + await rejection; + expect(await cancellation).toEqual(expect.objectContaining({ status: 'cancelled', attempt: 0 })); + await repository.persist(); + await state.close(); + const envelope = runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))); + expect(envelope.state.tasks).toEqual([expect.objectContaining({ status: 'cancelled', attempt: 0 })]); + } finally { + release?.(); + await state.close(); + await rm(directory, { recursive: true, force: true }); + } + }); + + it('does not leak rejected cancellation, result or usage into a later accepted commit', async () => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-transition-commit-')); + const file = join(directory, 'state.json'); + const boundary = new StateBoundary(file); + const state = new FileStateStore(file, boundary.io); + try { + await state.load(); + const repository = new TaskRepository(state, new SystemClock()); + await repository.load(); + const task = await repository.create(request); + await repository.start(task.id, 'analysis'); + const running = repository.get(task.id); + boundary.failNext('write'); + await expect(repository.cancel(task.id)).rejects.toBe(boundary.error); + expect(repository.get(task.id)).toEqual(running); + boundary.failNext('sync'); + await expect(repository.completeWithUsage(task.id, { data: { rejected: true }, summary: 'Not committed' }, five)).rejects.toBe(boundary.error); + expect(repository.get(task.id)).toEqual(running); + boundary.failNext('write'); + await expect(repository.recordUsage(task.id, five)).rejects.toBe(boundary.error); + expect(repository.get(task.id)?.usage).toBeUndefined(); + const completed = await repository.completeWithUsage(task.id, { data: { accepted: true }, summary: 'Committed' }, two); + expect(completed.usage).toEqual(two); + expect(completed.result?.data).toEqual({ accepted: true }); + await repository.persist(); + await state.close(); + const reopened = new FileStateStore(file); + try { + await reopened.load(); + const recovered = new TaskRepository(reopened, new SystemClock()); + await recovered.load(); + expect(recovered.get(task.id)).toEqual(completed); + } finally { await reopened.close(); } + } finally { + await state.close(); + await rm(directory, { recursive: true, force: true }); + } + }); + + it('does not publish rejected interrupted-task recovery and can recover the original durable task later', async () => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-recovery-commit-')); + const file = join(directory, 'state.json'); + const first = new FileStateStore(file); + let second: FileStateStore | undefined; + let third: FileStateStore | undefined; + try { + await first.load(); + const original = new TaskRepository(first, new SystemClock()); + await original.load(); + const task = await original.create(request); + await original.start(task.id, 'analysis'); + const running = original.get(task.id); + await first.close(); + const boundary = new StateBoundary(file); + second = new FileStateStore(file, boundary.io); + await second.load(); + const recovering = new TaskRepository(second, new SystemClock()); + await recovering.load(); + boundary.failNext('write'); + await expect(recovering.recoverInterruptedTasks()).rejects.toBe(boundary.error); + expect(recovering.get(task.id)).toEqual(running); + await recovering.persist(); + await second.close(); + third = new FileStateStore(file); + await third.load(); + const reloaded = new TaskRepository(third, new SystemClock()); + await reloaded.load(); + expect(reloaded.get(task.id)).toEqual(running); + expect(await reloaded.recoverInterruptedTasks()).toEqual([ + expect.objectContaining({ id: task.id, status: 'queued', attempt: 1 }), + ]); + expect(reloaded.get(task.id)).toEqual(expect.objectContaining({ status: 'queued', attempt: 1 })); + } finally { + await third?.close(); + await second?.close(); + await first.close(); + await rm(directory, { recursive: true, force: true }); + } + }); + it('does not publish a partially admitted persisted collection when relationship validation fails', async () => { + const state = new MemoryStateStore(); + try { + const original = new TaskRepository(state, new SystemClock()); + await original.load(); + const accepted = await original.create(request); + await state.set('tasks', jsonValueSchema.parse([ + accepted, + { ...accepted, id: randomUUID(), dependencyIds: [randomUUID()] }, + ])); + const loading = new TaskRepository(state, new SystemClock()); + await expect(loading.load()).rejects.toBeInstanceOf(NoOrgError); + expect(loading.list()).toEqual([]); + expect(loading.get(accepted.id)).toBeUndefined(); + } finally { await state.close(); } + }); +}); + +describe('MemoryStateStore admission and close', () => { + it('captures caller-owned values before queueing and drains only pre-close admissions', async () => { + const state = new MemoryStateStore(); + const input = { value: 1 }; + const accepted = state.set('input', input); + input.value = 99; + const close = state.close(); + expect(state.isOpen()).toBe(false); + await expect(state.set('late', true)).rejects.toMatchObject({ code: 'STATE_STORE_CLOSED' }); + await accepted; + await close; + expect(state.snapshot()).toEqual({ input: { value: 1 } }); + expect(state.revision()).toBe(1); + }); +}); diff --git a/tests/integration/usage-accounting.test.ts b/tests/integration/usage-accounting.test.ts new file mode 100644 index 0000000..8024e8a --- /dev/null +++ b/tests/integration/usage-accounting.test.ts @@ -0,0 +1,197 @@ +/// +import { mkdtemp, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { z } from 'zod'; +import { AbstractAgent } from '../../src/agents/abstract-agent'; +import { AgentRegistry } from '../../src/agents/registry'; +import { Coordinator } from '../../src/coordination/coordinator'; +import { delay, SystemClock } from '../../src/domain/clock'; +import { NoOrgError } from '../../src/domain/errors'; +import type { AgentResult, AgentTask, UsageMetadata } from '../../src/domain/types'; +import { EventBus } from '../../src/events/event-bus'; +import { MemoryLogger } from '../../src/logging/logger'; +import { Metrics } from '../../src/metrics/metrics'; +import { GovernedProvider } from '../../src/providers/governed-provider'; +import { OpenAIProvider } from '../../src/providers/provider'; +import { FileStateStore, MemoryStateStore, type StateStore } from '../../src/state/state-store'; +import { TaskRepository } from '../../src/tasks/task-repository'; +import { StateBoundary } from '../fixtures/state-boundary'; + +const output = z.object({ ok: z.boolean() }); +const five: UsageMetadata = { promptTokens: 3, completionTokens: 2, totalTokens: 5, costUsd: 5 }; +const two: UsageMetadata = { promptTokens: 1, completionTokens: 1, totalTokens: 2, costUsd: 2 }; +const seven: UsageMetadata = { promptTokens: 4, completionTokens: 3, totalTokens: 7, costUsd: 7 }; + +function offlineMeter(receipts: readonly UsageMetadata[]) { + const effects = { calls: 0 }; + const provider = new GovernedProvider(new OpenAIProvider({ + apiKey: 'unused-offline-test-key', model: 'offline-test', timeoutMs: 1000, maxRetries: 0, retryBaseMs: 1, + promptCostPerMillionUsd: 1_000_000, completionCostPerMillionUsd: 1_000_000, + transport: { complete: async () => { + const receipt = receipts[effects.calls]; + effects.calls += 1; + if (receipt === undefined) throw new Error('Unexpected offline transport invocation'); + return { id: `offline-${effects.calls}`, model: 'offline-test', parsed: { ok: true }, usage: { + prompt_tokens: receipt.promptTokens, completion_tokens: receipt.completionTokens, total_tokens: receipt.totalTokens, + } }; + } }, + })); + return { provider, effects }; +} + +class ChargedAgent extends AbstractAgent { + public attempts = 0; + public readonly ready = Promise.withResolvers(); + private readonly settlement = Promise.withResolvers(); + public constructor(private readonly mode: 'retry' | 'late' | 'complete', private readonly beforeComplete?: () => void) { + super({ id: 'charged', name: 'Charged', capabilities: ['charged'], maxConcurrentTasks: 1 }); + } + public release(): void { this.settlement.resolve(); } + protected override async run(task: AgentTask, signal: AbortSignal): Promise { + this.attempts += 1; + await this.ask({ operation: 'charge', input: null }, output, signal, task.id); + if (this.mode === 'retry' && this.attempts === 1) throw new NoOrgError('TRANSIENT', 'Retry after a charged attempt', true); + if (this.mode === 'late') { + this.ready.resolve(); + await this.settlement.promise; + // Deliberately uncooperative work: real offline receipt after cancellation/timeout. + await this.context.provider.complete({ operation: 'late-charge', input: null, taskId: task.id }, output); + } + this.beforeComplete?.(); + return { data: { ok: true, attempt: this.attempts }, summary: 'Charged work settled' }; + } +} + +function usageRuntime(state: StateStore, provider: GovernedProvider, agent: ChargedAgent): { coordinator: Coordinator; registry: AgentRegistry } { + const registry = new AgentRegistry(); + registry.register(agent); + const coordinator = new Coordinator({ state, registry, provider, events: new EventBus(), metrics: new Metrics(), + logger: new MemoryLogger(), clock: new SystemClock(), pollIntervalMs: 5, maxConcurrentTasks: 1, + defaultTaskTimeoutMs: 2000 }); + return { coordinator, registry }; +} + +async function waitFor(condition: () => boolean): Promise { + const deadline = Date.now() + 3000; + while (!condition()) { + if (Date.now() >= deadline) throw new Error('Usage condition did not become true'); + await delay(5); + } +} + +const request = { agentId: 'charged', name: 'Account usage', description: 'Charge each provider receipt once', input: null, maxAttempts: 2, retryBackoffMs: 0 }; + +describe('Provider-instance usage deltas', () => { + it('adds 5 then 2 as 7, not cumulative 5 plus cumulative 7, across retries', async () => { + const meter = offlineMeter([five, two]); + const agent = new ChargedAgent('retry'); + const { coordinator } = usageRuntime(new MemoryStateStore(), meter.provider, agent); + try { + await coordinator.start(); + const task = await coordinator.submitTask(request); + await waitFor(() => coordinator.getTask(task.id)?.status === 'completed'); + expect(coordinator.getTask(task.id)?.usage).toEqual(seven); + expect(meter.provider.getTaskUsage(task.id)).toEqual(seven); + expect(meter.effects.calls).toBe(2); + expect(agent.attempts).toBe(2); + } finally { agent.release(); await coordinator.shutdown(); } + }); + + it('advances the checkpoint only after a successful durable commit, including completion-failure re-entry', async () => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-usage-commit-')); + const file = join(directory, 'state.json'); + const boundary = new StateBoundary(file, new NoOrgError('TRANSIENT_STATE', 'Controlled pre-commit failure', true)); + const state = new FileStateStore(file, boundary.io); + const meter = offlineMeter([five, two]); + let completions = 0; + const agent = new ChargedAgent('complete', () => { completions += 1; if (completions === 1) boundary.failNext('write'); }); + const { coordinator } = usageRuntime(state, meter.provider, agent); + try { + await coordinator.start(); + const task = await coordinator.submitTask(request); + await waitFor(() => coordinator.getTask(task.id)?.status === 'completed'); + expect(coordinator.getTask(task.id)?.usage).toEqual(seven); + expect(coordinator.getTask(task.id)?.result?.data).toEqual({ ok: true, attempt: 2 }); + expect(meter.effects.calls).toBe(2); + await coordinator.shutdown(); + const reopened = new FileStateStore(file); + try { + await reopened.load(); + const repository = new TaskRepository(reopened, new SystemClock()); + await repository.load(); + expect(repository.get(task.id)?.usage).toEqual(seven); + } finally { await reopened.close(); } + } finally { + agent.release(); + await coordinator.shutdown(); + await rm(directory, { recursive: true, force: true }); + } + }); + + it('preserves historical usage across a file-backed restart with a fresh provider counter', async () => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-usage-restart-')); + const file = join(directory, 'state.json'); + const previous = new FileStateStore(file); + const meter = offlineMeter([two]); + const agent = new ChargedAgent('complete'); + const { coordinator } = usageRuntime(new FileStateStore(file), meter.provider, agent); + try { + await previous.load(); + const repository = new TaskRepository(previous, new SystemClock()); + await repository.load(); + const task = await repository.create(request); + await repository.recordUsage(task.id, five); + await previous.close(); + await coordinator.start(); + await waitFor(() => coordinator.getTask(task.id)?.status === 'completed'); + expect(coordinator.getTask(task.id)?.usage).toEqual(seven); + expect(meter.provider.getTaskUsage(task.id)).toEqual(two); + expect(meter.effects.calls).toBe(1); + } finally { + agent.release(); + await coordinator.shutdown(); + await previous.close(); + await rm(directory, { recursive: true, force: true }); + } + }); + + it('does not replay already-accounted counters when the same provider instance is handed to a new coordinator', async () => { + const state = new MemoryStateStore(); + const repository = new TaskRepository(state, new SystemClock()); + await repository.load(); + const task = await repository.create(request); + const meter = offlineMeter([five, two]); + await meter.provider.complete({ operation: 'prior-session', input: null, taskId: task.id }, output); + await repository.recordUsage(task.id, five); + const agent = new ChargedAgent('complete'); + const { coordinator } = usageRuntime(state, meter.provider, agent); + try { + await coordinator.start(); + await waitFor(() => coordinator.getTask(task.id)?.status === 'completed'); + expect(coordinator.getTask(task.id)?.usage).toEqual(seven); + expect(meter.effects.calls).toBe(2); + } finally { agent.release(); await coordinator.shutdown(); } + }); + + it.each(['cancel', 'timeout'] as const)('records late settled usage once after %s without publishing a result or releasing the agent early', async kind => { + const meter = offlineMeter([five, two]); + const agent = new ChargedAgent('late'); + const { coordinator, registry } = usageRuntime(new MemoryStateStore(), meter.provider, agent); + try { + await coordinator.start(); + const task = await coordinator.submitTask({ ...request, maxAttempts: 1, timeoutMs: kind === 'timeout' ? 100 : 2000 }); + await agent.ready.promise; + if (kind === 'cancel') await coordinator.cancelTask(task.id); + else await waitFor(() => coordinator.getTask(task.id)?.status === 'failed'); + if (kind === 'timeout') expect(coordinator.getTask(task.id)?.error?.code).toBe('TASK_TIMEOUT'); + expect(registry.activeCount()).toBe(1); + agent.release(); + await waitFor(() => coordinator.getTask(task.id)?.usage?.totalTokens === 7 && registry.activeCount() === 0); + expect(coordinator.getTask(task.id)?.usage).toEqual(seven); + expect(coordinator.getTask(task.id)?.status).toBe(kind === 'cancel' ? 'cancelled' : 'failed'); + expect(coordinator.getTask(task.id)?.result).toBeUndefined(); + expect(meter.effects.calls).toBe(2); + } finally { agent.release(); await coordinator.shutdown(); } + }); +}); diff --git a/tests/unit/config-launch.test.ts b/tests/unit/config-launch.test.ts new file mode 100644 index 0000000..e7a5bed --- /dev/null +++ b/tests/unit/config-launch.test.ts @@ -0,0 +1,66 @@ +import { execFile } from 'node:child_process'; +import { mkdtemp, readFile, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { parseEnv, promisify } from 'node:util'; +import { z } from 'zod'; +import { loadConfig } from '../../src/config/config'; + +const runFile = promisify(execFile); + +describe('Explicit configuration admission', () => { + it('accepts the actual example without inventing provider credentials or a URL', async () => { + const example = parseEnv(await readFile(resolve(__dirname, '../../.env.example'), 'utf8')); + const config = loadConfig(example); + expect(config.provider).toBe('local'); + expect(config.openai.apiKey).toBeUndefined(); + expect(config.openai.baseUrl).toBeUndefined(); + expect(config.server.authToken).toBeUndefined(); + expect(config.providerGovernance.budgetUsd).toBeUndefined(); + }); + + it.each(['', ' ', '\t\n'])('treats optional blank values as absent (%j)', blank => { + expect(loadConfig({ + NOORG_LLM_PROVIDER: 'local', + OPENAI_API_KEY: blank, + OPENAI_BASE_URL: blank, + NOORG_HOST: blank, + NOORG_HTTP_AUTH_TOKEN: blank, + NOORG_PROVIDER_BUDGET_USD: blank, + })).toEqual(loadConfig({ NOORG_LLM_PROVIDER: 'local' })); + }); + + it('preserves nonempty validation and explicit provider/production-auth requirements', () => { + expect(() => loadConfig({ OPENAI_API_KEY: 'unused-test-key' })).toThrow(); + expect(() => loadConfig({ NOORG_LLM_PROVIDER: 'openai', OPENAI_API_KEY: ' ' })).toThrow(); + expect(() => loadConfig({ NOORG_LLM_PROVIDER: 'local', OPENAI_BASE_URL: 'not a URL' })).toThrow(); + expect(() => loadConfig({ NOORG_LLM_PROVIDER: 'local', NOORG_PROVIDER_BUDGET_USD: 'not a number' })).toThrow(); + expect(() => loadConfig({ NODE_ENV: 'production', NOORG_LLM_PROVIDER: 'local', NOORG_HTTP_AUTH_TOKEN: ' ' })).toThrow(); + const secured = loadConfig({ NODE_ENV: 'production', NOORG_LLM_PROVIDER: 'local', NOORG_HTTP_AUTH_TOKEN: 'test-secret' }); + expect(secured.server.authToken).toBe('test-secret'); + expect(secured.server.host).toBe('0.0.0.0'); + }); + + it('loads only an explicitly supplied Node env-file and keeps inherited environment precedence', async () => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-env-file-')); + const envFile = join(directory, '.env'); + try { + await writeFile(envFile, 'NOORG_LLM_PROVIDER=local\nOPENAI_MODEL=file-model\nOPENAI_API_KEY=\nOPENAI_BASE_URL=\nNOORG_HTTP_AUTH_TOKEN=\nNOORG_PROVIDER_BUDGET_USD=\n'); + const configModule = require.resolve('../../src/config/config'); + const script = `const { loadConfig } = require(${JSON.stringify(configModule)}); const config = loadConfig(process.env); process.stdout.write(JSON.stringify({ provider: config.provider, model: config.openai.model }));`; + const result = await runFile(process.execPath, ['--env-file=' + envFile, '--import', require.resolve('tsx'), '-e', script], { + cwd: directory, + env: { PATH: process.env.PATH, NODE_ENV: 'test', OPENAI_MODEL: 'inherited-model' }, + timeout: 5000, + }); + expect(z.object({ provider: z.literal('local'), model: z.string() }).parse(JSON.parse(result.stdout))).toEqual({ + provider: 'local', model: 'inherited-model', + }); + await expect(runFile(process.execPath, ['--import', require.resolve('tsx'), '-e', script], { + cwd: directory, + env: { PATH: process.env.PATH, NODE_ENV: 'test' }, + timeout: 5000, + })).rejects.toMatchObject({ code: 1 }); + } finally { await rm(directory, { recursive: true, force: true }); } + }); +}); diff --git a/tests/unit/provider-retention.test.ts b/tests/unit/provider-retention.test.ts new file mode 100644 index 0000000..7581a6c --- /dev/null +++ b/tests/unit/provider-retention.test.ts @@ -0,0 +1,52 @@ +import { z } from 'zod'; +import { GovernedProvider } from '../../src/providers/governed-provider'; +import { DeterministicProvider } from '../../src/providers/provider'; + +const schema = z.object({ value: z.number() }); +const request = { operation: 'local-retention', input: null, localDerivation: () => ({ value: 1 }) }; +const invalidRequest = { ...request, localDerivation: () => ({ value: Number.NaN }) }; + +describe('GovernedProvider rate-history ownership', () => { + it('retains no rate history when explicitly disabled, across successful and failed local requests', async () => { + const provider = new GovernedProvider(new DeterministicProvider(), { + maxRequestsPerMinute: 0, + circuitFailureThreshold: 1000, + }); + try { + for (let index = 0; index < 128; index += 1) { + expect((await provider.complete(request, schema)).data).toEqual({ value: 1 }); + await expect(provider.complete(invalidRequest, schema)).rejects.toThrow(); + } + // This is the retained-memory contract, not a timing/heap-size heuristic. + const history = z.array(z.number()).parse(Reflect.get(provider, 'requestTimes')); + expect(history).toEqual([]); + expect(provider.getUsage()).toEqual({ promptTokens: 0, completionTokens: 0, totalTokens: 0, costUsd: 0 }); + } finally { await provider.close(); } + }); + + it('counts enabled failed attempts, rejects excess requests, and expires the exact rolling boundary', async () => { + let now = 0; + const provider = new GovernedProvider(new DeterministicProvider(), { + maxRequestsPerMinute: 2, + circuitFailureThreshold: 1000, + clock: { now: () => new Date(now) }, + }); + try { + await provider.complete(request, schema); + await expect(provider.complete(invalidRequest, schema)).rejects.toThrow(); + await expect(provider.complete(request, schema)).rejects.toMatchObject({ code: 'PROVIDER_RATE_LIMITED' }); + now = 59_999; + await expect(provider.complete(request, schema)).rejects.toMatchObject({ code: 'PROVIDER_RATE_LIMITED' }); + now = 60_000; + expect((await provider.complete(request, schema)).data).toEqual({ value: 1 }); + for (let window = 0; window < 32; window += 1) { + now += 60_000; + await provider.complete(request, schema); + await provider.complete(request, schema); + await expect(provider.complete(request, schema)).rejects.toMatchObject({ code: 'PROVIDER_RATE_LIMITED' }); + const history = z.array(z.number()).parse(Reflect.get(provider, 'requestTimes')); + expect(history).toEqual([now, now]); + } + } finally { await provider.close(); } + }); +}); diff --git a/tests/unit/security.test.ts b/tests/unit/security.test.ts index a47e20f..ce1cfe4 100644 --- a/tests/unit/security.test.ts +++ b/tests/unit/security.test.ts @@ -62,6 +62,10 @@ describe('runtime safety helpers', () => { expect(isBearerAuthorized('Bearer secret', 'different')).toBe(false); expect(isBearerAuthorized('Bearer secret extra', 'secret')).toBe(false); expect(isBearerAuthorized('bearer secret', 'secret')).toBe(false); + expect(isBearerAuthorized('Bearer é', 'x')).toBe(false); + expect(isBearerAuthorized('Bearer 🙂', 'ab')).toBe(false); + expect(isBearerAuthorized('Bearer é', 'é')).toBe(true); + expect(isBearerAuthorized('Bearer ê', 'é')).toBe(false); }); it('limits repeated authentication failures and resets after the window', () => { diff --git a/tests/unit/state.test.ts b/tests/unit/state.test.ts index 18e8a0a..87a6891 100644 --- a/tests/unit/state.test.ts +++ b/tests/unit/state.test.ts @@ -1,4 +1,4 @@ -import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'; +import { mkdir, mkdtemp, rm, utimes, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { MemoryStateStore } from '../../src/state/state-store'; @@ -109,7 +109,9 @@ describe('MemoryStateStore', () => { const filePath = join(directory, 'state.json'); const lockPath = `${filePath}.lock`; await mkdir(lockPath); - await writeFile(join(lockPath, 'owner'), '99999999#'); + await writeFile(join(lockPath, 'owner'), '99999999'); + const old = new Date(Date.now() - 120_000); + await utimes(lockPath, old, old); const state = new FileStateStore(filePath); await state.load(); expect(state.isOpen()).toBe(true); From de91624ceea99588c3fc1a5b19ec1ee19d20e993 Mon Sep 17 00:00:00 2001 From: Daniel Ari Friedman <6911384+docxology@users.noreply.github.com> Date: Mon, 16 Nov 2026 12:55:03 -0800 Subject: [PATCH 2/2] Fix strict IPC typing and preserve cleanup error causes - Preserve both state operation and cleanup errors while attaching the caught cleanup cause. - Guard optional Node IPC disconnect methods and retain numeric owner compatibility. - Apply the reported formatting repairs and accurate native-filesystem wording without changing assertions or policies. --- src/config/config.ts | 2 +- src/coordination/coordinator.ts | 25 +- src/state/state-store.ts | 180 +++++++-- src/tasks/task-repository.ts | 32 +- tests/README.md | 2 +- tests/fixtures/state-boundary.ts | 52 ++- tests/fixtures/state-lock-worker.ts | 10 +- tests/integration/coordinator-commit.test.ts | 288 +++++++++----- tests/integration/http-boundaries.test.ts | 120 +++++- tests/integration/queue-traversal.test.ts | 104 ++++- tests/integration/state-lock.test.ts | 389 +++++++++++-------- tests/integration/task-commit.test.ts | 61 ++- tests/integration/usage-accounting.test.ts | 172 +++++--- tests/unit/config-launch.test.ts | 84 ++-- tests/unit/provider-retention.test.ts | 33 +- 15 files changed, 1071 insertions(+), 483 deletions(-) diff --git a/src/config/config.ts b/src/config/config.ts index ab816b7..8680ed0 100644 --- a/src/config/config.ts +++ b/src/config/config.ts @@ -19,7 +19,7 @@ function integer(name: string, fallback: number, minimum: number, maximum?: numb function blankOptional(schema: T) { return z.preprocess( - value => typeof value === 'string' && value.trim() === '' ? undefined : value, + value => (typeof value === 'string' && value.trim() === '' ? undefined : value), schema.optional() ); } diff --git a/src/coordination/coordinator.ts b/src/coordination/coordinator.ts index c205d6c..62d8869 100644 --- a/src/coordination/coordinator.ts +++ b/src/coordination/coordinator.ts @@ -579,14 +579,23 @@ export class Coordinator implements TaskScheduler { const observed = this.dependencies.provider.getTaskUsage?.(taskId); const current = observed === undefined ? undefined : usageMetadataSchema.parse(observed); const checkpoint = this.usageCheckpoints.get(taskId) ?? { - promptTokens: 0, completionTokens: 0, totalTokens: 0, costUsd: 0, + promptTokens: 0, + completionTokens: 0, + totalTokens: 0, + costUsd: 0, }; let delta: UsageMetadata | undefined; if (current !== undefined) { - if (current.promptTokens < checkpoint.promptTokens || + if ( + current.promptTokens < checkpoint.promptTokens || current.completionTokens < checkpoint.completionTokens || - current.totalTokens < checkpoint.totalTokens || current.costUsd < checkpoint.costUsd) - throw new NoOrgError('PROVIDER_USAGE_REGRESSED', 'Provider-instance usage counters decreased'); + current.totalTokens < checkpoint.totalTokens || + current.costUsd < checkpoint.costUsd + ) + throw new NoOrgError( + 'PROVIDER_USAGE_REGRESSED', + 'Provider-instance usage counters decreased' + ); const added = { promptTokens: current.promptTokens - checkpoint.promptTokens, completionTokens: current.completionTokens - checkpoint.completionTokens, @@ -599,7 +608,10 @@ export class Coordinator implements TaskScheduler { if (current !== undefined) this.usageCheckpoints.set(taskId, current); return result; }); - const settled = operation.then(() => undefined, () => undefined); + const settled = operation.then( + () => undefined, + () => undefined + ); this.usageMutations.set(taskId, settled); void settled.then(() => { if (this.usageMutations.get(taskId) === settled) this.usageMutations.delete(taskId); @@ -669,7 +681,8 @@ export class Coordinator implements TaskScheduler { this.dependencyState(task) !== 'ready' || this.parentState(task) !== 'ready' || (task.nextAttemptAt !== undefined && Date.parse(task.nextAttemptAt) > now) - ) total += 1; + ) + total += 1; } return { total, capability, capacity }; } diff --git a/src/state/state-store.ts b/src/state/state-store.ts index f6c57fa..ad7c645 100644 --- a/src/state/state-store.ts +++ b/src/state/state-store.ts @@ -97,7 +97,10 @@ interface ObservedLock { readonly owner: LockOwner | undefined; } -function sameIdentity(left: { dev: number; ino: number }, right: { dev: number; ino: number }): boolean { +function sameIdentity( + left: { dev: number; ino: number }, + right: { dev: number; ino: number } +): boolean { return left.dev === right.dev && left.ino === right.ino; } @@ -105,18 +108,26 @@ function parseOwner(raw: string): LockOwner | undefined { try { const parsed: unknown = JSON.parse(raw); if ( - typeof parsed === 'object' && parsed !== null && - 'pid' in parsed && Number.isSafeInteger(parsed.pid) && Number(parsed.pid) > 0 && - 'createdAt' in parsed && typeof parsed.createdAt === 'number' && - Number.isFinite(parsed.createdAt) && parsed.createdAt >= 0 && - 'token' in parsed && typeof parsed.token === 'string' && parsed.token.length > 0 - ) return parsed as LockOwner; + typeof parsed === 'object' && + parsed !== null && + 'pid' in parsed && + Number.isSafeInteger(parsed.pid) && + Number(parsed.pid) > 0 && + 'createdAt' in parsed && + typeof parsed.createdAt === 'number' && + Number.isFinite(parsed.createdAt) && + parsed.createdAt >= 0 && + 'token' in parsed && + typeof parsed.token === 'string' && + parsed.token.length > 0 + ) + return parsed as LockOwner; } catch { // Only the exact historical decimal-PID representation is admitted below. } - const legacy = raw.trim(); - if (/^[1-9][0-9]*$/.test(legacy) && Number.isSafeInteger(Number(legacy))) { - return { pid: Number(legacy), createdAt: 0, token: 'legacy-pid' }; + const decimalOwner = raw.trim(); + if (/^[1-9][0-9]*$/.test(decimalOwner) && Number.isSafeInteger(Number(decimalOwner))) { + return { pid: Number(decimalOwner), createdAt: 0, token: 'decimal-pid' }; } return undefined; } @@ -135,7 +146,10 @@ export class FileStateStore implements StateStore { private currentRevision = 0; private readonly filePath: string; - public constructor(filePath: string, private readonly io: FileStateStoreIO = nodeStateStoreIO) { + public constructor( + filePath: string, + private readonly io: FileStateStoreIO = nodeStateStoreIO + ) { this.filePath = resolve(filePath); } @@ -195,7 +209,9 @@ export class FileStateStore implements StateStore { this.dirty = true; } await this.assertOwnership(guard); - this.values = new Map(Object.entries(parsed.state).map(([key, value]) => [key, clone(value)])); + this.values = new Map( + Object.entries(parsed.state).map(([key, value]) => [key, clone(value)]) + ); this.currentRevision = parsed.revision; this.loaded = true; } catch (error) { @@ -203,10 +219,17 @@ export class FileStateStore implements StateStore { await this.releaseLock(guard); } catch (cleanupError) { this.fence('STATE_LOCK_LOST', 'State lock ownership became ambiguous', cleanupError); - throw new AggregateError([error, cleanupError], 'State acquisition and owned cleanup failed'); + throw new AggregateError( + [error, cleanupError], + 'State acquisition and owned cleanup failed', + { cause: cleanupError } + ); } if (error instanceof NoOrgError) throw error; - const failure = new NoOrgError('STATE_LOAD_FAILED', `Unable to load state from ${this.filePath}`); + const failure = new NoOrgError( + 'STATE_LOAD_FAILED', + `Unable to load state from ${this.filePath}` + ); failure.cause = error; throw failure; } @@ -250,7 +273,9 @@ export class FileStateStore implements StateStore { return this.schedule(async () => { this.assertAvailable(); const current = this.values.get(key); - const next = clone(jsonValueSchema.parse(updater(current === undefined ? undefined : clone(current) as T))) as T; + const next = clone( + jsonValueSchema.parse(updater(current === undefined ? undefined : (clone(current) as T))) + ) as T; const candidate = new Map(this.values); candidate.set(key, next); const revision = this.currentRevision + 1; @@ -287,7 +312,10 @@ export class FileStateStore implements StateStore { try { await this.lease.handle.close(); } catch (error) { - failure = failure === undefined ? error : new AggregateError([failure, error], 'State close and handle cleanup failed'); + failure = + failure === undefined + ? error + : new AggregateError([failure, error], 'State close and handle cleanup failed'); } } if (failure !== undefined) throw failure; @@ -298,7 +326,8 @@ export class FileStateStore implements StateStore { private assertAvailable(requireLoaded = true): void { if (this.failure !== undefined) throw this.failure; if (this.closed) throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closed'); - if (requireLoaded && !this.loaded) throw new NoOrgError('STATE_NOT_LOADED', 'State store must be loaded before mutation'); + if (requireLoaded && !this.loaded) + throw new NoOrgError('STATE_NOT_LOADED', 'State store must be loaded before mutation'); } private assertAdmission(): void { @@ -308,7 +337,10 @@ export class FileStateStore implements StateStore { private schedule(operation: () => Promise): Promise { const next = this.mutation.then(operation); - this.mutation = next.then(() => undefined, () => undefined); + this.mutation = next.then( + () => undefined, + () => undefined + ); return next; } @@ -321,10 +353,18 @@ export class FileStateStore implements StateStore { this.dirty = false; } - private async writeCandidate(values: ReadonlyMap, revision: number, guard: DirectoryLease): Promise { + private async writeCandidate( + values: ReadonlyMap, + revision: number, + guard: DirectoryLease + ): Promise { await this.assertOwnership(guard); const temporaryPath = `${this.filePath}.${process.pid}.${randomUUID()}.tmp`; - const body = JSON.stringify({ schemaVersion: 2, revision, state: Object.fromEntries(values) }, null, 2); + const body = JSON.stringify( + { schemaVersion: 2, revision, state: Object.fromEntries(values) }, + null, + 2 + ); const handle = await this.io.open(temporaryPath, 'wx', 0o600); let identity: Stats | undefined; let failure: unknown; @@ -338,7 +378,10 @@ export class FileStateStore implements StateStore { try { await handle.close(); } catch (error) { - failure = failure === undefined ? error : new AggregateError([failure, error], 'State write and close failed'); + failure = + failure === undefined + ? error + : new AggregateError([failure, error], 'State write and close failed'); } if (failure === undefined) { try { @@ -356,16 +399,25 @@ export class FileStateStore implements StateStore { return; } catch (error) { this.retainGuard = true; - throw this.fence('STATE_COMMIT_UNCERTAIN', 'State replacement durability is uncertain; store is fenced', error); + throw this.fence( + 'STATE_COMMIT_UNCERTAIN', + 'State replacement durability is uncertain; store is fenced', + error + ); } } try { await this.assertDirectory(guard); const current = await this.io.lstat(temporaryPath); - if (identity !== undefined && sameIdentity(current, identity)) await this.io.rm(temporaryPath); + if (identity !== undefined && sameIdentity(current, identity)) + await this.io.rm(temporaryPath); } catch (cleanupError) { if (!isMissingFile(cleanupError)) { - throw new AggregateError([failure, cleanupError], 'State write and owned temporary cleanup failed'); + throw new AggregateError( + [failure, cleanupError], + 'State write and owned temporary cleanup failed', + { cause: cleanupError } + ); } } throw failure; @@ -379,7 +431,10 @@ export class FileStateStore implements StateStore { } private async openDirectory(path: string): Promise { - const handle = await this.io.open(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW); + const handle = await this.io.open( + path, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW + ); try { const identity = await handle.stat(); return { path, handle, dev: identity.dev, ino: identity.ino }; @@ -415,8 +470,16 @@ export class FileStateStore implements StateStore { try { await this.io.mkdir(path, { mode: 0o700 }); } catch (error) { - if (typeof error === 'object' && error !== null && 'code' in error && error.code === 'EEXIST') { - throw new NoOrgError('STATE_LOCKED', `State file ${this.filePath} is already in use or requires guard recovery`); + if ( + typeof error === 'object' && + error !== null && + 'code' in error && + error.code === 'EEXIST' + ) { + throw new NoOrgError( + 'STATE_LOCKED', + `State file ${this.filePath} is already in use or requires guard recovery` + ); } throw error; } @@ -440,22 +503,45 @@ export class FileStateStore implements StateStore { await this.assertDirectory(guard); } catch (error) { owned = false; - failure = failure === undefined ? error : new AggregateError([failure, error], 'State operation and namespace identity check failed'); + failure = + failure === undefined + ? error + : new AggregateError( + [failure, error], + 'State operation and namespace identity check failed' + ); } try { await guard.handle.close(); } catch (error) { owned = false; this.retainGuard = true; - const cleanup = this.fence('STATE_LOCK_LOST', 'State namespace guard handle could not be closed', error); - failure = failure === undefined ? cleanup : new AggregateError([failure, cleanup], 'State operation and handle cleanup failed'); + const cleanup = this.fence( + 'STATE_LOCK_LOST', + 'State namespace guard handle could not be closed', + error + ); + failure = + failure === undefined + ? cleanup + : new AggregateError([failure, cleanup], 'State operation and handle cleanup failed'); } if (owned && !this.retainGuard) { try { await this.io.rmdir(path); } catch (error) { - const cleanup = this.fence('STATE_LOCK_LOST', 'State namespace guard could not be released safely', error); - failure = failure === undefined ? cleanup : new AggregateError([failure, cleanup], 'State operation and namespace release failed'); + const cleanup = this.fence( + 'STATE_LOCK_LOST', + 'State namespace guard could not be released safely', + error + ); + failure = + failure === undefined + ? cleanup + : new AggregateError( + [failure, cleanup], + 'State operation and namespace release failed' + ); } } if (failure !== undefined) throw failure; @@ -471,7 +557,8 @@ export class FileStateStore implements StateStore { if (isMissingFile(error)) return undefined; throw error; } - if (!identity.isDirectory()) throw new NoOrgError('STATE_LOCKED', 'State lock path is not an owned directory'); + if (!identity.isDirectory()) + throw new NoOrgError('STATE_LOCKED', 'State lock path is not an owned directory'); let owner: LockOwner | undefined; try { owner = parseOwner(await this.io.readFile(`${path}/owner`, 'utf8')); @@ -483,7 +570,8 @@ export class FileStateStore implements StateStore { private assertReclaimable(lock: ObservedLock): void { const owner = lock.owner; - if (owner === undefined) throw new NoOrgError('STATE_LOCKED', 'State lock owner is incomplete or ambiguous'); + if (owner === undefined) + throw new NoOrgError('STATE_LOCKED', 'State lock owner is incomplete or ambiguous'); const age = Date.now() - (owner.createdAt === 0 ? lock.identity.mtimeMs : owner.createdAt); if (age < staleLockMs || isLiveProcess(owner.pid)) { throw new NoOrgError('STATE_LOCKED', `State file ${this.filePath} is already in use`); @@ -493,10 +581,15 @@ export class FileStateStore implements StateStore { private async assertOwnership(guard: DirectoryLease): Promise { await this.assertDirectory(guard); const lease = this.lease; - if (lease === undefined) throw this.fence('STATE_LOCK_LOST', 'State store has no acquired lock', undefined); + if (lease === undefined) + throw this.fence('STATE_LOCK_LOST', 'State store has no acquired lock', undefined); const current = await this.observeLock(); - if (current === undefined || !sameIdentity(current.identity, lease) || - current.owner?.token !== lease.owner.token || current.owner.pid !== lease.owner.pid) { + if ( + current === undefined || + !sameIdentity(current.identity, lease) || + current.owner?.token !== lease.owner.token || + current.owner.pid !== lease.owner.pid + ) { throw this.fence('STATE_LOCK_LOST', 'State lock no longer belongs to this store', undefined); } } @@ -524,7 +617,8 @@ export class MemoryStateStore implements StateStore { } public async load(): Promise { - if (this.closed || this.closing) throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closed'); + if (this.closed || this.closing) + throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closed'); } public get(key: string): T | undefined { @@ -541,7 +635,8 @@ export class MemoryStateStore implements StateStore { } public async set(key: string, value: JsonValue): Promise { - if (this.closed || this.closing) throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closed'); + if (this.closed || this.closing) + throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closed'); const admitted = clone(jsonValueSchema.parse(value)); await this.enqueue(() => { this.values.set(key, admitted); @@ -571,13 +666,16 @@ export class MemoryStateStore implements StateStore { public async close(): Promise { if (this.closePromise !== undefined) return this.closePromise; this.closing = true; - this.closePromise = this.mutation.then(() => { this.closed = true; }); + this.closePromise = this.mutation.then(() => { + this.closed = true; + }); this.mutation = this.closePromise; return this.closePromise; } private async enqueue(operation: () => void): Promise { - if (this.closed || this.closing) throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closed'); + if (this.closed || this.closing) + throw new NoOrgError('STATE_STORE_CLOSED', 'State store is closed'); const next = this.mutation.then(operation); this.mutation = next.catch(() => undefined); await next; diff --git a/src/tasks/task-repository.ts b/src/tasks/task-repository.ts index 4f55e1b..3bebbca 100644 --- a/src/tasks/task-repository.ts +++ b/src/tasks/task-repository.ts @@ -72,13 +72,19 @@ export class TaskRepository { if (persisted === undefined) return; const parsed = taskListSchema.safeParse(persisted); if (!parsed.success) - throw new NoOrgError('TASK_STATE_INVALID', 'Persisted task records do not match the task schema'); + throw new NoOrgError( + 'TASK_STATE_INVALID', + 'Persisted task records do not match the task schema' + ); const candidate = new Map(parsed.data.map(task => [task.id, copy(task) as TaskRecord])); for (const task of candidate.values()) { if (task.parentTaskId !== undefined && !candidate.has(task.parentTaskId)) throw new NoOrgError('TASK_PARENT_NOT_FOUND', 'Persisted task parent was not found'); if (task.dependencyIds.some(dependencyId => !candidate.has(dependencyId))) - throw new NoOrgError('TASK_DEPENDENCY_NOT_FOUND', 'Persisted task dependency was not found'); + throw new NoOrgError( + 'TASK_DEPENDENCY_NOT_FOUND', + 'Persisted task dependency was not found' + ); } if (this.hasDependencyCycle(candidate) || this.hasParentCycle(candidate)) throw new NoOrgError('TASK_DEPENDENCY_CYCLE', 'Persisted tasks contain a dependency cycle'); @@ -226,7 +232,10 @@ export class TaskRepository { return sorted.slice(parsedQuery.offset, end).map(copy); } - public internalSnapshot(status?: TaskStatus): { readonly tasks: readonly TaskRecord[]; readonly total: number } { + public internalSnapshot(status?: TaskStatus): { + readonly tasks: readonly TaskRecord[]; + readonly total: number; + } { const tasks = this.select(status === undefined ? {} : { status }).map(copy); return { tasks, total: tasks.length }; } @@ -239,7 +248,8 @@ export class TaskRepository { (query.agentId === undefined || task.agentId === query.agentId) && (query.capability === undefined || task.requiredCapabilities.includes(query.capability)) && (query.idempotencyKey === undefined || task.idempotencyKey === query.idempotencyKey) - ) selected.push(task); + ) + selected.push(task); } return selected.sort((left, right) => { const priorityDifference = this.effectivePriority(right) - this.effectivePriority(left); @@ -412,10 +422,15 @@ export class TaskRepository { return copy(changed); } - private async enqueue(operation: (candidate: Map) => void | boolean): Promise { + private async enqueue( + operation: (candidate: Map) => void | boolean + ): Promise { await this.serialize(async () => { if (!this.state.isOpen()) - throw new NoOrgError('STATE_STORE_UNAVAILABLE', 'Task mutations require an open state store'); + throw new NoOrgError( + 'STATE_STORE_UNAVAILABLE', + 'Task mutations require an open state store' + ); const candidate = new Map(this.tasks); if (operation(candidate) === false) return; await this.state.set('tasks', [...candidate.values()] as unknown as JsonValue); @@ -425,7 +440,10 @@ export class TaskRepository { private serialize(operation: () => Promise): Promise { const next = this.mutation.then(operation); - this.mutation = next.then(() => undefined, () => undefined); + this.mutation = next.then( + () => undefined, + () => undefined + ); return next; } diff --git a/tests/README.md b/tests/README.md index d11ab9c..3fc0ea7 100644 --- a/tests/README.md +++ b/tests/README.md @@ -29,7 +29,7 @@ Coverage thresholds are 90% lines and 80% branches. `tests/live` is excluded fro `task-commit`, `state-lock` and `coordinator-commit` exercise real file-state admission, pre-rename failure visibility, post-rename fencing, physical/token/ guard ownership and agent reservations. The stale-owner race uses independent -Node processes and IPC barriers, not mocked filesystem outcomes. Both timer +Node processes and IPC barriers, using native filesystem outcomes. Both timer resolve/reject cases hold real completion/accounting writes past the execution deadline, then require exact usage/events and reservation/file-state lease release. diff --git a/tests/fixtures/state-boundary.ts b/tests/fixtures/state-boundary.ts index c411462..87024bd 100644 --- a/tests/fixtures/state-boundary.ts +++ b/tests/fixtures/state-boundary.ts @@ -3,7 +3,8 @@ import type { FileHandle } from 'node:fs/promises'; import { dirname } from 'node:path'; import { nodeStateStoreIO, type FileStateStoreIO } from '../../src/state/state-store'; -export type StateFault = 'write' | 'sync' | 'rename-before' | 'rename-after' | 'directory-sync' | 'guard-open'; +export type StateFault = + 'write' | 'sync' | 'rename-before' | 'rename-after' | 'directory-sync' | 'guard-open'; export interface CommitBarrier { readonly entered: Promise; release(): void; @@ -16,7 +17,10 @@ export class StateBoundary { private renamed = false; private barrier: { enter(): void; readonly released: Promise } | undefined; - public constructor(private readonly filePath: string, public readonly error = new Error('controlled state I/O failure')) { + public constructor( + private readonly filePath: string, + public readonly error = new Error('controlled state I/O failure') + ) { this.io = { ...nodeStateStoreIO, open: async (path, flags, mode) => { @@ -25,32 +29,38 @@ export class StateBoundary { throw this.error; } const handle = await nodeStateStoreIO.open(path, flags, mode); - const temporary = String(path).startsWith(`${this.filePath}.`) && String(path).endsWith('.tmp'); + const temporary = + String(path).startsWith(`${this.filePath}.`) && String(path).endsWith('.tmp'); const parent = String(path) === dirname(this.filePath); return new Proxy(handle, { get: (target, property) => { - if (property === 'writeFile') return async (...args: Parameters) => { - if (temporary) { - const barrier = this.barrier; - this.barrier = undefined; - if (barrier !== undefined) { - barrier.enter(); - await barrier.released; + if (property === 'writeFile') + return async (...args: Parameters) => { + if (temporary) { + const barrier = this.barrier; + this.barrier = undefined; + if (barrier !== undefined) { + barrier.enter(); + await barrier.released; + } + if (this.fault === 'write') { + this.fault = undefined; + throw this.error; + } } - if (this.fault === 'write') { + return target.writeFile(...args); + }; + if (property === 'sync') + return async () => { + if ( + (temporary && this.fault === 'sync') || + (parent && this.renamed && this.fault === 'directory-sync') + ) { this.fault = undefined; throw this.error; } - } - return target.writeFile(...args); - }; - if (property === 'sync') return async () => { - if ((temporary && this.fault === 'sync') || (parent && this.renamed && this.fault === 'directory-sync')) { - this.fault = undefined; - throw this.error; - } - await target.sync(); - }; + await target.sync(); + }; const value: unknown = Reflect.get(target, property, target); return typeof value === 'function' ? value.bind(target) : value; }, diff --git a/tests/fixtures/state-lock-worker.ts b/tests/fixtures/state-lock-worker.ts index 93eb191..885f5a9 100644 --- a/tests/fixtures/state-lock-worker.ts +++ b/tests/fixtures/state-lock-worker.ts @@ -1,7 +1,11 @@ import { on, once } from 'node:events'; import { z } from 'zod'; import { NoOrgError } from '../../src/domain/errors'; -import { FileStateStore, nodeStateStoreIO, type FileStateStoreIO } from '../../src/state/state-store'; +import { + FileStateStore, + nodeStateStoreIO, + type FileStateStoreIO, +} from '../../src/state/state-store'; async function main(): Promise { const file = z.string().min(1).parse(process.argv[2]); @@ -45,12 +49,12 @@ async function main(): Promise { } } finally { await store.close(); - if (process.connected) process.disconnect(); + if (process.connected && process.disconnect !== undefined) process.disconnect(); } } void main().catch(error => { process.stderr.write(String(error) + '\n'); process.exitCode = 1; - if (process.connected) process.disconnect(); + if (process.connected && process.disconnect !== undefined) process.disconnect(); }); diff --git a/tests/integration/coordinator-commit.test.ts b/tests/integration/coordinator-commit.test.ts index f9cbf9d..c57a32a 100644 --- a/tests/integration/coordinator-commit.test.ts +++ b/tests/integration/coordinator-commit.test.ts @@ -7,19 +7,30 @@ import { AgentRegistry } from '../../src/agents/registry'; import { Coordinator } from '../../src/coordination/coordinator'; import { delay, SystemClock } from '../../src/domain/clock'; import { NoOrgError } from '../../src/domain/errors'; -import { runtimeStateSchema, type AgentResult, type AgentTask, type TaskRecord } from '../../src/domain/types'; +import { + runtimeStateSchema, + type AgentResult, + type AgentTask, + type TaskRecord, +} from '../../src/domain/types'; import { EventBus } from '../../src/events/event-bus'; import { MemoryLogger } from '../../src/logging/logger'; import { Metrics } from '../../src/metrics/metrics'; import { GovernedProvider } from '../../src/providers/governed-provider'; -import { DeterministicProvider, OpenAIProvider, type LLMProvider } from '../../src/providers/provider'; +import { + DeterministicProvider, + OpenAIProvider, + type LLMProvider, +} from '../../src/providers/provider'; import { FileStateStore } from '../../src/state/state-store'; import { TaskRepository } from '../../src/tasks/task-repository'; import { StateBoundary } from '../fixtures/state-boundary'; class EffectAgent extends AbstractAgent { public effects = 0; - public constructor() { super({ id: 'effect', name: 'Effect', capabilities: ['effect'], maxConcurrentTasks: 1 }); } + public constructor() { + super({ id: 'effect', name: 'Effect', capabilities: ['effect'], maxConcurrentTasks: 1 }); + } protected override async run(_task: AgentTask): Promise { this.effects += 1; return { data: { executed: true }, summary: 'Executed' }; @@ -48,7 +59,11 @@ class CommitAgent extends AbstractAgent { ); this.beforeSettlement(); if (this.outcome === 'reject') - throw new NoOrgError('EXECUTION_REJECTED', 'Agent execution rejected before its deadline', false); + throw new NoOrgError( + 'EXECUTION_REJECTED', + 'Agent execution rejected before its deadline', + false + ); return { data, summary: 'Executed' }; } } @@ -71,20 +86,39 @@ function commitRuntime( ): { coordinator: Coordinator; registry: AgentRegistry } { const registry = new AgentRegistry(); registry.register(agent); - return { registry, coordinator: new Coordinator({ state, repository, registry, - provider, events, metrics: new Metrics(), logger: new MemoryLogger(), - clock: new SystemClock(), pollIntervalMs: 5, maxConcurrentTasks: 1, defaultTaskTimeoutMs: 1000 }) }; + return { + registry, + coordinator: new Coordinator({ + state, + repository, + registry, + provider, + events, + metrics: new Metrics(), + logger: new MemoryLogger(), + clock: new SystemClock(), + pollIntervalMs: 5, + maxConcurrentTasks: 1, + defaultTaskTimeoutMs: 1000, + }), + }; } async function waitFor(condition: () => boolean): Promise { const deadline = Date.now() + 3000; while (!condition()) { - if (Date.now() >= deadline) throw new Error('Durable task execution did not reach the expected boundary'); + if (Date.now() >= deadline) + throw new Error('Durable task execution did not reach the expected boundary'); await delay(5); } } -const request = { agentId: 'effect', name: 'Durable effect', description: 'Do not execute an uncommitted start', input: null }; +const request = { + agentId: 'effect', + name: 'Durable effect', + description: 'Do not execute an uncommitted start', + input: null, +}; describe('Coordinator durable task/lease admission', () => { it('keeps a pending start invisible and releases the real reservation when it fails before a queued cancellation/shutdown', async () => { @@ -111,13 +145,15 @@ describe('Coordinator durable task/lease admission', () => { const shutdown = coordinator.shutdown(); boundary.failNext('write'); barrier.release(); - expect(await cancellation).toEqual(expect.objectContaining({ status: 'cancelled', attempt: 0 })); + expect(await cancellation).toEqual( + expect.objectContaining({ status: 'cancelled', attempt: 0 }) + ); await shutdown; expect(registry.activeCount()).toBe(0); expect(agent.effects).toBe(0); - expect(runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state.tasks).toEqual([ - expect.objectContaining({ id: task.id, status: 'cancelled', attempt: 0 }), - ]); + expect( + runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state.tasks + ).toEqual([expect.objectContaining({ id: task.id, status: 'cancelled', attempt: 0 })]); } finally { release?.(); await coordinator.shutdown(); @@ -171,9 +207,15 @@ describe('Coordinator durable task/lease admission', () => { expect(coordinator.getStatus()).toBe('stopping'); barrier.release(); await shutdown; - expect(coordinator.getTask(task.id)).toEqual(expect.objectContaining({ status: 'cancelled', attempt: 1 })); - expect(coordinator.getTask(queued.id)).toEqual(expect.objectContaining({ status: 'queued', attempt: 0 })); - expect(runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state.tasks).toEqual([ + expect(coordinator.getTask(task.id)).toEqual( + expect.objectContaining({ status: 'cancelled', attempt: 1 }) + ); + expect(coordinator.getTask(queued.id)).toEqual( + expect.objectContaining({ status: 'queued', attempt: 0 }) + ); + expect( + runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state.tasks + ).toEqual([ expect.objectContaining({ id: task.id, status: 'cancelled', attempt: 1 }), expect.objectContaining({ id: queued.id, status: 'queued', attempt: 0 }), ]); @@ -186,100 +228,132 @@ describe('Coordinator durable task/lease admission', () => { } }); - it.each(['resolve', 'reject'] as const)('clears the execution timer before a %s outcome waits for a durable commit', async outcome => { - const directory = await mkdtemp(join(tmpdir(), 'noorg-execution-deadline-')); - const file = join(directory, 'state.json'); - const boundary = new StateBoundary(file); - const state = new FileStateStore(file, boundary.io); - const repository = new TaskRepository(state, new SystemClock()); - const usage = { promptTokens: 3, completionTokens: 2, totalTokens: 5, costUsd: 5 }; - let transportCalls = 0; - const provider = new GovernedProvider(new OpenAIProvider({ - apiKey: 'unused-offline-test-key', model: 'offline-test', timeoutMs: 1000, maxRetries: 0, retryBaseMs: 1, - promptCostPerMillionUsd: 1_000_000, completionCostPerMillionUsd: 1_000_000, - transport: { complete: async () => { - transportCalls += 1; - return { id: `offline-${transportCalls}`, model: 'offline-test', parsed: { executed: true }, usage: { - prompt_tokens: usage.promptTokens, completion_tokens: usage.completionTokens, total_tokens: usage.totalTokens, - } }; - } }, - })); - const events = new EventBus(); - let commitEntered = false; - let release: (() => void) | undefined; - const agent = new CommitAgent(outcome, () => { - const barrier = boundary.pauseNextWrite(); - release = barrier.release; - void barrier.entered.then(() => { commitEntered = true; }); - }); - const { coordinator, registry } = commitRuntime(state, repository, agent, provider, events); - try { - await state.load(); - await repository.load(); - const task = await repository.create({ ...request, timeoutMs: 100, maxAttempts: 2, retryBackoffMs: 0 }); - await coordinator.start(); - await waitFor(() => commitEntered); - expect(agent.effects).toBe(1); - expect(registry.activeCount()).toBe(1); - expect(provider.getTaskUsage(task.id)).toEqual(usage); - - // The real next write is completion on resolve, or usage accounting on rejection. - // Keep it blocked past the execution deadline without faking either the clock or commit. - await delay(task.timeoutMs + 25); - expect(agent.signal?.aborted).toBe(false); - expect(registry.activeCount()).toBe(1); - expect(coordinator.getTask(task.id)).toEqual(expect.objectContaining({ status: 'running', attempt: 1 })); - expect(coordinator.getTask(task.id)?.usage).toBeUndefined(); - expect(coordinator.getHealth().lastFailure).toBeUndefined(); - expect(events.getHistory()).toEqual([]); - expect(runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state.tasks).toEqual([ - expect.objectContaining({ id: task.id, status: 'running', attempt: 1 }), - ]); + it.each(['resolve', 'reject'] as const)( + 'clears the execution timer before a %s outcome waits for a durable commit', + async outcome => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-execution-deadline-')); + const file = join(directory, 'state.json'); + const boundary = new StateBoundary(file); + const state = new FileStateStore(file, boundary.io); + const repository = new TaskRepository(state, new SystemClock()); + const usage = { promptTokens: 3, completionTokens: 2, totalTokens: 5, costUsd: 5 }; + let transportCalls = 0; + const provider = new GovernedProvider( + new OpenAIProvider({ + apiKey: 'unused-offline-test-key', + model: 'offline-test', + timeoutMs: 1000, + maxRetries: 0, + retryBaseMs: 1, + promptCostPerMillionUsd: 1_000_000, + completionCostPerMillionUsd: 1_000_000, + transport: { + complete: async () => { + transportCalls += 1; + return { + id: `offline-${transportCalls}`, + model: 'offline-test', + parsed: { executed: true }, + usage: { + prompt_tokens: usage.promptTokens, + completion_tokens: usage.completionTokens, + total_tokens: usage.totalTokens, + }, + }; + }, + }, + }) + ); + const events = new EventBus(); + let commitEntered = false; + let release: (() => void) | undefined; + const agent = new CommitAgent(outcome, () => { + const barrier = boundary.pauseNextWrite(); + release = barrier.release; + void barrier.entered.then(() => { + commitEntered = true; + }); + }); + const { coordinator, registry } = commitRuntime(state, repository, agent, provider, events); + try { + await state.load(); + await repository.load(); + const task = await repository.create({ + ...request, + timeoutMs: 100, + maxAttempts: 2, + retryBackoffMs: 0, + }); + await coordinator.start(); + await waitFor(() => commitEntered); + expect(agent.effects).toBe(1); + expect(registry.activeCount()).toBe(1); + expect(provider.getTaskUsage(task.id)).toEqual(usage); - release?.(); - await waitFor(() => registry.activeCount() === 0); - const settled = coordinator.getTask(task.id); - expect(settled).toEqual(expect.objectContaining({ - status: outcome === 'resolve' ? 'completed' : 'failed', attempt: 1, usage, - })); - expect(agent.effects).toBe(1); - expect(transportCalls).toBe(1); - expect(provider.getTaskUsage(task.id)).toEqual(usage); - expect(agent.signal?.aborted).toBe(false); - expect(events.getHistory().map(event => event.type)).toEqual([ - outcome === 'resolve' ? 'task.completed' : 'task.failed', - ]); - const metrics = coordinator.getMetrics().renderPrometheus(); - expect(metrics).not.toContain('noorg_tasks_retried '); - expect(metrics).not.toContain('noorg_tasks_cancelled '); - if (outcome === 'resolve') { - expect(settled?.result?.data).toEqual({ executed: true }); - expect(settled?.error).toBeUndefined(); + // The real next write is completion on resolve, or usage accounting on rejection. + // Keep it blocked past the execution deadline without faking either the clock or commit. + await delay(task.timeoutMs + 25); + expect(agent.signal?.aborted).toBe(false); + expect(registry.activeCount()).toBe(1); + expect(coordinator.getTask(task.id)).toEqual( + expect.objectContaining({ status: 'running', attempt: 1 }) + ); + expect(coordinator.getTask(task.id)?.usage).toBeUndefined(); expect(coordinator.getHealth().lastFailure).toBeUndefined(); - expect(metrics).toContain('noorg_tasks_completed 1\n'); - expect(metrics).not.toContain('noorg_tasks_failed '); - } else { - expect(settled?.result).toBeUndefined(); - expect(settled?.error?.code).toBe('EXECUTION_REJECTED'); - expect(coordinator.getHealth().lastFailure?.code).toBe('EXECUTION_REJECTED'); - expect(metrics).toContain('noorg_tasks_failed 1\n'); - expect(metrics).not.toContain('noorg_tasks_completed '); - } - await coordinator.shutdown(); - expect(registry.activeCount()).toBe(0); - const reopened = new FileStateStore(file); - try { - await reopened.load(); - const persisted = new TaskRepository(reopened, new SystemClock()); - await persisted.load(); - expect(persisted.internalSnapshot().tasks).toEqual([settled]); + expect(events.getHistory()).toEqual([]); + expect( + runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state.tasks + ).toEqual([expect.objectContaining({ id: task.id, status: 'running', attempt: 1 })]); + + release?.(); + await waitFor(() => registry.activeCount() === 0); + const settled = coordinator.getTask(task.id); + expect(settled).toEqual( + expect.objectContaining({ + status: outcome === 'resolve' ? 'completed' : 'failed', + attempt: 1, + usage, + }) + ); + expect(agent.effects).toBe(1); + expect(transportCalls).toBe(1); + expect(provider.getTaskUsage(task.id)).toEqual(usage); + expect(agent.signal?.aborted).toBe(false); + expect(events.getHistory().map(event => event.type)).toEqual([ + outcome === 'resolve' ? 'task.completed' : 'task.failed', + ]); + const metrics = coordinator.getMetrics().renderPrometheus(); + expect(metrics).not.toContain('noorg_tasks_retried '); + expect(metrics).not.toContain('noorg_tasks_cancelled '); + if (outcome === 'resolve') { + expect(settled?.result?.data).toEqual({ executed: true }); + expect(settled?.error).toBeUndefined(); + expect(coordinator.getHealth().lastFailure).toBeUndefined(); + expect(metrics).toContain('noorg_tasks_completed 1\n'); + expect(metrics).not.toContain('noorg_tasks_failed '); + } else { + expect(settled?.result).toBeUndefined(); + expect(settled?.error?.code).toBe('EXECUTION_REJECTED'); + expect(coordinator.getHealth().lastFailure?.code).toBe('EXECUTION_REJECTED'); + expect(metrics).toContain('noorg_tasks_failed 1\n'); + expect(metrics).not.toContain('noorg_tasks_completed '); + } + await coordinator.shutdown(); + expect(registry.activeCount()).toBe(0); + const reopened = new FileStateStore(file); + try { + await reopened.load(); + const persisted = new TaskRepository(reopened, new SystemClock()); + await persisted.load(); + expect(persisted.internalSnapshot().tasks).toEqual([settled]); + } finally { + await reopened.close(); + } } finally { - await reopened.close(); + release?.(); + await coordinator.shutdown(); + await rm(directory, { recursive: true, force: true }); } - } finally { - release?.(); - await coordinator.shutdown(); - await rm(directory, { recursive: true, force: true }); } - }); + ); }); diff --git a/tests/integration/http-boundaries.test.ts b/tests/integration/http-boundaries.test.ts index e2b9c3d..73188ee 100644 --- a/tests/integration/http-boundaries.test.ts +++ b/tests/integration/http-boundaries.test.ts @@ -11,22 +11,37 @@ import { Metrics } from '../../src/metrics/metrics'; import { DeterministicProvider } from '../../src/providers/provider'; import { MemoryStateStore } from '../../src/state/state-store'; -const errorSchema = z.object({ error: z.object({ code: z.string(), requestId: z.string().min(1) }) }); +const errorSchema = z.object({ + error: z.object({ code: z.string(), requestId: z.string().min(1) }), +}); describe('HTTP authentication and workflow boundaries', () => { it('returns structured 401/429 rather than 500 for equal-character unequal-byte bearer tokens', async () => { const coordinator = new Coordinator({ - state: new MemoryStateStore(), registry: new AgentRegistry(), provider: new DeterministicProvider(), - events: new EventBus(), metrics: new Metrics(), logger: new MemoryLogger(), clock: new SystemClock(), - pollIntervalMs: 5, maxConcurrentTasks: 1, defaultTaskTimeoutMs: 1000, + state: new MemoryStateStore(), + registry: new AgentRegistry(), + provider: new DeterministicProvider(), + events: new EventBus(), + metrics: new Metrics(), + logger: new MemoryLogger(), + clock: new SystemClock(), + pollIntervalMs: 5, + maxConcurrentTasks: 1, + defaultTaskTimeoutMs: 1000, + }); + const server = createHttpServer({ + coordinator, + metricsEnabled: false, + authToken: 'x', + authFailureThreshold: 2, }); - const server = createHttpServer({ coordinator, metricsEnabled: false, authToken: 'x', authFailureThreshold: 2 }); const listening = once(server, 'listening'); server.listen(0, '127.0.0.1'); try { await listening; const address = server.address(); - if (address === null || typeof address === 'string') throw new Error('HTTP fixture did not bind'); + if (address === null || typeof address === 'string') + throw new Error('HTTP fixture did not bind'); const url = `http://127.0.0.1:${address.port}/v1/tasks`; const first = await fetch(url, { headers: { authorization: 'Bearer é' } }); expect(first.status).toBe(401); @@ -49,10 +64,18 @@ describe('HTTP authentication and workflow boundaries', () => { it('accepts at-limit workflows, rejects breadth/depth as 400, and preserves 409 and the public page cap', async () => { const coordinator = new Coordinator({ - state: new MemoryStateStore(), registry: new AgentRegistry(), provider: new DeterministicProvider(), - events: new EventBus(), metrics: new Metrics(), logger: new MemoryLogger(), clock: new SystemClock(), - pollIntervalMs: 5, maxConcurrentTasks: 1, defaultTaskTimeoutMs: 1000, - maxWorkflowDepth: 2, maxWorkflowChildren: 1, + state: new MemoryStateStore(), + registry: new AgentRegistry(), + provider: new DeterministicProvider(), + events: new EventBus(), + metrics: new Metrics(), + logger: new MemoryLogger(), + clock: new SystemClock(), + pollIntervalMs: 5, + maxConcurrentTasks: 1, + defaultTaskTimeoutMs: 1000, + maxWorkflowDepth: 2, + maxWorkflowChildren: 1, }); const server = createHttpServer({ coordinator, metricsEnabled: false }); await coordinator.start(); @@ -61,42 +84,97 @@ describe('HTTP authentication and workflow boundaries', () => { try { await listening; const address = server.address(); - if (address === null || typeof address === 'string') throw new Error('HTTP fixture did not bind'); + if (address === null || typeof address === 'string') + throw new Error('HTTP fixture did not bind'); const base = `http://127.0.0.1:${address.port}/v1/tasks`; - const rootRequest = { name: 'Root', description: 'Blocked workflow root', input: 'x', idempotencyKey: 'http-root', requiredCapabilities: ['unavailable'] }; + const rootRequest = { + name: 'Root', + description: 'Blocked workflow root', + input: 'x', + idempotencyKey: 'http-root', + requiredCapabilities: ['unavailable'], + }; const headers = { 'content-type': 'application/json' }; - const rootResponse = await fetch(base, { method: 'POST', headers, body: JSON.stringify(rootRequest) }); + const rootResponse = await fetch(base, { + method: 'POST', + headers, + body: JSON.stringify(rootRequest), + }); expect(rootResponse.status).toBe(202); const root = taskRecordSchema.parse(await rootResponse.json()); - const childResponse = await fetch(base, { method: 'POST', headers, body: JSON.stringify({ ...rootRequest, idempotencyKey: 'http-child', parentTaskId: root.id }) }); + const childResponse = await fetch(base, { + method: 'POST', + headers, + body: JSON.stringify({ + ...rootRequest, + idempotencyKey: 'http-child', + parentTaskId: root.id, + }), + }); expect(childResponse.status).toBe(202); let leaf = taskRecordSchema.parse(await childResponse.json()); - const broad = await fetch(base, { method: 'POST', headers, body: JSON.stringify({ ...rootRequest, idempotencyKey: 'http-sibling', parentTaskId: root.id }) }); + const broad = await fetch(base, { + method: 'POST', + headers, + body: JSON.stringify({ + ...rootRequest, + idempotencyKey: 'http-sibling', + parentTaskId: root.id, + }), + }); expect(broad.status).toBe(400); - expect(errorSchema.parse(await broad.json()).error.code).toBe('TASK_WORKFLOW_BREADTH_EXCEEDED'); + expect(errorSchema.parse(await broad.json()).error.code).toBe( + 'TASK_WORKFLOW_BREADTH_EXCEEDED' + ); while (leaf.workflowDepth < 2) { - const next = await fetch(base, { method: 'POST', headers, body: JSON.stringify({ ...rootRequest, idempotencyKey: `depth-${leaf.id}`, parentTaskId: leaf.id }) }); + const next = await fetch(base, { + method: 'POST', + headers, + body: JSON.stringify({ + ...rootRequest, + idempotencyKey: `depth-${leaf.id}`, + parentTaskId: leaf.id, + }), + }); expect(next.status).toBe(202); leaf = taskRecordSchema.parse(await next.json()); } expect(leaf.workflowDepth).toBe(2); const before = coordinator.listTasks().map(task => task.id); - const deep = await fetch(base, { method: 'POST', headers, body: JSON.stringify({ ...rootRequest, idempotencyKey: 'too-deep', parentTaskId: leaf.id }) }); + const deep = await fetch(base, { + method: 'POST', + headers, + body: JSON.stringify({ ...rootRequest, idempotencyKey: 'too-deep', parentTaskId: leaf.id }), + }); expect(deep.status).toBe(400); expect(errorSchema.parse(await deep.json()).error.code).toBe('TASK_WORKFLOW_DEPTH_EXCEEDED'); expect(coordinator.listTasks().map(task => task.id)).toEqual(before); - const conflict = await fetch(base, { method: 'POST', headers, body: JSON.stringify({ ...rootRequest, input: 'changed' }) }); + const conflict = await fetch(base, { + method: 'POST', + headers, + body: JSON.stringify({ ...rootRequest, input: 'changed' }), + }); expect(conflict.status).toBe(409); expect(errorSchema.parse(await conflict.json()).error.code).toBe('IDEMPOTENCY_CONFLICT'); const tooLarge = await fetch(`${base}?limit=101`); expect(tooLarge.status).toBe(400); for (let index = 0; index < 105; index += 1) { - await coordinator.submitTask({ name: `Public page ${index}`, description: 'Keep public bounds', input: 'x', requiredCapabilities: ['unavailable'] }); + await coordinator.submitTask({ + name: `Public page ${index}`, + description: 'Keep public bounds', + input: 'x', + requiredCapabilities: ['unavailable'], + }); } const publicPage = await fetch(base); expect(z.array(taskRecordSchema).parse(await publicPage.json())).toHaveLength(100); const tail = await fetch(`${base}?offset=100&limit=100`); - expect(z.array(taskRecordSchema).parse(await tail.json()).map(task => task.id)).toEqual(coordinator.listTasks(undefined, { offset: 100, limit: 100 }).map(task => task.id)); + expect( + z + .array(taskRecordSchema) + .parse(await tail.json()) + .map(task => task.id) + ).toEqual(coordinator.listTasks(undefined, { offset: 100, limit: 100 }).map(task => task.id)); expect(coordinator.getHealth().queueDepth).toBe(before.length + 105); } finally { if (server.listening) { diff --git a/tests/integration/queue-traversal.test.ts b/tests/integration/queue-traversal.test.ts index b1191aa..a274c19 100644 --- a/tests/integration/queue-traversal.test.ts +++ b/tests/integration/queue-traversal.test.ts @@ -15,8 +15,12 @@ import { TaskRepository } from '../../src/tasks/task-repository'; class HoldingAgent extends AbstractAgent { public started = 0; private readonly settlement = Promise.withResolvers(); - public constructor() { super({ id: 'holding', name: 'Holding', capabilities: ['holding'], maxConcurrentTasks: 1 }); } - public release(): void { this.settlement.resolve({ data: { done: true }, summary: 'Released' }); } + public constructor() { + super({ id: 'holding', name: 'Holding', capabilities: ['holding'], maxConcurrentTasks: 1 }); + } + public release(): void { + this.settlement.resolve({ data: { done: true }, summary: 'Released' }); + } protected override async run(_task: AgentTask): Promise { this.started += 1; return this.settlement.promise; @@ -31,7 +35,11 @@ async function waitFor(condition: () => boolean): Promise { } } -const base = { name: 'Queue fixture', description: 'Exercise complete internal traversal', input: 'A complete paragraph for local analysis.' }; +const base = { + name: 'Queue fixture', + description: 'Exercise complete internal traversal', + input: 'A complete paragraph for local analysis.', +}; describe('Complete internal queue traversal', () => { it.each(['capability', 'capacity', 'backoff', 'workflow'] as const)( @@ -46,33 +54,61 @@ describe('Complete internal queue traversal', () => { registry.register(holding); registry.register(new AnalysisAgent()); await repository.load(); - const parent = kind === 'workflow' ? await repository.create({ ...base, priority: 100, requiredCapabilities: ['missing'] }) : undefined; + const parent = + kind === 'workflow' + ? await repository.create({ ...base, priority: 100, requiredCapabilities: ['missing'] }) + : undefined; for (let index = 0; index < 101; index += 1) { if (kind === 'backoff') { - const task = await repository.create({ ...base, agentId: 'analysis', priority: 100, maxAttempts: 2, retryBackoffMs: 60_000 }); + const task = await repository.create({ + ...base, + agentId: 'analysis', + priority: 100, + maxAttempts: 2, + retryBackoffMs: 60_000, + }); await repository.start(task.id, 'analysis'); - await repository.fail(task.id, { code: 'TRANSIENT', message: 'Wait for retry', retryable: true }); + await repository.fail(task.id, { + code: 'TRANSIENT', + message: 'Wait for retry', + retryable: true, + }); } else { await repository.create({ - ...base, priority: 100, - ...(kind === 'capability' ? { requiredCapabilities: ['missing'] } : { agentId: kind === 'capacity' ? 'holding' : 'analysis' }), + ...base, + priority: 100, + ...(kind === 'capability' + ? { requiredCapabilities: ['missing'] } + : { agentId: kind === 'capacity' ? 'holding' : 'analysis' }), ...(parent === undefined ? {} : { dependencyIds: [parent.id] }), }); } } const runnable = await repository.create({ ...base, priority: 1, agentId: 'analysis' }); const coordinator = new Coordinator({ - state, repository, registry, clock, provider: new DeterministicProvider(), events: new EventBus(), - metrics: new Metrics(), logger: new MemoryLogger(), pollIntervalMs: 5, - maxConcurrentTasks: 2, defaultTaskTimeoutMs: 10_000, + state, + repository, + registry, + clock, + provider: new DeterministicProvider(), + events: new EventBus(), + metrics: new Metrics(), + logger: new MemoryLogger(), + pollIntervalMs: 5, + maxConcurrentTasks: 2, + defaultTaskTimeoutMs: 10_000, }); try { await coordinator.start(); await waitFor(() => coordinator.getTask(runnable.id)?.status === 'completed'); const expected = kind === 'workflow' ? 102 : kind === 'capacity' ? 100 : 101; - expect(coordinator.getHealth()).toEqual(expect.objectContaining({ queueDepth: expected, blockedTasks: expected })); + expect(coordinator.getHealth()).toEqual( + expect.objectContaining({ queueDepth: expected, blockedTasks: expected }) + ); expect(coordinator.listTasks('queued')).toHaveLength(100); - expect(coordinator.getMetrics().renderPrometheus().split('\n')).toContain(`noorg_queue_depth ${expected}`); + expect(coordinator.getMetrics().renderPrometheus().split('\n')).toContain( + `noorg_queue_depth ${expected}` + ); if (kind === 'capacity') { expect(holding.started).toBe(1); expect(coordinator.getHealth().blockedCapacityTasks).toBe(100); @@ -100,20 +136,48 @@ describe('Complete internal queue traversal', () => { await repository.create({ ...base, priority: 99, requiredCapabilities: ['missing'] }); const parent = await repository.create({ ...base, requiredCapabilities: ['missing'] }); await repository.cancel(parent.id); - const expired = await repository.create({ ...base, priority: 1, deadlineAt: new Date(now + 1000).toISOString(), requiredCapabilities: ['missing'] }); - const dependency = await repository.create({ ...base, priority: 1, agentId: 'analysis', dependencyIds: [parent.id] }); - const child = await repository.create({ ...base, priority: 1, agentId: 'analysis', parentTaskId: parent.id, parentCompletionPolicy: 'wait_for_success' }); + const expired = await repository.create({ + ...base, + priority: 1, + deadlineAt: new Date(now + 1000).toISOString(), + requiredCapabilities: ['missing'], + }); + const dependency = await repository.create({ + ...base, + priority: 1, + agentId: 'analysis', + dependencyIds: [parent.id], + }); + const child = await repository.create({ + ...base, + priority: 1, + agentId: 'analysis', + parentTaskId: parent.id, + parentCompletionPolicy: 'wait_for_success', + }); const runnable = await repository.create({ ...base, priority: 0, agentId: 'analysis' }); now += 2000; const coordinator = new Coordinator({ - state, repository, registry, clock, provider: new DeterministicProvider(), events: new EventBus(), - metrics: new Metrics(), logger: new MemoryLogger(), pollIntervalMs: 5, - maxConcurrentTasks: 1, defaultTaskTimeoutMs: 10_000, + state, + repository, + registry, + clock, + provider: new DeterministicProvider(), + events: new EventBus(), + metrics: new Metrics(), + logger: new MemoryLogger(), + pollIntervalMs: 5, + maxConcurrentTasks: 1, + defaultTaskTimeoutMs: 10_000, }); try { await coordinator.start(); await waitFor(() => coordinator.getTask(occupied.id)?.status === 'running'); - await waitFor(() => [expired.id, dependency.id, child.id].every(id => coordinator.getTask(id)?.status === 'failed')); + await waitFor(() => + [expired.id, dependency.id, child.id].every( + id => coordinator.getTask(id)?.status === 'failed' + ) + ); expect(registry.activeCount()).toBe(1); expect(holding.started).toBe(1); expect(coordinator.getTask(expired.id)?.error?.code).toBe('TASK_DEADLINE_EXCEEDED'); diff --git a/tests/integration/state-lock.test.ts b/tests/integration/state-lock.test.ts index 176e4d5..4f44b33 100644 --- a/tests/integration/state-lock.test.ts +++ b/tests/integration/state-lock.test.ts @@ -8,8 +8,18 @@ import { runtimeStateSchema } from '../../src/domain/types'; import { FileStateStore } from '../../src/state/state-store'; import { StateBoundary } from '../fixtures/state-boundary'; -const messageSchema = z.object({ type: z.string(), code: z.string().optional(), pid: z.number().optional(), revision: z.number().optional(), value: z.unknown().optional() }); -interface LockChild { readonly process: ChildProcess; readonly exited: Promise; readonly initial: Promise> } +const messageSchema = z.object({ + type: z.string(), + code: z.string().optional(), + pid: z.number().optional(), + revision: z.number().optional(), + value: z.unknown().optional(), +}); +interface LockChild { + readonly process: ChildProcess; + readonly exited: Promise; + readonly initial: Promise>; +} async function nextMessage(child: ChildProcess): Promise> { const [message] = await once(child, 'message', { signal: AbortSignal.timeout(5000) }); @@ -18,8 +28,10 @@ async function nextMessage(child: ChildProcess): Promise z.number().nullable().parse(code)); @@ -28,48 +40,62 @@ function contender(file: string, mode: 'pause-stale-read' | 'writer'): LockChild async function stopChildren(children: readonly LockChild[]): Promise { for (const child of children) { - if (child.process.exitCode === null && child.process.signalCode === null) child.process.kill('SIGKILL'); + if (child.process.exitCode === null && child.process.signalCode === null) + child.process.kill('SIGKILL'); } await Promise.all(children.map(child => child.exited)); } describe('Physical state lock ownership', () => { - it.each([0, 1, 2])('rechecks stale O under arbitration after A replaces it while B is paused (%i)', async () => { - const directory = await mkdtemp(join(tmpdir(), 'noorg-lock-race-')); - const file = join(directory, 'state.json'); - const lock = `${file}.lock`; - const children: LockChild[] = []; - const third = new FileStateStore(file); - try { - await mkdir(lock); - await writeFile(join(lock, 'owner'), JSON.stringify({ pid: 99999999, createdAt: Date.now() - 120_000, token: 'stale-O' })); - const b = contender(file, 'pause-stale-read'); - children.push(b); - expect(await b.initial).toEqual({ type: 'observed' }); - const a = contender(file, 'writer'); - children.push(a); - expect(await a.initial).toEqual(expect.objectContaining({ type: 'owned', pid: a.process.pid, revision: 1 })); - const ownerA = await readFile(join(lock, 'owner'), 'utf8'); - const inodeA = await lstat(lock); - const rejected = nextMessage(b.process); - b.process.send({ type: 'resume' }); - expect(await rejected).toEqual({ type: 'rejected', code: 'STATE_LOCKED' }); - expect(await b.exited).toBe(0); - expect(await readFile(join(lock, 'owner'), 'utf8')).toBe(ownerA); - expect(await lstat(lock)).toEqual(expect.objectContaining({ dev: inodeA.dev, ino: inodeA.ino })); - await expect(third.load()).rejects.toMatchObject({ code: 'STATE_LOCKED' }); - const flushed = nextMessage(a.process); - a.process.send({ type: 'flush' }); - expect(await flushed).toEqual({ type: 'flushed', revision: 1, value: 'A' }); - expect(runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state.writer).toBe('A'); - a.process.send({ type: 'close' }); - expect(await a.exited).toBe(0); - } finally { - await stopChildren(children); - await third.close(); - await rm(directory, { recursive: true, force: true }); - } - }, 15_000); + it.each([0, 1, 2])( + 'rechecks stale O under arbitration after A replaces it while B is paused (%i)', + async () => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-lock-race-')); + const file = join(directory, 'state.json'); + const lock = `${file}.lock`; + const children: LockChild[] = []; + const third = new FileStateStore(file); + try { + await mkdir(lock); + await writeFile( + join(lock, 'owner'), + JSON.stringify({ pid: 99999999, createdAt: Date.now() - 120_000, token: 'stale-O' }) + ); + const b = contender(file, 'pause-stale-read'); + children.push(b); + expect(await b.initial).toEqual({ type: 'observed' }); + const a = contender(file, 'writer'); + children.push(a); + expect(await a.initial).toEqual( + expect.objectContaining({ type: 'owned', pid: a.process.pid, revision: 1 }) + ); + const ownerA = await readFile(join(lock, 'owner'), 'utf8'); + const inodeA = await lstat(lock); + const rejected = nextMessage(b.process); + b.process.send({ type: 'resume' }); + expect(await rejected).toEqual({ type: 'rejected', code: 'STATE_LOCKED' }); + expect(await b.exited).toBe(0); + expect(await readFile(join(lock, 'owner'), 'utf8')).toBe(ownerA); + expect(await lstat(lock)).toEqual( + expect.objectContaining({ dev: inodeA.dev, ino: inodeA.ino }) + ); + await expect(third.load()).rejects.toMatchObject({ code: 'STATE_LOCKED' }); + const flushed = nextMessage(a.process); + a.process.send({ type: 'flush' }); + expect(await flushed).toEqual({ type: 'flushed', revision: 1, value: 'A' }); + expect( + runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state.writer + ).toBe('A'); + a.process.send({ type: 'close' }); + expect(await a.exited).toBe(0); + } finally { + await stopChildren(children); + await third.close(); + await rm(directory, { recursive: true, force: true }); + } + }, + 15_000 + ); it('cannot flush or close through a replacement physical owner, even when the replacement advertises the same pid', async () => { const directory = await mkdtemp(join(tmpdir(), 'noorg-lock-replace-')); @@ -81,7 +107,11 @@ describe('Physical state lock ownership', () => { const before = await readFile(file, 'utf8'); await rename(`${file}.lock`, `${file}.lock.detached`); await mkdir(`${file}.lock`); - const replacement = JSON.stringify({ pid: process.pid, createdAt: Date.now(), token: 'replacement' }); + const replacement = JSON.stringify({ + pid: process.pid, + createdAt: Date.now(), + token: 'replacement', + }); await writeFile(`${file}.lock/owner`, replacement); await expect(state.set('accepted', 99)).rejects.toMatchObject({ code: 'STATE_LOCK_LOST' }); expect(state.get('accepted')).toBe(1); @@ -113,7 +143,11 @@ describe('Physical state lock ownership', () => { await barrier.entered; await rename(`${file}.lock`, `${file}.lock.detached`); await mkdir(`${file}.lock`); - const replacement = JSON.stringify({ pid: process.pid, createdAt: Date.now(), token: 'foreign-during-write' }); + const replacement = JSON.stringify({ + pid: process.pid, + createdAt: Date.now(), + token: 'foreign-during-write', + }); await writeFile(`${file}.lock/owner`, replacement); barrier.release(); await rejection; @@ -127,89 +161,107 @@ describe('Physical state lock ownership', () => { } }); - it.each(['missing', 'invalid', 'numeric-junk', 'guard'] as const)('fails closed for an ambiguous %s owner without deleting evidence', async kind => { - const directory = await mkdtemp(join(tmpdir(), 'noorg-lock-ambiguous-')); - const file = join(directory, 'state.json'); - const state = new FileStateStore(file); - const evidencePath = kind === 'guard' ? `${file}.lock.guard` : `${file}.lock`; - try { - await mkdir(evidencePath); - if (kind === 'invalid') await writeFile(join(evidencePath, 'owner'), '{not-json'); - if (kind === 'numeric-junk') await writeFile(join(evidencePath, 'owner'), '99999999#'); - const identity = await lstat(evidencePath); - await expect(state.load()).rejects.toMatchObject({ code: 'STATE_LOCKED' }); - expect(await lstat(evidencePath)).toEqual(expect.objectContaining({ dev: identity.dev, ino: identity.ino })); - expect(state.isOpen()).toBe(false); - } finally { - await state.close(); - await rm(directory, { recursive: true, force: true }); + it.each(['missing', 'invalid', 'numeric-junk', 'guard'] as const)( + 'fails closed for an ambiguous %s owner without deleting evidence', + async kind => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-lock-ambiguous-')); + const file = join(directory, 'state.json'); + const state = new FileStateStore(file); + const evidencePath = kind === 'guard' ? `${file}.lock.guard` : `${file}.lock`; + try { + await mkdir(evidencePath); + if (kind === 'invalid') await writeFile(join(evidencePath, 'owner'), '{not-json'); + if (kind === 'numeric-junk') await writeFile(join(evidencePath, 'owner'), '99999999#'); + const identity = await lstat(evidencePath); + await expect(state.load()).rejects.toMatchObject({ code: 'STATE_LOCKED' }); + expect(await lstat(evidencePath)).toEqual( + expect.objectContaining({ dev: identity.dev, ino: identity.ino }) + ); + expect(state.isOpen()).toBe(false); + } finally { + await state.close(); + await rm(directory, { recursive: true, force: true }); + } } - }); + ); }); describe('Durable state commit failure boundaries', () => { - it.each(['write', 'sync'] as const)('keeps rejected %s candidates out of get/snapshot/revision and later flush/close/reload', async fault => { - const directory = await mkdtemp(join(tmpdir(), 'noorg-commit-failure-')); - const file = join(directory, 'state.json'); - const boundary = new StateBoundary(file); - const state = new FileStateStore(file, boundary.io); - try { - await state.load(); - await state.set('accepted', { value: 1 }); - const revision = state.revision(); - boundary.failNext(fault); - await expect(state.set('accepted', { value: 99 })).rejects.toBe(boundary.error); - expect(state.get('accepted')).toEqual({ value: 1 }); - expect(state.snapshot()).toEqual({ accepted: { value: 1 } }); - expect(state.revision()).toBe(revision); - const updaterError = new Error('Rejected updater'); - await expect(state.update<{ value: number }>('accepted', current => { - if (current === undefined) throw new Error('Missing fixture'); - current.value = 77; - throw updaterError; - })).rejects.toBe(updaterError); - await state.flush(); - await state.close(); - const reopened = new FileStateStore(file); + it.each(['write', 'sync'] as const)( + 'keeps rejected %s candidates out of get/snapshot/revision and later flush/close/reload', + async fault => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-commit-failure-')); + const file = join(directory, 'state.json'); + const boundary = new StateBoundary(file); + const state = new FileStateStore(file, boundary.io); try { - await reopened.load(); - expect(reopened.snapshot()).toEqual({ accepted: { value: 1 } }); - expect(reopened.revision()).toBe(revision); - } finally { await reopened.close(); } - } finally { - await state.close(); - await rm(directory, { recursive: true, force: true }); + await state.load(); + await state.set('accepted', { value: 1 }); + const revision = state.revision(); + boundary.failNext(fault); + await expect(state.set('accepted', { value: 99 })).rejects.toBe(boundary.error); + expect(state.get('accepted')).toEqual({ value: 1 }); + expect(state.snapshot()).toEqual({ accepted: { value: 1 } }); + expect(state.revision()).toBe(revision); + const updaterError = new Error('Rejected updater'); + await expect( + state.update<{ value: number }>('accepted', current => { + if (current === undefined) throw new Error('Missing fixture'); + current.value = 77; + throw updaterError; + }) + ).rejects.toBe(updaterError); + await state.flush(); + await state.close(); + const reopened = new FileStateStore(file); + try { + await reopened.load(); + expect(reopened.snapshot()).toEqual({ accepted: { value: 1 } }); + expect(reopened.revision()).toBe(revision); + } finally { + await reopened.close(); + } + } finally { + await state.close(); + await rm(directory, { recursive: true, force: true }); + } } - }); + ); - it.each(['rename-before', 'rename-after', 'directory-sync'] as const)('quarantines uncertain %s outcomes without publishing or automatically unlocking them', async fault => { - const directory = await mkdtemp(join(tmpdir(), 'noorg-commit-uncertain-')); - const file = join(directory, 'state.json'); - const boundary = new StateBoundary(file); - const state = new FileStateStore(file, boundary.io); - const follower = new FileStateStore(file); - try { - await state.load(); - await state.set('accepted', 1); - const revision = state.revision(); - boundary.failNext(fault); - await expect(state.set('accepted', 99)).rejects.toMatchObject({ code: 'STATE_COMMIT_UNCERTAIN', cause: boundary.error }); - expect(state.snapshot()).toEqual({ accepted: 1 }); - expect(state.revision()).toBe(revision); - expect(state.isOpen()).toBe(false); - expect((await lstat(`${file}.lock.guard`)).isDirectory()).toBe(true); - await expect(state.flush()).rejects.toMatchObject({ code: 'STATE_COMMIT_UNCERTAIN' }); - await expect(state.close()).rejects.toMatchObject({ code: 'STATE_COMMIT_UNCERTAIN' }); - await expect(follower.load()).rejects.toMatchObject({ code: 'STATE_LOCKED' }); - const disk = runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))); - expect(disk.state.accepted).toBe(fault === 'rename-before' ? 1 : 99); - // A post-effect file is evidence of uncertainty, not an accepted repository revision. - expect(state.revision()).toBe(revision); - } finally { - await Promise.allSettled([state.close(), follower.close()]); - await rm(directory, { recursive: true, force: true }); + it.each(['rename-before', 'rename-after', 'directory-sync'] as const)( + 'quarantines uncertain %s outcomes without publishing or automatically unlocking them', + async fault => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-commit-uncertain-')); + const file = join(directory, 'state.json'); + const boundary = new StateBoundary(file); + const state = new FileStateStore(file, boundary.io); + const follower = new FileStateStore(file); + try { + await state.load(); + await state.set('accepted', 1); + const revision = state.revision(); + boundary.failNext(fault); + await expect(state.set('accepted', 99)).rejects.toMatchObject({ + code: 'STATE_COMMIT_UNCERTAIN', + cause: boundary.error, + }); + expect(state.snapshot()).toEqual({ accepted: 1 }); + expect(state.revision()).toBe(revision); + expect(state.isOpen()).toBe(false); + expect((await lstat(`${file}.lock.guard`)).isDirectory()).toBe(true); + await expect(state.flush()).rejects.toMatchObject({ code: 'STATE_COMMIT_UNCERTAIN' }); + await expect(state.close()).rejects.toMatchObject({ code: 'STATE_COMMIT_UNCERTAIN' }); + await expect(follower.load()).rejects.toMatchObject({ code: 'STATE_LOCKED' }); + const disk = runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))); + expect(disk.state.accepted).toBe(fault === 'rename-before' ? 1 : 99); + // A post-effect file is evidence of uncertainty, not an accepted repository revision. + expect(state.revision()).toBe(revision); + } finally { + await Promise.allSettled([state.close(), follower.close()]); + await rm(directory, { recursive: true, force: true }); + } } - }); + ); it('does not expose a pending candidate and drains it before a concurrent close', async () => { const directory = await mkdtemp(join(tmpdir(), 'noorg-commit-close-')); @@ -231,7 +283,9 @@ describe('Durable state commit failure boundaries', () => { barrier.release(); await pending; await closing; - expect(runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state).toEqual({ accepted: 2 }); + expect(runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state).toEqual({ + accepted: 2, + }); } finally { release?.(); await state.close(); @@ -252,14 +306,19 @@ describe('Incomplete namespace acquisition', () => { await state.set('accepted', 1); const revision = state.revision(); boundary.failNext('guard-open'); - await expect(state.set('rejected', 99)).rejects.toMatchObject({ code: 'STATE_LOCK_LOST', cause: boundary.error }); + await expect(state.set('rejected', 99)).rejects.toMatchObject({ + code: 'STATE_LOCK_LOST', + cause: boundary.error, + }); expect(state.snapshot()).toEqual({ accepted: 1 }); expect(state.revision()).toBe(revision); expect(state.isOpen()).toBe(false); expect((await lstat(`${file}.lock.guard`)).isDirectory()).toBe(true); await expect(state.close()).rejects.toMatchObject({ code: 'STATE_LOCK_LOST' }); await expect(follower.load()).rejects.toMatchObject({ code: 'STATE_LOCKED' }); - expect(runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state).toEqual({ accepted: 1 }); + expect(runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))).state).toEqual({ + accepted: 1, + }); } finally { await Promise.allSettled([state.close(), follower.close()]); await rm(directory, { recursive: true, force: true }); @@ -268,46 +327,54 @@ describe('Incomplete namespace acquisition', () => { }); describe('Retained owner identities', () => { - it.each(['token', 'guard'] as const)('does not publish or clean through a changed %s identity during a real write', async kind => { - const directory = await mkdtemp(join(tmpdir(), 'noorg-retained-identity-')); - const file = join(directory, 'state.json'); - const boundary = new StateBoundary(file); - const state = new FileStateStore(file, boundary.io); - let release: (() => void) | undefined; - try { - await state.load(); - await state.set('accepted', 1); - const before = await readFile(file, 'utf8'); - const barrier = boundary.pauseNextWrite(); - release = barrier.release; - const pending = state.set('rejected', 99); - const rejection = kind === 'token' - ? expect(pending).rejects.toMatchObject({ code: 'STATE_LOCK_LOST' }) - : expect(pending).rejects.toBeInstanceOf(AggregateError); - await barrier.entered; - let evidencePath: string; - let evidence: string; - if (kind === 'token') { - evidencePath = `${file}.lock/owner`; - evidence = JSON.stringify({ pid: process.pid, createdAt: Date.now(), token: 'new-token-same-directory' }); - } else { - await rename(`${file}.lock.guard`, `${file}.lock.guard.detached`); - await mkdir(`${file}.lock.guard`); - evidencePath = `${file}.lock.guard/foreign.txt`; - evidence = 'Foreign guard evidence'; + it.each(['token', 'guard'] as const)( + 'does not publish or clean through a changed %s identity during a real write', + async kind => { + const directory = await mkdtemp(join(tmpdir(), 'noorg-retained-identity-')); + const file = join(directory, 'state.json'); + const boundary = new StateBoundary(file); + const state = new FileStateStore(file, boundary.io); + let release: (() => void) | undefined; + try { + await state.load(); + await state.set('accepted', 1); + const before = await readFile(file, 'utf8'); + const barrier = boundary.pauseNextWrite(); + release = barrier.release; + const pending = state.set('rejected', 99); + const rejection = + kind === 'token' + ? expect(pending).rejects.toMatchObject({ code: 'STATE_LOCK_LOST' }) + : expect(pending).rejects.toBeInstanceOf(AggregateError); + await barrier.entered; + let evidencePath: string; + let evidence: string; + if (kind === 'token') { + evidencePath = `${file}.lock/owner`; + evidence = JSON.stringify({ + pid: process.pid, + createdAt: Date.now(), + token: 'new-token-same-directory', + }); + } else { + await rename(`${file}.lock.guard`, `${file}.lock.guard.detached`); + await mkdir(`${file}.lock.guard`); + evidencePath = `${file}.lock.guard/foreign.txt`; + evidence = 'Foreign guard evidence'; + } + await writeFile(evidencePath, evidence); + barrier.release(); + await rejection; + expect(state.snapshot()).toEqual({ accepted: 1 }); + expect(state.isOpen()).toBe(false); + await expect(state.close()).rejects.toMatchObject({ code: 'STATE_LOCK_LOST' }); + expect(await readFile(file, 'utf8')).toBe(before); + expect(await readFile(evidencePath, 'utf8')).toBe(evidence); + } finally { + release?.(); + await Promise.allSettled([state.close()]); + await rm(directory, { recursive: true, force: true }); } - await writeFile(evidencePath, evidence); - barrier.release(); - await rejection; - expect(state.snapshot()).toEqual({ accepted: 1 }); - expect(state.isOpen()).toBe(false); - await expect(state.close()).rejects.toMatchObject({ code: 'STATE_LOCK_LOST' }); - expect(await readFile(file, 'utf8')).toBe(before); - expect(await readFile(evidencePath, 'utf8')).toBe(evidence); - } finally { - release?.(); - await Promise.allSettled([state.close()]); - await rm(directory, { recursive: true, force: true }); } - }); + ); }); diff --git a/tests/integration/task-commit.test.ts b/tests/integration/task-commit.test.ts index a888a51..a505571 100644 --- a/tests/integration/task-commit.test.ts +++ b/tests/integration/task-commit.test.ts @@ -9,7 +9,13 @@ import { FileStateStore, MemoryStateStore } from '../../src/state/state-store'; import { TaskRepository } from '../../src/tasks/task-repository'; import { StateBoundary } from '../fixtures/state-boundary'; -const request = { name: 'Durable task', description: 'Publish only committed work', input: 'x', agentId: 'analysis', maxAttempts: 2 }; +const request = { + name: 'Durable task', + description: 'Publish only committed work', + input: 'x', + agentId: 'analysis', + maxAttempts: 2, +}; const two = { promptTokens: 1, completionTokens: 1, totalTokens: 2, costUsd: 0.02 }; const five = { promptTokens: 3, completionTokens: 2, totalTokens: 5, costUsd: 0.05 }; @@ -42,7 +48,9 @@ describe('TaskRepository durable publication', () => { await rejection; const accepted = await duplicate; expect(repository.list().map(task => task.id)).toEqual([accepted.id]); - expect((await repository.create({ ...request, idempotencyKey: 'same-key' })).id).toBe(accepted.id); + expect((await repository.create({ ...request, idempotencyKey: 'same-key' })).id).toBe( + accepted.id + ); await repository.persist(); await state.close(); const reopened = new FileStateStore(file); @@ -52,7 +60,9 @@ describe('TaskRepository durable publication', () => { await recovered.load(); expect(recovered.list().map(task => task.id)).toEqual([accepted.id]); expect(recovered.get(accepted.id)?.attempt).toBe(0); - } finally { await reopened.close(); } + } finally { + await reopened.close(); + } } finally { barrier.release(); await state.close(); @@ -82,11 +92,15 @@ describe('TaskRepository durable publication', () => { boundary.failNext('sync'); barrier.release(); await rejection; - expect(await cancellation).toEqual(expect.objectContaining({ status: 'cancelled', attempt: 0 })); + expect(await cancellation).toEqual( + expect.objectContaining({ status: 'cancelled', attempt: 0 }) + ); await repository.persist(); await state.close(); const envelope = runtimeStateSchema.parse(JSON.parse(await readFile(file, 'utf8'))); - expect(envelope.state.tasks).toEqual([expect.objectContaining({ status: 'cancelled', attempt: 0 })]); + expect(envelope.state.tasks).toEqual([ + expect.objectContaining({ status: 'cancelled', attempt: 0 }), + ]); } finally { release?.(); await state.close(); @@ -110,12 +124,22 @@ describe('TaskRepository durable publication', () => { await expect(repository.cancel(task.id)).rejects.toBe(boundary.error); expect(repository.get(task.id)).toEqual(running); boundary.failNext('sync'); - await expect(repository.completeWithUsage(task.id, { data: { rejected: true }, summary: 'Not committed' }, five)).rejects.toBe(boundary.error); + await expect( + repository.completeWithUsage( + task.id, + { data: { rejected: true }, summary: 'Not committed' }, + five + ) + ).rejects.toBe(boundary.error); expect(repository.get(task.id)).toEqual(running); boundary.failNext('write'); await expect(repository.recordUsage(task.id, five)).rejects.toBe(boundary.error); expect(repository.get(task.id)?.usage).toBeUndefined(); - const completed = await repository.completeWithUsage(task.id, { data: { accepted: true }, summary: 'Committed' }, two); + const completed = await repository.completeWithUsage( + task.id, + { data: { accepted: true }, summary: 'Committed' }, + two + ); expect(completed.usage).toEqual(two); expect(completed.result?.data).toEqual({ accepted: true }); await repository.persist(); @@ -126,7 +150,9 @@ describe('TaskRepository durable publication', () => { const recovered = new TaskRepository(reopened, new SystemClock()); await recovered.load(); expect(recovered.get(task.id)).toEqual(completed); - } finally { await reopened.close(); } + } finally { + await reopened.close(); + } } finally { await state.close(); await rm(directory, { recursive: true, force: true }); @@ -165,7 +191,9 @@ describe('TaskRepository durable publication', () => { expect(await reloaded.recoverInterruptedTasks()).toEqual([ expect.objectContaining({ id: task.id, status: 'queued', attempt: 1 }), ]); - expect(reloaded.get(task.id)).toEqual(expect.objectContaining({ status: 'queued', attempt: 1 })); + expect(reloaded.get(task.id)).toEqual( + expect.objectContaining({ status: 'queued', attempt: 1 }) + ); } finally { await third?.close(); await second?.close(); @@ -179,15 +207,20 @@ describe('TaskRepository durable publication', () => { const original = new TaskRepository(state, new SystemClock()); await original.load(); const accepted = await original.create(request); - await state.set('tasks', jsonValueSchema.parse([ - accepted, - { ...accepted, id: randomUUID(), dependencyIds: [randomUUID()] }, - ])); + await state.set( + 'tasks', + jsonValueSchema.parse([ + accepted, + { ...accepted, id: randomUUID(), dependencyIds: [randomUUID()] }, + ]) + ); const loading = new TaskRepository(state, new SystemClock()); await expect(loading.load()).rejects.toBeInstanceOf(NoOrgError); expect(loading.list()).toEqual([]); expect(loading.get(accepted.id)).toBeUndefined(); - } finally { await state.close(); } + } finally { + await state.close(); + } }); }); diff --git a/tests/integration/usage-accounting.test.ts b/tests/integration/usage-accounting.test.ts index 8024e8a..8f18210 100644 --- a/tests/integration/usage-accounting.test.ts +++ b/tests/integration/usage-accounting.test.ts @@ -25,18 +25,34 @@ const seven: UsageMetadata = { promptTokens: 4, completionTokens: 3, totalTokens function offlineMeter(receipts: readonly UsageMetadata[]) { const effects = { calls: 0 }; - const provider = new GovernedProvider(new OpenAIProvider({ - apiKey: 'unused-offline-test-key', model: 'offline-test', timeoutMs: 1000, maxRetries: 0, retryBaseMs: 1, - promptCostPerMillionUsd: 1_000_000, completionCostPerMillionUsd: 1_000_000, - transport: { complete: async () => { - const receipt = receipts[effects.calls]; - effects.calls += 1; - if (receipt === undefined) throw new Error('Unexpected offline transport invocation'); - return { id: `offline-${effects.calls}`, model: 'offline-test', parsed: { ok: true }, usage: { - prompt_tokens: receipt.promptTokens, completion_tokens: receipt.completionTokens, total_tokens: receipt.totalTokens, - } }; - } }, - })); + const provider = new GovernedProvider( + new OpenAIProvider({ + apiKey: 'unused-offline-test-key', + model: 'offline-test', + timeoutMs: 1000, + maxRetries: 0, + retryBaseMs: 1, + promptCostPerMillionUsd: 1_000_000, + completionCostPerMillionUsd: 1_000_000, + transport: { + complete: async () => { + const receipt = receipts[effects.calls]; + effects.calls += 1; + if (receipt === undefined) throw new Error('Unexpected offline transport invocation'); + return { + id: `offline-${effects.calls}`, + model: 'offline-test', + parsed: { ok: true }, + usage: { + prompt_tokens: receipt.promptTokens, + completion_tokens: receipt.completionTokens, + total_tokens: receipt.totalTokens, + }, + }; + }, + }, + }) + ); return { provider, effects }; } @@ -44,31 +60,53 @@ class ChargedAgent extends AbstractAgent { public attempts = 0; public readonly ready = Promise.withResolvers(); private readonly settlement = Promise.withResolvers(); - public constructor(private readonly mode: 'retry' | 'late' | 'complete', private readonly beforeComplete?: () => void) { + public constructor( + private readonly mode: 'retry' | 'late' | 'complete', + private readonly beforeComplete?: () => void + ) { super({ id: 'charged', name: 'Charged', capabilities: ['charged'], maxConcurrentTasks: 1 }); } - public release(): void { this.settlement.resolve(); } + public release(): void { + this.settlement.resolve(); + } protected override async run(task: AgentTask, signal: AbortSignal): Promise { this.attempts += 1; await this.ask({ operation: 'charge', input: null }, output, signal, task.id); - if (this.mode === 'retry' && this.attempts === 1) throw new NoOrgError('TRANSIENT', 'Retry after a charged attempt', true); + if (this.mode === 'retry' && this.attempts === 1) + throw new NoOrgError('TRANSIENT', 'Retry after a charged attempt', true); if (this.mode === 'late') { this.ready.resolve(); await this.settlement.promise; // Deliberately uncooperative work: real offline receipt after cancellation/timeout. - await this.context.provider.complete({ operation: 'late-charge', input: null, taskId: task.id }, output); + await this.context.provider.complete( + { operation: 'late-charge', input: null, taskId: task.id }, + output + ); } this.beforeComplete?.(); return { data: { ok: true, attempt: this.attempts }, summary: 'Charged work settled' }; } } -function usageRuntime(state: StateStore, provider: GovernedProvider, agent: ChargedAgent): { coordinator: Coordinator; registry: AgentRegistry } { +function usageRuntime( + state: StateStore, + provider: GovernedProvider, + agent: ChargedAgent +): { coordinator: Coordinator; registry: AgentRegistry } { const registry = new AgentRegistry(); registry.register(agent); - const coordinator = new Coordinator({ state, registry, provider, events: new EventBus(), metrics: new Metrics(), - logger: new MemoryLogger(), clock: new SystemClock(), pollIntervalMs: 5, maxConcurrentTasks: 1, - defaultTaskTimeoutMs: 2000 }); + const coordinator = new Coordinator({ + state, + registry, + provider, + events: new EventBus(), + metrics: new Metrics(), + logger: new MemoryLogger(), + clock: new SystemClock(), + pollIntervalMs: 5, + maxConcurrentTasks: 1, + defaultTaskTimeoutMs: 2000, + }); return { coordinator, registry }; } @@ -80,7 +118,14 @@ async function waitFor(condition: () => boolean): Promise { } } -const request = { agentId: 'charged', name: 'Account usage', description: 'Charge each provider receipt once', input: null, maxAttempts: 2, retryBackoffMs: 0 }; +const request = { + agentId: 'charged', + name: 'Account usage', + description: 'Charge each provider receipt once', + input: null, + maxAttempts: 2, + retryBackoffMs: 0, +}; describe('Provider-instance usage deltas', () => { it('adds 5 then 2 as 7, not cumulative 5 plus cumulative 7, across retries', async () => { @@ -95,17 +140,26 @@ describe('Provider-instance usage deltas', () => { expect(meter.provider.getTaskUsage(task.id)).toEqual(seven); expect(meter.effects.calls).toBe(2); expect(agent.attempts).toBe(2); - } finally { agent.release(); await coordinator.shutdown(); } + } finally { + agent.release(); + await coordinator.shutdown(); + } }); it('advances the checkpoint only after a successful durable commit, including completion-failure re-entry', async () => { const directory = await mkdtemp(join(tmpdir(), 'noorg-usage-commit-')); const file = join(directory, 'state.json'); - const boundary = new StateBoundary(file, new NoOrgError('TRANSIENT_STATE', 'Controlled pre-commit failure', true)); + const boundary = new StateBoundary( + file, + new NoOrgError('TRANSIENT_STATE', 'Controlled pre-commit failure', true) + ); const state = new FileStateStore(file, boundary.io); const meter = offlineMeter([five, two]); let completions = 0; - const agent = new ChargedAgent('complete', () => { completions += 1; if (completions === 1) boundary.failNext('write'); }); + const agent = new ChargedAgent('complete', () => { + completions += 1; + if (completions === 1) boundary.failNext('write'); + }); const { coordinator } = usageRuntime(state, meter.provider, agent); try { await coordinator.start(); @@ -121,7 +175,9 @@ describe('Provider-instance usage deltas', () => { const repository = new TaskRepository(reopened, new SystemClock()); await repository.load(); expect(repository.get(task.id)?.usage).toEqual(seven); - } finally { await reopened.close(); } + } finally { + await reopened.close(); + } } finally { agent.release(); await coordinator.shutdown(); @@ -162,7 +218,10 @@ describe('Provider-instance usage deltas', () => { await repository.load(); const task = await repository.create(request); const meter = offlineMeter([five, two]); - await meter.provider.complete({ operation: 'prior-session', input: null, taskId: task.id }, output); + await meter.provider.complete( + { operation: 'prior-session', input: null, taskId: task.id }, + output + ); await repository.recordUsage(task.id, five); const agent = new ChargedAgent('complete'); const { coordinator } = usageRuntime(state, meter.provider, agent); @@ -171,27 +230,46 @@ describe('Provider-instance usage deltas', () => { await waitFor(() => coordinator.getTask(task.id)?.status === 'completed'); expect(coordinator.getTask(task.id)?.usage).toEqual(seven); expect(meter.effects.calls).toBe(2); - } finally { agent.release(); await coordinator.shutdown(); } - }); - - it.each(['cancel', 'timeout'] as const)('records late settled usage once after %s without publishing a result or releasing the agent early', async kind => { - const meter = offlineMeter([five, two]); - const agent = new ChargedAgent('late'); - const { coordinator, registry } = usageRuntime(new MemoryStateStore(), meter.provider, agent); - try { - await coordinator.start(); - const task = await coordinator.submitTask({ ...request, maxAttempts: 1, timeoutMs: kind === 'timeout' ? 100 : 2000 }); - await agent.ready.promise; - if (kind === 'cancel') await coordinator.cancelTask(task.id); - else await waitFor(() => coordinator.getTask(task.id)?.status === 'failed'); - if (kind === 'timeout') expect(coordinator.getTask(task.id)?.error?.code).toBe('TASK_TIMEOUT'); - expect(registry.activeCount()).toBe(1); + } finally { agent.release(); - await waitFor(() => coordinator.getTask(task.id)?.usage?.totalTokens === 7 && registry.activeCount() === 0); - expect(coordinator.getTask(task.id)?.usage).toEqual(seven); - expect(coordinator.getTask(task.id)?.status).toBe(kind === 'cancel' ? 'cancelled' : 'failed'); - expect(coordinator.getTask(task.id)?.result).toBeUndefined(); - expect(meter.effects.calls).toBe(2); - } finally { agent.release(); await coordinator.shutdown(); } + await coordinator.shutdown(); + } }); + + it.each(['cancel', 'timeout'] as const)( + 'records late settled usage once after %s without publishing a result or releasing the agent early', + async kind => { + const meter = offlineMeter([five, two]); + const agent = new ChargedAgent('late'); + const { coordinator, registry } = usageRuntime(new MemoryStateStore(), meter.provider, agent); + try { + await coordinator.start(); + const task = await coordinator.submitTask({ + ...request, + maxAttempts: 1, + timeoutMs: kind === 'timeout' ? 100 : 2000, + }); + await agent.ready.promise; + if (kind === 'cancel') await coordinator.cancelTask(task.id); + else await waitFor(() => coordinator.getTask(task.id)?.status === 'failed'); + if (kind === 'timeout') + expect(coordinator.getTask(task.id)?.error?.code).toBe('TASK_TIMEOUT'); + expect(registry.activeCount()).toBe(1); + agent.release(); + await waitFor( + () => + coordinator.getTask(task.id)?.usage?.totalTokens === 7 && registry.activeCount() === 0 + ); + expect(coordinator.getTask(task.id)?.usage).toEqual(seven); + expect(coordinator.getTask(task.id)?.status).toBe( + kind === 'cancel' ? 'cancelled' : 'failed' + ); + expect(coordinator.getTask(task.id)?.result).toBeUndefined(); + expect(meter.effects.calls).toBe(2); + } finally { + agent.release(); + await coordinator.shutdown(); + } + } + ); }); diff --git a/tests/unit/config-launch.test.ts b/tests/unit/config-launch.test.ts index e7a5bed..5f71255 100644 --- a/tests/unit/config-launch.test.ts +++ b/tests/unit/config-launch.test.ts @@ -20,23 +20,39 @@ describe('Explicit configuration admission', () => { }); it.each(['', ' ', '\t\n'])('treats optional blank values as absent (%j)', blank => { - expect(loadConfig({ - NOORG_LLM_PROVIDER: 'local', - OPENAI_API_KEY: blank, - OPENAI_BASE_URL: blank, - NOORG_HOST: blank, - NOORG_HTTP_AUTH_TOKEN: blank, - NOORG_PROVIDER_BUDGET_USD: blank, - })).toEqual(loadConfig({ NOORG_LLM_PROVIDER: 'local' })); + expect( + loadConfig({ + NOORG_LLM_PROVIDER: 'local', + OPENAI_API_KEY: blank, + OPENAI_BASE_URL: blank, + NOORG_HOST: blank, + NOORG_HTTP_AUTH_TOKEN: blank, + NOORG_PROVIDER_BUDGET_USD: blank, + }) + ).toEqual(loadConfig({ NOORG_LLM_PROVIDER: 'local' })); }); it('preserves nonempty validation and explicit provider/production-auth requirements', () => { expect(() => loadConfig({ OPENAI_API_KEY: 'unused-test-key' })).toThrow(); expect(() => loadConfig({ NOORG_LLM_PROVIDER: 'openai', OPENAI_API_KEY: ' ' })).toThrow(); - expect(() => loadConfig({ NOORG_LLM_PROVIDER: 'local', OPENAI_BASE_URL: 'not a URL' })).toThrow(); - expect(() => loadConfig({ NOORG_LLM_PROVIDER: 'local', NOORG_PROVIDER_BUDGET_USD: 'not a number' })).toThrow(); - expect(() => loadConfig({ NODE_ENV: 'production', NOORG_LLM_PROVIDER: 'local', NOORG_HTTP_AUTH_TOKEN: ' ' })).toThrow(); - const secured = loadConfig({ NODE_ENV: 'production', NOORG_LLM_PROVIDER: 'local', NOORG_HTTP_AUTH_TOKEN: 'test-secret' }); + expect(() => + loadConfig({ NOORG_LLM_PROVIDER: 'local', OPENAI_BASE_URL: 'not a URL' }) + ).toThrow(); + expect(() => + loadConfig({ NOORG_LLM_PROVIDER: 'local', NOORG_PROVIDER_BUDGET_USD: 'not a number' }) + ).toThrow(); + expect(() => + loadConfig({ + NODE_ENV: 'production', + NOORG_LLM_PROVIDER: 'local', + NOORG_HTTP_AUTH_TOKEN: ' ', + }) + ).toThrow(); + const secured = loadConfig({ + NODE_ENV: 'production', + NOORG_LLM_PROVIDER: 'local', + NOORG_HTTP_AUTH_TOKEN: 'test-secret', + }); expect(secured.server.authToken).toBe('test-secret'); expect(secured.server.host).toBe('0.0.0.0'); }); @@ -45,22 +61,38 @@ describe('Explicit configuration admission', () => { const directory = await mkdtemp(join(tmpdir(), 'noorg-env-file-')); const envFile = join(directory, '.env'); try { - await writeFile(envFile, 'NOORG_LLM_PROVIDER=local\nOPENAI_MODEL=file-model\nOPENAI_API_KEY=\nOPENAI_BASE_URL=\nNOORG_HTTP_AUTH_TOKEN=\nNOORG_PROVIDER_BUDGET_USD=\n'); + await writeFile( + envFile, + 'NOORG_LLM_PROVIDER=local\nOPENAI_MODEL=file-model\nOPENAI_API_KEY=\nOPENAI_BASE_URL=\nNOORG_HTTP_AUTH_TOKEN=\nNOORG_PROVIDER_BUDGET_USD=\n' + ); const configModule = require.resolve('../../src/config/config'); const script = `const { loadConfig } = require(${JSON.stringify(configModule)}); const config = loadConfig(process.env); process.stdout.write(JSON.stringify({ provider: config.provider, model: config.openai.model }));`; - const result = await runFile(process.execPath, ['--env-file=' + envFile, '--import', require.resolve('tsx'), '-e', script], { - cwd: directory, - env: { PATH: process.env.PATH, NODE_ENV: 'test', OPENAI_MODEL: 'inherited-model' }, - timeout: 5000, - }); - expect(z.object({ provider: z.literal('local'), model: z.string() }).parse(JSON.parse(result.stdout))).toEqual({ - provider: 'local', model: 'inherited-model', + const result = await runFile( + process.execPath, + ['--env-file=' + envFile, '--import', require.resolve('tsx'), '-e', script], + { + cwd: directory, + env: { PATH: process.env.PATH, NODE_ENV: 'test', OPENAI_MODEL: 'inherited-model' }, + timeout: 5000, + } + ); + expect( + z + .object({ provider: z.literal('local'), model: z.string() }) + .parse(JSON.parse(result.stdout)) + ).toEqual({ + provider: 'local', + model: 'inherited-model', }); - await expect(runFile(process.execPath, ['--import', require.resolve('tsx'), '-e', script], { - cwd: directory, - env: { PATH: process.env.PATH, NODE_ENV: 'test' }, - timeout: 5000, - })).rejects.toMatchObject({ code: 1 }); - } finally { await rm(directory, { recursive: true, force: true }); } + await expect( + runFile(process.execPath, ['--import', require.resolve('tsx'), '-e', script], { + cwd: directory, + env: { PATH: process.env.PATH, NODE_ENV: 'test' }, + timeout: 5000, + }) + ).rejects.toMatchObject({ code: 1 }); + } finally { + await rm(directory, { recursive: true, force: true }); + } }); }); diff --git a/tests/unit/provider-retention.test.ts b/tests/unit/provider-retention.test.ts index 7581a6c..2d4aa78 100644 --- a/tests/unit/provider-retention.test.ts +++ b/tests/unit/provider-retention.test.ts @@ -3,7 +3,11 @@ import { GovernedProvider } from '../../src/providers/governed-provider'; import { DeterministicProvider } from '../../src/providers/provider'; const schema = z.object({ value: z.number() }); -const request = { operation: 'local-retention', input: null, localDerivation: () => ({ value: 1 }) }; +const request = { + operation: 'local-retention', + input: null, + localDerivation: () => ({ value: 1 }), +}; const invalidRequest = { ...request, localDerivation: () => ({ value: Number.NaN }) }; describe('GovernedProvider rate-history ownership', () => { @@ -20,8 +24,15 @@ describe('GovernedProvider rate-history ownership', () => { // This is the retained-memory contract, not a timing/heap-size heuristic. const history = z.array(z.number()).parse(Reflect.get(provider, 'requestTimes')); expect(history).toEqual([]); - expect(provider.getUsage()).toEqual({ promptTokens: 0, completionTokens: 0, totalTokens: 0, costUsd: 0 }); - } finally { await provider.close(); } + expect(provider.getUsage()).toEqual({ + promptTokens: 0, + completionTokens: 0, + totalTokens: 0, + costUsd: 0, + }); + } finally { + await provider.close(); + } }); it('counts enabled failed attempts, rejects excess requests, and expires the exact rolling boundary', async () => { @@ -34,19 +45,27 @@ describe('GovernedProvider rate-history ownership', () => { try { await provider.complete(request, schema); await expect(provider.complete(invalidRequest, schema)).rejects.toThrow(); - await expect(provider.complete(request, schema)).rejects.toMatchObject({ code: 'PROVIDER_RATE_LIMITED' }); + await expect(provider.complete(request, schema)).rejects.toMatchObject({ + code: 'PROVIDER_RATE_LIMITED', + }); now = 59_999; - await expect(provider.complete(request, schema)).rejects.toMatchObject({ code: 'PROVIDER_RATE_LIMITED' }); + await expect(provider.complete(request, schema)).rejects.toMatchObject({ + code: 'PROVIDER_RATE_LIMITED', + }); now = 60_000; expect((await provider.complete(request, schema)).data).toEqual({ value: 1 }); for (let window = 0; window < 32; window += 1) { now += 60_000; await provider.complete(request, schema); await provider.complete(request, schema); - await expect(provider.complete(request, schema)).rejects.toMatchObject({ code: 'PROVIDER_RATE_LIMITED' }); + await expect(provider.complete(request, schema)).rejects.toMatchObject({ + code: 'PROVIDER_RATE_LIMITED', + }); const history = z.array(z.number()).parse(Reflect.get(provider, 'requestTimes')); expect(history).toEqual([now, now]); } - } finally { await provider.close(); } + } finally { + await provider.close(); + } }); });