fix(server): the mcp relay reaches its runtime again within seconds, and says why when it cannot

packages/server/npm/README.md

@@ -19,6 +19,8 @@ codex mcp add auto-runtime -- npx -y @on.auto/runtime@0.15.0 mcp
 
 When your agent connects, the command starts Auto Runtime in the background if it is not running yet, and relays your agent to it. The runtime it starts keeps running after your agent leaves, so the workflows it runs carry on and other agents reach the same one. It reads its settings from `~/.auto-runtime/.env`, where you put a model provider's key, such as `ANTHROPIC_API_KEY`, and writes its log to `~/.auto-runtime/runtime.log`. When no model provider is set up, your agent is told which file the key goes in and how to restart the runtime once it is there.
 
+If the runtime stops while your agent is connected, the next request starts it again within seconds, and is sent again unless it may already have run, in which case your agent is told to check before it repeats it. A runtime that stops answering is stopped and started again when the command started it; one you started yourself, or another program on the port, is left alone, and your agent is told what to stop. A runtime on the port that asks for a credential is left alone too, and your agent is told to stop it or connect over HTTP.
+
 ## Start it yourself
 
 ```bash

packages/server/src/local-runtime/agent-recovery.test.ts

@@ -0,0 +1,144 @@
+import { createServer } from 'node:http';
+
+import { createApiKey } from '@on.auto/identity';
+import { Function } from 'effect';
+import { describe, expect, it, onTestFinished } from 'vitest';
+
+import { ranToEnd, spawnRuntimeCommand } from '../testing/processes/runtime-command.ts';
+import {
+  answered,
+  answerWithId,
+  initializedLine,
+  initializeLine,
+  lineOf,
+  placeOfTheTest,
+  readiesIn,
+} from '../testing/relays/relayed-processes.ts';
+import { until } from '../testing/waiting/until.ts';
+import { processesOfThisComputer } from './background-process.ts';
+import { restartWaitMs } from './background-start.ts';
+
+const homePrefix = 'auto-runtime-recovery-';
+
+const listPlaybooks = {
+  jsonrpc: '2.0',
+  id: 2,
+  method: 'tools/call',
+  params: { name: 'list_playbooks', arguments: {} },
+};
+
+const listTools = { jsonrpc: '2.0', id: 2, method: 'tools/list', params: {} };
+
+describe('an agent whose runtime in a terminal is stopped', () => {
+  it('has its next tool call answered by a runtime the relay starts, without waiting as long as a restart can take', async () => {
+    const { home, port } = await placeOfTheTest(homePrefix);
+    const terminal = spawnRuntimeCommand([], { home, port });
+    await until(() => {
+      expect(readiesIn(terminal.stdout())).toBe(1);
+    });
+    const relay = spawnRuntimeCommand(['mcp'], { home, port });
+    relay.write(lineOf(initializeLine));
+    await answered(relay, 1);
+    relay.write(lineOf(initializedLine));
+    terminal.interrupt();
+    await terminal.exited;
+
+    const asked = performance.now();
+    relay.write(lineOf(listPlaybooks));
+    const answers = await answered(relay, 2);
+    const tookMs = performance.now() - asked;
+    const status = await ranToEnd(['status'], { home, port });
+    relay.end();
+
+    expect(answerWithId(answers, 2)).toMatchObject({ id: 2, result: { structuredContent: { playbooks: [] } } });
+    expect(tookMs).toBeLessThan(restartWaitMs);
+    expect(status.stdout).toContain('by your agent');
+    expect(await relay.exited).toBe(0);
+  });
+});
+
+describe('an agent whose runtime, which it started, stops answering', () => {
+  it('has the runtime stopped and another started, and its request answered by that one', async () => {
+    const { home, port, runtimes } = await placeOfTheTest(homePrefix);
+    const relay = spawnRuntimeCommand(['mcp'], { home, port });
+    relay.write(lineOf(initializeLine));
+    await answered(relay, 1);
+    relay.write(lineOf(initializedLine));
+    const frozen = await runtimes.pid();
+    onTestFinished(() => {
+      processesOfThisComputer.signal(frozen, 'SIGKILL');
+    });
+    process.kill(frozen, 'SIGSTOP');
+
+    relay.write(lineOf(listTools));
+    const answers = await answered(relay, 2);
+    const replacing = await runtimes.pid();
+    relay.end();
+
+    expect(JSON.stringify(answerWithId(answers, 2))).toContain('"name":"list_playbooks"');
+    expect(replacing).not.toBe(frozen);
+    expect(relay.stderr()).toContain(`process ${frozen}, did not answer, so it was stopped and another started`);
+    expect(await relay.exited).toBe(0);
+  });
+});
+
+describe('an agent whose runtime port a program holds that never answers', () => {
+  it('is told so, and that it can stop that program or set PORT, and nothing is started', async () => {
+    const { home, port } = await placeOfTheTest(homePrefix);
+    const silent = createServer(Function.constVoid).listen(port, '127.0.0.1');
+    onTestFinished(() => {
+      silent.closeAllConnections();
+      silent.close();
+    });
+    const relay = spawnRuntimeCommand(['mcp'], { home, port });
+
+    relay.write(lineOf(initializeLine));
+    const answers = await answered(relay, 1);
+    relay.end();
+
+    expect(answers).toEqual([
+      {
+        jsonrpc: '2.0',
+        id: 1,
+        error: {
+          code: -32_603,
+          message: `The program on port ${port} does not answer; stop it, or set PORT to a free port, and this MCP entry starts Auto Runtime`,
+        },
+      },
+    ]);
+    expect(await relay.exited).toBe(0);
+  });
+});
+
+describe('an agent whose runtime asks for a credential', () => {
+  it('is told that this entry sends none, and how to reach it instead', async () => {
+    const { home, port } = await placeOfTheTest(homePrefix);
+    const { entry } = createApiKey({ id: 'ci-1', org: 'acme', permissions: ['org:read'], playbooks: '*' });
+    const guarded = spawnRuntimeCommand([], {
+      home,
+      port,
+      settings: { HOST: '0.0.0.0', API_KEYS: JSON.stringify([entry]) },
+    });
+    await until(() => {
+      expect(readiesIn(guarded.stdout())).toBe(1);
+    });
+    const relay = spawnRuntimeCommand(['mcp'], { home, port });
+
+    relay.write(lineOf(initializeLine));
+    const answers = await answered(relay, 1);
+    relay.end();
+    guarded.interrupt();
+
+    expect(answers).toEqual([
+      {
+        jsonrpc: '2.0',
+        id: 1,
+        error: {
+          code: -32_603,
+          message: `Auto Runtime on port ${port} asks for a credential, which this MCP entry does not send; stop it, and this entry starts one, or connect your agent to http://localhost:${port}/mcp over HTTP`,
+        },
+      },
+    ]);
+    expect([await relay.exited, await guarded.exited]).toEqual([0, 0]);
+  });
+});

packages/server/src/local-runtime/agent-relay.test.ts

@@ -13,8 +13,15 @@ import {
   runtimeOnThePortEndedWithTheTest,
   spawnRuntimeCommand,
   type RuntimeCommandRun,
-  type RuntimeCommandPlace,
 } from '../testing/processes/runtime-command.ts';
+import {
+  answered,
+  answerWithId,
+  initializeLine,
+  lineOf,
+  placeOfTheTest,
+  readiesIn,
+} from '../testing/relays/relayed-processes.ts';
 import { until } from '../testing/waiting/until.ts';
 
 const homePrefix = 'auto-runtime-agent-';
@@ -25,53 +32,13 @@ async function statusAt(port: number): Promise<ReturnType<typeof statusOf>> {
   return statusOf(await (await fetch(`http://127.0.0.1:${port}/runtime`)).json());
 }
 
-async function placeOfTheTest(): Promise<RuntimeCommandPlace> {
-  const home = homeOfTheTest(homePrefix);
-  const port = await aPortNothingListensOn();
-  runtimeOnThePortEndedWithTheTest(home, port);
-  return { home, port };
-}
-
-const initialize = {
-  jsonrpc: '2.0',
-  id: 1,
-  method: 'initialize',
-  params: { protocolVersion: '2025-11-25', capabilities: {}, clientInfo: { name: 'agent', version: '1.0.0' } },
-};
-
-function answersIn(stdout: string): readonly unknown[] {
-  return stdout
-    .split('\n')
-    .filter((line) => line !== '')
-    .map((line): unknown => JSON.parse(line));
-}
-
-function answerWithId(answers: readonly unknown[], id: number): unknown {
-  return answers.find((answer) => typeof answer === 'object' && answer !== null && 'id' in answer && answer.id === id);
-}
-
-function lineOf(message: object): string {
-  return `${JSON.stringify(message)}\n`;
-}
-
-async function answered(relay: Readonly<RuntimeCommandRun>, count: number): Promise<readonly unknown[]> {
-  await until(() => {
-    expect(answersIn(relay.stdout())).toHaveLength(count);
-  });
-  return answersIn(relay.stdout());
-}
-
-function readiesIn(text: string): number {
-  return text.split('Press Ctrl+C to stop.').length - 1;
-}
-
 function markedAsRestartingBy(home: string, pid: number): void {
   mkdirSync(join(home, '.auto-runtime'), { recursive: true });
   writeFileSync(join(home, '.auto-runtime', 'restart-mark.json'), JSON.stringify({ pid, at: Date.now() }));
 }
 
 async function leftAfterItsAgentLeft(relay: Readonly<RuntimeCommandRun>): Promise<unknown> {
-  relay.write(lineOf(initialize));
+  relay.write(lineOf(initializeLine));
   await answered(relay, 1);
   relay.end();
   return relay.exited;
@@ -86,7 +53,7 @@ describe('an agent that starts auto-runtime mcp', () => {
     const starting = spawnRuntimeCommand(['mcp'], { home, port, inAGroupOfItsOwn: true });
     const pid = await runtimes.pid();
 
-    starting.write(lineOf(initialize));
+    starting.write(lineOf(initializeLine));
     await answered(starting, 1);
     starting.write(lineOf({ jsonrpc: '2.0', method: 'notifications/initialized' }));
     starting.write(lineOf({ jsonrpc: '2.0', id: 2, method: 'tools/list', params: {} }));
@@ -126,13 +93,13 @@ describe('an agent that starts auto-runtime mcp', () => {
 
 describe('an agent whose runtime restarts', () => {
   it('has a request it sends while the runtime is down sent again once the runtime is back', async () => {
-    const { home, port } = await placeOfTheTest();
+    const { home, port } = await placeOfTheTest(homePrefix);
     const runtime = spawnRuntimeCommand([], { home, port });
     await until(() => {
       expect(readiesIn(runtime.stdout())).toBe(1);
     });
     const relay = spawnRuntimeCommand(['mcp'], { home, port });
-    relay.write(lineOf(initialize));
+    relay.write(lineOf(initializeLine));
     await answered(relay, 1);
     relay.write(lineOf({ jsonrpc: '2.0', method: 'notifications/initialized' }));
 
@@ -151,10 +118,10 @@ describe('an agent whose runtime restarts', () => {
 
 describe('an agent that starts while a runtime restarts', () => {
   it('waits for that runtime to come back instead of starting a second one', async () => {
-    const { home, port } = await placeOfTheTest();
+    const { home, port } = await placeOfTheTest(homePrefix);
     markedAsRestartingBy(home, process.pid);
     const relay = spawnRuntimeCommand(['mcp'], { home, port });
-    relay.write(lineOf(initialize));
+    relay.write(lineOf(initializeLine));
 
     const runtime = spawnRuntimeCommand([], { home, port });
     await answered(relay, 1);
@@ -170,7 +137,7 @@ describe('an agent that starts while a runtime restarts', () => {
 
 describe('an agent whose runtime port another program holds', () => {
   it('is told why it cannot reach the runtime, and answered once with what its request met', async () => {
-    const { home, port } = await placeOfTheTest();
+    const { home, port } = await placeOfTheTest(homePrefix);
     const other = createServer((_request, response) => {
       response.end('hello');
     }).listen(port, '127.0.0.1');
@@ -179,7 +146,7 @@ describe('an agent whose runtime port another program holds', () => {
     });
     const relay = spawnRuntimeCommand(['mcp'], { home, port });
 
-    relay.write(lineOf(initialize));
+    relay.write(lineOf(initializeLine));
     const answers = await answered(relay, 1);
     relay.end();
 

packages/server/src/local-runtime/agent-relay.ts

@@ -5,11 +5,17 @@ import { StdioServerTransport } from '@modelcontextprotocol/server/stdio';
 import type { Environment } from '@on.auto/config';
 
 import { release } from '../composition/release.ts';
-import { noticesFor, restartMarkIn, startInBackgroundOf, startingLockIn } from './background-process.ts';
-import { impatient, runningRuntime, throughARestart } from './background-start.ts';
+import {
+  noticesFor,
+  processesOfThisComputer,
+  restartMarkIn,
+  startInBackgroundOf,
+  startingLockIn,
+} from './background-process.ts';
+import { briefly, impatient, runningRuntime, sightingsOfARelay } from './background-start.ts';
 import { logFileIn } from './local-control.ts';
-import { relay } from './mcp-relay.ts';
-import { runtimeAddressOf, type Waiting } from './runtime-probe.ts';
+import { relay, type Timer } from './mcp-relay.ts';
+import { isAnswering, probeAt, runtimeAddressOf, type Waiting } from './runtime-probe.ts';
 
 export interface AgentRelaying {
   readonly home: string;
@@ -21,6 +27,14 @@ function report(line: string): void {
   process.stderr.write(`${line}\n`);
 }
 
+const timer: Timer = (milliseconds, then) => {
+  const timeout = setTimeout(then, milliseconds);
+  timeout.unref();
+  return () => {
+    clearTimeout(timeout);
+  };
+};
+
 export async function relayedAgent(port: number, { home, environment, waiting }: AgentRelaying): Promise<void> {
   const starting = {
     ...waiting,
@@ -29,6 +43,8 @@ export async function relayedAgent(port: number, { home, environment, waiting }:
     startingLock: startingLockIn(home),
     notice: noticesFor(home, report),
     restartUnderWay: restartMarkIn(home).isUnderWay,
+    processes: processesOfThisComputer,
+    sightings: sightingsOfARelay(),
     startInBackground: startInBackgroundOf({
       execPath: process.execPath,
       execArgv: process.execArgv,
@@ -45,7 +61,9 @@ export async function relayedAgent(port: number, { home, environment, waiting }:
   await relay({
     agent: () => new StdioServerTransport(),
     runtime: () => new StreamableHTTPClientTransport(new URL(`${runtimeAddressOf(port)}/mcp`)),
-    recover: () => runningRuntime(port, starting, throughARestart),
+    recover: () => runningRuntime(port, starting, briefly),
+    answering: async () => isAnswering(await probeAt(port, waiting.fetch)),
+    timer,
     report,
   });
 }

packages/server/src/local-runtime/background-process.test.ts

@@ -1,4 +1,4 @@
-import { spawnSync } from 'node:child_process';
+import { spawn, spawnSync } from 'node:child_process';
 import {
   chmodSync,
   existsSync,
@@ -18,10 +18,16 @@ import { join } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 import { Function } from 'effect';
-import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+import { afterEach, beforeEach, describe, expect, it, onTestFinished } from 'vitest';
 
 import { until } from '../testing/waiting/until.ts';
-import { noticesFor, restartMarkIn, startInBackgroundOf, startingLockIn } from './background-process.ts';
+import {
+  noticesFor,
+  processesOfThisComputer,
+  restartMarkIn,
+  startInBackgroundOf,
+  startingLockIn,
+} from './background-process.ts';
 import { restartWaitMs, startWaitMs } from './background-start.ts';
 
 const recordingEntry = fileURLToPath(new URL('../testing/entries/record-background-start.ts', import.meta.url));
@@ -225,3 +231,21 @@ describe('the mark a runtime leaves while it restarts', () => {
     expect(restartMarkIn(home).leave(process.pid)).toBe(false);
   });
 });
+
+describe('the processes of this computer', () => {
+  it('signals one that runs, which is then gone, and signals none once it is gone', async () => {
+    const child = spawn(process.execPath, ['-e', 'setInterval(() => {}, 1000)'], { stdio: 'ignore' });
+    onTestFinished(() => {
+      child.kill('SIGKILL');
+    });
+    const running = Number(child.pid);
+
+    const aliveAtFirst = processesOfThisComputer.alive(running);
+    const killed = processesOfThisComputer.signal(running, 'SIGKILL');
+    await until(() => {
+      expect(processesOfThisComputer.alive(running)).toBe(false);
+    });
+
+    expect([aliveAtFirst, killed, processesOfThisComputer.signal(running, 'SIGTERM')]).toEqual([true, true, false]);
+  });
+});