From fdf3941739e110d37fd8d1e1e5137f26ebdd926f Mon Sep 17 00:00:00 2001 From: Andre Brait Date: Sat, 14 Nov 2026 08:19:51 +0000 Subject: [PATCH 1/2] fix(hosts): preserve maintenance across protocol upgrades Introduce independent maintenance API v1 alongside application Protocol v12. Welcome v10 and v11 daemons through an update-only shim using their unchanged updater contract, including shared-install restart-only descriptors. Preserve native update safety and verify real historical daemon upgrades. --- CHANGELOG.md | 10 + README.md | 24 +- app/api/hosts/[id]/maintenance/route.ts | 16 + app/api/hosts/maintenance/route.ts | 8 + components/HostUpdates.test.mjs | 83 ++++ components/HostUpdates.tsx | 5 +- docs/specs/2026-10-28-remote-hosts-design.md | 72 ++-- lib/hosts/daemon-channels.ts | 6 + lib/hosts/daemon-link.ts | 12 +- lib/hosts/daemon-links.test.mjs | 18 +- lib/hosts/daemon-maintenance.ts | 237 ++++++++++++ lib/hosts/daemon.test.mjs | 13 +- lib/hosts/daemon.ts | 53 ++- lib/hosts/hosts-route.test.mjs | 2 +- lib/hosts/hub.test.mjs | 4 +- lib/hosts/hub.ts | 102 ++++- lib/hosts/link-protocol.ts | 9 +- .../maintenance-daemon.integration.test.mjs | 145 +++++++ lib/hosts/maintenance-hub.test.mjs | 291 ++++++++++++++ lib/hosts/maintenance-hub.ts | 298 ++++++++++++++ lib/hosts/maintenance-protocol.test.mjs | 36 ++ lib/hosts/maintenance-protocol.ts | 89 +++++ lib/hosts/maintenance-transport.test.mjs | 276 +++++++++++++ lib/hosts/update-worker.test.mjs | 25 ++ lib/hosts/update-worker.ts | 8 +- lib/hosts/upgrade-shim.test.mjs | 109 ++++++ lib/hosts/v10-upgrade.integration.test.mjs | 362 ++++++++++++++++++ package-lock.json | 4 +- package.json | 2 +- proxy.ts | 3 +- server/app.ts | 22 +- server/routes.ts | 9 +- 32 files changed, 2262 insertions(+), 91 deletions(-) create mode 100644 app/api/hosts/[id]/maintenance/route.ts create mode 100644 app/api/hosts/maintenance/route.ts create mode 100644 components/HostUpdates.test.mjs create mode 100644 lib/hosts/daemon-maintenance.ts create mode 100644 lib/hosts/maintenance-daemon.integration.test.mjs create mode 100644 lib/hosts/maintenance-hub.test.mjs create mode 100644 lib/hosts/maintenance-hub.ts create mode 100644 lib/hosts/maintenance-protocol.test.mjs create mode 100644 lib/hosts/maintenance-protocol.ts create mode 100644 lib/hosts/maintenance-transport.test.mjs create mode 100644 lib/hosts/upgrade-shim.test.mjs create mode 100644 lib/hosts/v10-upgrade.integration.test.mjs diff --git a/CHANGELOG.md b/CHANGELOG.md index 51f48852f..eaa3b0f03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ All notable changes to **omp-web** (`@kahme247/ompweb`) are documented in this f ## [Unreleased] +## [v0.7.1] - 2026-11-14 + +### Fixed + +- **Upgrade v10 and v11 hosts from Settings again.** Update the web server first, then use **Settings → Hosts → Update**. An isolated shim welcomes old daemons as their own version and speaks only their existing update contract; chat, workspace operations and old session frames remain disabled. Native busy-host, preflight and restart safeguards are unchanged. Shared installations use the existing restart-only update after the server's files are replaced. An old daemon that already exited needs starting once. + +### Added + +- **Maintenance API v1, independent of application Protocol v12.** A separate host-token-authenticated connection provides status, session inventory and activity, capabilities and software updates even when application protocols differ. Session traffic uses Protocol v12; future application incompatibility no longer needs to disconnect maintenance. Reverse proxies must forward `/api/hosts/maintenance` WebSocket upgrades as well as `/api/hosts/link`. + ## [v0.7.0] - 2026-11-14 ### Breaking changes diff --git a/README.md b/README.md index b9f241ae0..27e6bb596 100644 --- a/README.md +++ b/README.md @@ -698,10 +698,26 @@ protocol is blocked too. omp is part of each ompweb release and updates with it. Before updating a daemon, close its session processes: the updater refuses to interrupt sessions or subagents. **Settings → System** keeps the web server's own application update and service settings. -Hosts need a daemon build that supports these update operations; older builds -show an unsupported message and need one manual update first. If the daemon -shares its installation with the web server, update the server first, then use -the host's restart-only update. Unsupported service wrappers and detected restart + +**From 0.7.1 onward, maintenance API v1 is separate from application Protocol v12.** +The daemon connects to `/api/hosts/maintenance` with the same host token; status, +session inventory and activity, capabilities, update checks and installation do not +require a compatible application connection. An application mismatch still blocks +chat, workspace operations and session attachment, not maintenance. + +**Upgrading v10 and v11 daemons:** update the web server to 0.7.1 first, then use +**Settings → Hosts → Update**. An upgrade-only shim welcomes each old daemon using +its own protocol and speaks only its existing updater's requests. It never adopts +or stops its sessions, and the old updater still refuses a busy host. Close those +session processes before the cutover. If server and daemon share an installation, +the server update puts the new files in place and the host update only restarts +the daemon. A daemon that already exited after an earlier version rejection needs +starting once; an exited process cannot be reached by the shim. + +Reverse proxies must forward WebSocket upgrades for `/api/hosts/maintenance` +without an additional authentication prompt, just as for `/api/hosts/link`. +Daemons below v10 or without the existing update operations need one manual update. +Unsupported service wrappers and detected restart scripts show a manual-update reason. Stop external restart loops before updating a foreground daemon. diff --git a/app/api/hosts/[id]/maintenance/route.ts b/app/api/hosts/[id]/maintenance/route.ts new file mode 100644 index 000000000..7a5dcf114 --- /dev/null +++ b/app/api/hosts/[id]/maintenance/route.ts @@ -0,0 +1,16 @@ +import { getHost } from "@/lib/hosts/hub"; +import { MaintenanceError, requestMaintenance } from "@/lib/hosts/maintenance-hub"; + +/** Browser access stays behind the web gate; no application connection is needed. */ +export async function GET(request: Request, { params }: { params: Promise<{ id: string }> }) { + const { id } = await params; + const headers = { "cache-control": "no-store" }; + if (!getHost(id)) return Response.json({ error: "Unknown host", code: "host_not_found" }, { status: 404, headers }); + try { + const op = new URL(request.url).searchParams.get("sessions") === "1" ? "listSessions" : "status"; + return Response.json(await requestMaintenance(id, op, {}), { headers }); + } catch (error) { + if (error instanceof MaintenanceError) return Response.json({ error: error.message, code: error.code }, { status: error.status, headers }); + return Response.json({ error: "The host maintenance request failed", code: "maintenance_failed" }, { status: 500, headers }); + } +} diff --git a/app/api/hosts/maintenance/route.ts b/app/api/hosts/maintenance/route.ts new file mode 100644 index 000000000..b3174f4b6 --- /dev/null +++ b/app/api/hosts/maintenance/route.ts @@ -0,0 +1,8 @@ +import { handleMaintenanceUpgrade, type MaintenanceSocketData } from "@/lib/hosts/maintenance-hub"; +import { LINK_PROTOCOL_VERSION } from "@/lib/hosts/link-protocol"; +import packageJson from "@/package.json"; + +/** API v1 discovery and its independent host-token-authenticated WebSocket. */ +export function GET(request: Request, { upgrade }: { upgrade: (data: MaintenanceSocketData) => boolean }) { + return handleMaintenanceUpgrade(request, upgrade, { protocolVersion: LINK_PROTOCOL_VERSION, ompwebVersion: packageJson.version }); +} diff --git a/components/HostUpdates.test.mjs b/components/HostUpdates.test.mjs new file mode 100644 index 000000000..43ee3f85b --- /dev/null +++ b/components/HostUpdates.test.mjs @@ -0,0 +1,83 @@ +import "../tests/setup-dom.mjs"; +import assert from "node:assert/strict"; +import test from "node:test"; +import { act, createElement } from "react"; +import { createRoot } from "react-dom/client"; +import { HostUpdates } from "./HostUpdates"; + +const endpoint = "/api/hosts/box/updates"; +const info = { + ompweb: { currentVersion: "0.7.0", availableVersion: "0.7.1", updateAvailable: true, canUpdate: true, blockedReason: null, updateCommand: "npm install -g @kahme247/ompweb", installMethod: "npm", protocolVersion: 12 }, + omp: { currentVersion: "18.20.0" }, + operation: null, +}; + +/** Render the real host row and record every request it sends. */ +async function renderHost(t, host) { + const requests = []; + const originalFetch = globalThis.fetch; + globalThis.fetch = async (url, init) => { + requests.push({ url: String(url), method: init?.method ?? "GET", body: init?.body }); + if (init?.method === "POST") return Response.json({ accepted: true, operation: { state: "running", target: "ompweb", version: "0.7.1" } }); + return Response.json(info); + }; + const container = document.createElement("div"); + document.body.append(container); + const root = createRoot(container); + t.after(() => { + act(() => root.unmount()); + container.remove(); + globalThis.fetch = originalFetch; + }); + await act(async () => { + root.render(createElement(HostUpdates, { host })); + }); + await act(() => new Promise((resolve) => setTimeout(resolve, 20))); + return { container, requests }; +} + +const button = (root, label) => [...root.querySelectorAll("button")].find((candidate) => candidate.textContent.trim() === label); + +test("an upgrade-only legacy host can check for and submit an ompweb update", async (t) => { + const { container, requests } = await renderHost(t, { id: "box", label: "Box", status: "needs-upgrade", upgradeOnly: true, ompVersion: "omp/18.20.0", ompwebVersion: "0.7.0" }); + assert.equal(requests[0]?.url, endpoint, "the status is read on mount"); + assert.equal(button(container, "Check updates").disabled, false); + + await act(async () => button(container, "Check updates").click()); + await act(() => new Promise((resolve) => setTimeout(resolve, 20))); + assert.ok(requests.some((request) => request.url === `${endpoint}?force=1` && request.method === "GET"), requests.map((request) => request.url).join(", ")); + + act(() => button(container, "Update").click()); + const confirm = [...document.querySelectorAll(".dialog-actions button")].find((candidate) => candidate.textContent.trim() === "Update"); + assert.ok(confirm, "the update confirmation did not open"); + await act(async () => confirm.click()); + await act(() => new Promise((resolve) => setTimeout(resolve, 20))); + const post = requests.find((request) => request.method === "POST"); + assert.ok(post, "the update was not submitted"); + assert.deepEqual(JSON.parse(post.body), { target: "ompweb", version: "0.7.1" }); +}); + +test("a connected maintenance API can update a compatible host while its application connection is offline", async (t) => { + const { container, requests } = await renderHost(t, { id: "box", label: "Box", status: "offline", apiVersion: 1, hostVersion: 12, ompwebVersion: "0.7.0" }); + assert.equal(requests[0]?.url, endpoint); + assert.equal(button(container, "Check updates").disabled, false); + assert.ok(button(container, "Update")); +}); + +test("an ordinary incompatible host cannot be checked or updated", async (t) => { + const { container, requests } = await renderHost(t, { id: "box", label: "Box", status: "needs-upgrade", ompVersion: "omp/18.20.0", ompwebVersion: "0.6.0" }); + assert.deepEqual(requests, [], "an unavailable host must not be read"); + const check = button(container, "Check updates"); + assert.equal(check.disabled, true); + act(() => check.click()); + assert.equal(button(container, "Update"), undefined); + assert.ok(container.textContent.includes("This host daemon does not support updates"), container.textContent); + assert.deepEqual(requests, []); +}); + +test("an offline host stays unavailable", async (t) => { + const { container, requests } = await renderHost(t, { id: "box", label: "Box", status: "offline", ompVersion: "omp/18.20.0", ompwebVersion: "0.7.0" }); + assert.deepEqual(requests, []); + assert.equal(button(container, "Check updates").disabled, true); + assert.ok(container.textContent.includes("Updates unavailable while this host is offline."), container.textContent); +}); diff --git a/components/HostUpdates.tsx b/components/HostUpdates.tsx index ed2fd5861..ef7eb4e1d 100644 --- a/components/HostUpdates.tsx +++ b/components/HostUpdates.tsx @@ -30,7 +30,8 @@ export function HostUpdates({ host }: { host: HostRow }) { const mounted = useRef(true); const postController = useRef(null); const endpoint = `/api/hosts/${encodeURIComponent(host.id)}/updates`; - const unavailable = host.status === "offline" || host.status === "needs-upgrade"; + const upgradeOnly = host.status === "needs-upgrade" && host.upgradeOnly === true; + const unavailable = (host.status === "offline" || host.status === "needs-upgrade") && !upgradeOnly && host.apiVersion !== 1; useEffect(() => { mounted.current = true; @@ -164,7 +165,7 @@ export function HostUpdates({ host }: { host: HostRow }) { const busy = loading || posting || monitoring; const disabled = busy || unavailable || unsupported; - const unavailableReason = host.status === "needs-upgrade" || unsupported ? t("hostUpdates.unsupported") : host.status === "offline" ? t("hostUpdates.offline") : null; + const unavailableReason = (host.status === "needs-upgrade" && unavailable) || unsupported ? t("hostUpdates.unsupported") : host.status === "offline" && unavailable ? t("hostUpdates.offline") : null; const operation = info?.operation; return (
diff --git a/docs/specs/2026-10-28-remote-hosts-design.md b/docs/specs/2026-10-28-remote-hosts-design.md index 927755b17..280937c6f 100644 --- a/docs/specs/2026-10-28-remote-hosts-design.md +++ b/docs/specs/2026-10-28-remote-hosts-design.md @@ -315,15 +315,15 @@ that upload's file. ops; 9: `open.launch.shareCredentials`; 10: `open.launch.configVersion`, `config_state`, the `clusterConfig` op; 11: channel lines are the omp runner's `{t}` frames, not omp RPC frames). Main links daemons from - `MIN_LINK_PROTOCOL_VERSION` (11) up to its own `LINK_PROTOCOL_VERSION` (11); - its `welcome` carries the version the link speaks (see "Protocol versions and - compatibility"). A hello whose `v` is outside that range (checked on the raw - JSON before decoding) gets close 4426 with reason `{"serverVersion":11}`; the - daemon exits 1, telling which side to - update ("update ompweb on this host", or "this host's ompweb is newer than - the server's: update ompweb on the server" when `serverVersion` is below its - own); Settings → Hosts shows "needs upgrade (host vN, server vM)" unless the - host has a healthy socket. + `MIN_LINK_PROTOCOL_VERSION` (12) up to its own `LINK_PROTOCOL_VERSION` (12). + A v10 or v11 daemon is welcomed with its own version by an upgrade-only shim, + outside that application range. The shim only sends `checkUpdates` and + `updateSoftware`; it acknowledges and discards application activity, never + adopts or kills sessions, and never starts cluster configuration synchronization. + All other incompatible versions get close 4426 with `{"serverVersion":12}`. + New daemons retain their separate maintenance API connection while an + application mismatch is retried. Old daemons may exit on rejection; an + already-exited daemon needs starting once. - **Identity.** The host is the record whose `tokenHash` matches. Nothing the daemon sends selects or changes its identity. - **One link per host.** A hello on a new socket with the same token closes the @@ -404,21 +404,15 @@ the `spawn` op are gone). message being understood unless the version (or, later, a capability) says so. Tolerance starts with v3: a daemon older than that is strict (an unknown type or op makes it refuse the line). -- **Older versions stay supported** from the next version on, until supporting - one becomes too much of a maintenance burden; dropping one is an explicit - decision (raising `MIN_LINK_PROTOCOL_VERSION`, which is 11 today). In - practice main keeps at least version N-1. For an older daemon main decodes - its hello into today's shape and sends only messages that version - understands (to a v4 daemon: no `part`, every line whole, and no unacked - window, since it acks only inside its next uplink frame). - A daemon on a dropped version gets 426 - (a POST) or close 4426 (a hello frame). Dropping versions ends with the - decision below. -- **Upgrade order: web app first, then hosts.** Main supports older daemons; a - daemon is not required to support a newer main, and one newer than main gets - close 4426. - `hostVersion` (Settings → Hosts) is the version a linked daemon speaks, - so Settings can later show "update available" for one behind the server. +- **Application compatibility is separate from maintenance compatibility.** + Dropping an application version raises `MIN_LINK_PROTOCOL_VERSION` without + dropping maintenance API v1. The current application range is 12–12. + Legacy v10/v11 are admitted only by the update shim, not the session runtime. +- **Upgrade order: web app first, then hosts.** The shim passes the new server's + application range to the old daemon's unchanged updater, while its `welcome` + echoes the old version. The updater can install Protocol v12 without needing + to understand it. Existing safety checks still refuse busy hosts and unsafe + supervisors; a shared installation uses restart-only after updating the server. - **v4 (the WebSocket link) dropped v3** before any release; from the next version on, main keeps N-1. A v3 daemon POSTs its hello to `/api/hosts/link`; main answers that POST 426 with both versions (and never @@ -443,16 +437,26 @@ the `spawn` op are gone). a channel; both sides refuse one with 426 / close 4426 instead of hanging. Prerelease builds already shipped v10 with omp RPC frames, so the runner could not reuse 10. -- **Decided: backward compatibility becomes enforced once the WebSocket - experiments end.** Until then (the browser transport experiment and the - host-link measurements), versions may still be dropped as above. Call N the - first link protocol version released after them. From N on the guarantees - are enforced, so the web app can always update an old host from Settings → - Hosts: the handshake (`hello`/`welcome` and the version close codes), the - version report, and an update op form a fixed core that every later web app - keeps working with every daemon on N or newer. `MIN_LINK_PROTOCOL_VERSION` - never goes above N. This replaces the update-only mode that was planned - for dropped versions. +- **Maintenance API v1 is a separate, stable contract.** New daemons discover + `/api/hosts/maintenance` with the same bearer token, then open a separate + WebSocket. Its `hello`/`welcome` negotiate `apiVersion: 1` independently of the + advisory `protocolVersion`. Its codec and transport do not import the + application protocol. A protocol mismatch cannot terminate this connection. + Requests have bounded IDs and explicit replies; disconnects and timeouts never + cause automatic replay of an update. The daemon remains authoritative for + whether it is safe to update. + - `status`: software/API/protocol versions, capabilities, workspace-operation + availability, session-process inventory and activity, drained/busy state and + the durable update operation. + - `listSessions`: session metadata, including `processRunning` and `active`; + no transcript or application frame decoding. + - `checkUpdates` and `updateSoftware`: the native updater and its safeguards. + - The web-authenticated `GET /api/hosts/[id]/maintenance` exposes status; + `?sessions=1` exposes inventory. Existing host update routes prefer API v1 + when available, otherwise use the legacy shim. + - Revoke and shutdown close both connections. API v1 remains supported across + future application-protocol changes; incompatible maintenance changes need + a separately negotiated API version. ## Host ops (this sub-project) diff --git a/lib/hosts/daemon-channels.ts b/lib/hosts/daemon-channels.ts index ba0f3e28d..003735354 100644 --- a/lib/hosts/daemon-channels.ts +++ b/lib/hosts/daemon-channels.ts @@ -81,6 +81,12 @@ export class ChannelTable { for (const channel of this.channels.values()) if (channel.lifecycle?.activeTurn ?? channel.pending.activeTurn) active++; return active; } + maintenanceSessions(): { id: string | null; cwd: string; active: boolean }[] { + return [...this.channels.values()].map(channel => ({ + id: channel.pending.sessionId, cwd: channel.cwd, + active: (channel.lifecycle?.isRunning ?? channel.pending.activeTurn) || channel.resuming === true, + })); + } cwdForSession(id: string): string | undefined { for (const channel of this.channels.values()) if (channel.pending.sessionId === id) return channel.cwd; return undefined; diff --git a/lib/hosts/daemon-link.ts b/lib/hosts/daemon-link.ts index 241dfe3d7..e335bcfd8 100644 --- a/lib/hosts/daemon-link.ts +++ b/lib/hosts/daemon-link.ts @@ -54,6 +54,8 @@ export function dialable(url: string, allowHttp: boolean): boolean { } export type LinkDeps = { allowHttp: boolean; primary: boolean; + /** Maintenance has its own lifecycle, so an application mismatch must not end this host. */ + keepOnVersionMismatch?: boolean; hello(link: DaemonLink, replay: boolean): string; handleDown(link: DaemonLink, message: DownMessage | UnknownMessage): void; onEnd(link: DaemonLink, reason: "revoked" | "replaced" | "version"): void; @@ -89,6 +91,7 @@ function linkRuntime(owner: DaemonLink, deps: LinkDeps) { const linkUrl = serverUrl + "/api/hosts/link"; let authorization = "Bearer " + owner.login.token; const log = deps.log; + let mismatched = false; const channels = new Map(); // Uplink messages wait in `sender` until they go out, then here until main acks them; a dropped socket resends // them after the next welcome. @@ -338,6 +341,7 @@ function linkRuntime(owner: DaemonLink, deps: LinkDeps) { case "welcome": if (message.v < MIN_LINK_PROTOCOL_VERSION || message.v > owner.version) throw new LinkProtocolError("Invalid welcome protocol version"); owner.version = message.v; + mismatched = false; log("linked to " + serverUrl); acknowledgeHostUpdateHealth(); unknownLog.reset(); linked = socket; redialDelay = FIRST_RETRY_MS; resend = outbox.pending(); resendAt = 0; sender.dropBlobs(); @@ -454,6 +458,7 @@ function linkRuntime(owner: DaemonLink, deps: LinkDeps) { if (stopped) return; const closed = await connect(sent, hello); if (stopped) return; + if (sent !== authorization) continue; if (typeof closed !== "string" && closed.code === LINK_CLOSE.versionMismatch) { let newest: unknown; try { newest = JSON.parse(closed.reason)?.serverVersion; } catch {} if (typeof newest === "number" && Number.isInteger(newest) && newest >= MIN_LINK_PROTOCOL_VERSION && newest < owner.version) { @@ -462,13 +467,16 @@ function linkRuntime(owner: DaemonLink, deps: LinkDeps) { log(typeof newest === "number" && newest < LINK_PROTOCOL_VERSION ? "this host's ompweb is newer than the server's: update ompweb on the server" : "the server speaks a different link protocol: update ompweb on this host"); - endLink("version"); return; + if (!deps.keepOnVersionMismatch) { endLink("version"); return; } + // Rejected application replies cannot be delivered, but remain queued if compatibility returns. + mismatched = true; } if (typeof closed !== "string" && closed.code === LINK_CLOSE.replaced) { log("another ompweb host linked with this token; exiting this link"); endLink("replaced"); return; } if (typeof closed !== "string" && closed.code === 1000) { endLink("revoked"); return; } if (typeof closed !== "string" && !closed.opened) { const { status, body } = await fetch(linkUrl, { headers: { authorization: sent }, signal: AbortSignal.any([stopSignal.signal, AbortSignal.timeout(LINK_LIMITS.offlineMs)]) }); void body?.cancel().catch(() => {}); + if (sent !== authorization) continue; if (status === 401) { if (deps.reload?.(owner, true)) continue; endLink("revoked"); return; } log(status === 503 ? "the server is unavailable (503); redialing" : "the server did not accept the WebSocket upgrade (HTTP " + status + "): check that a reverse proxy forwards Upgrade and Connection, and that the server runs an ompweb at least as new as this host's"); } else log(typeof closed === "string" ? "link lost (" + closed + "); redialing" : "link lost (code " + closed.code + "); redialing"); @@ -485,7 +493,7 @@ function linkRuntime(owner: DaemonLink, deps: LinkDeps) { stopped = true; stopSignal.abort(); socket?.close(); clearTimeout(flushTimer); clearTimeout(statsTimer); await loop; } - return { queue, output, discard, result, report, busy: () => reliablePending > 0, ackedDown: () => down.acked, isUp: () => linked !== null && !stopped, + return { queue, output, discard, result, report, busy: () => !mismatched && reliablePending > 0, ackedDown: () => down.acked, isUp: () => linked !== null && !stopped, rekey: (token: string) => { owner.login.token = token; authorization = "Bearer " + token; socket?.close(1001, "credentials changed"); }, start: () => { if (!started && !stopped) { started = true; loop = keepLinked(); } }, stop }; } diff --git a/lib/hosts/daemon-links.test.mjs b/lib/hosts/daemon-links.test.mjs index cd3d573a8..7290e3b33 100644 --- a/lib/hosts/daemon-links.test.mjs +++ b/lib/hosts/daemon-links.test.mjs @@ -19,9 +19,9 @@ const until = async predicate => { const end=Date.now()+5000;while(!predicate()) function hub(t, max=LINK_PROTOCOL_VERSION){ const hellos=[],messages=[];let socket,ack=0; const server=Bun.serve({port:0,hostname:"127.0.0.1",fetch(req,s){return s.upgrade(req)?undefined:new Response("",{status:426});},websocket:{message(ws,data){if(typeof data!=="string")return;for(const line of data.split("\n").filter(Boolean)){const m=JSON.parse(line);if(m.type==="hello"){hellos.push(m);socket=ws;if(m.v>max){ws.close(4426,JSON.stringify({serverVersion:max}));continue;}ws.send(encodeLine({type:"welcome",v:max,hostId:"h"}));}else if(m.seq){messages.push(m);ack=m.seq;}}if(ws.readyState===1)ws.send(encodeLine({type:"ack",ack}));}}});t.after(()=>server.stop(true)); - return {url:`http://127.0.0.1:${server.port}`,hellos,messages,close:(code)=>socket.close(code)}; + return {url:`http://127.0.0.1:${server.port}`,hellos,messages,close:(code)=>socket.close(code),setMax:(version)=>{max=version;}}; } -function link(server,id,ends){return new DaemonLink({url:server.url,token:"test"},id,{allowHttp:false,primary:id==="l1",hello:(link)=>encodeLine({type:"hello",v:link.version,ompVersion:null,platform:"linux",home:"/tmp",autoApproveExtension:false,ack:link.ackedDown,channels:[],link:{id:link.id,notifier:false}}),handleDown(){},onEnd:(link,reason)=>ends.push([link.id,reason]),log(){}});} +function link(server,id,ends,options={}){return new DaemonLink({url:server.url,token:"test"},id,{allowHttp:false,primary:id==="l1",hello:(link)=>encodeLine({type:"hello",v:link.version,ompVersion:null,platform:"linux",home:"/tmp",autoApproveExtension:false,ack:link.ackedDown,channels:[],link:{id:link.id,notifier:false}}),handleDown(){},onEnd:(link,reason)=>ends.push([link.id,reason]),log(){},...options});} test("a server below the minimum protocol ends only its own link and leaves the other live",async t=>{ const a=hub(t),b=hub(t,LINK_PROTOCOL_VERSION-1),ends=[],la=link(a,"l1",ends),lb=link(b,"l2",ends);t.after(()=>Promise.all([la.stop(),lb.stop()]));la.start();lb.start(); @@ -66,3 +66,17 @@ test("reliable results hold their own link busy until acknowledged without block await until(() => a.messages.some(message => message.id === "held-result") && !la.busy()); assert.equal(lb.isUp(), true); }); + +test("rejected application results do not wedge maintenance and still replay when compatibility returns", async t => { + const server = hub(t, LINK_PROTOCOL_VERSION - 1); + const connection = link(server, "l1", [], { keepOnVersionMismatch: true }); + t.after(() => connection.stop()); + connection.result("pending-before-mismatch", { completed: true }); + assert.equal(connection.busy(), true, "a compatible but disconnected link still protects reliable results"); + connection.start(); + await until(() => server.hellos.length > 0 && !connection.busy()); + assert.equal(connection.isUp(), false); + server.setMax(LINK_PROTOCOL_VERSION); + await until(() => connection.isUp() && server.messages.some(message => message.id === "pending-before-mismatch")); + assert.equal(server.messages.find(message => message.id === "pending-before-mismatch").result.completed, true); +}); diff --git a/lib/hosts/daemon-maintenance.ts b/lib/hosts/daemon-maintenance.ts new file mode 100644 index 000000000..2c4eaa3d5 --- /dev/null +++ b/lib/hosts/daemon-maintenance.ts @@ -0,0 +1,237 @@ +// Host end of the maintenance API (maintenance-protocol.ts): dials the server's /api/hosts/maintenance with the host's +// bearer token and answers its requests through `deps.handleRequest`. Independent of the application link +// (daemon-link.ts): its own socket and redial loop, and nothing it meets (a server without this API, another API version, +// a refused token) ever stops the daemon or its omp children. Node-only, relative imports. +import { setTimeout as sleep } from "node:timers/promises"; +import { isRecord } from "../type-guards"; +import { + decodeFrame, + encodeFrame, + failure, + MAINTENANCE_API_VERSION, + MAINTENANCE_CLOSE, + MAINTENANCE_LIMITS, + MAINTENANCE_PATH, + MaintenanceError, + maintenanceTimings, + PONG, + type MaintenanceHello, + type MaintenanceOp, +} from "./maintenance-protocol"; + +const REQUEST_ID = /^[A-Za-z0-9_-]{1,64}$/; +// Each wait varies ±20% so many hosts do not redial in step. +const withJitter = (ms: number) => ms * (0.8 + Math.random() * 0.4); +const reason = (error: unknown) => (error instanceof Error ? error.message : String(error)); + +export interface MaintenanceDeps { + /** The host's hello, built on every dial; its apiVersion is overridden with this API's. */ + hello(): MaintenanceHello; + /** Runs one request. Throw an error with a `status` or `httpStatus` (400..599) and a lowercase `code` (HostOpError, HostUpdateError) to answer with them. */ + handleRequest(op: MaintenanceOp, args: Record): Promise; + log(text: string): void; + /** Called before each redial, like the application link's reload. */ + reload?(): void; + /** The link came up (welcomed) or went down. */ + onState?(up: boolean): void; +} + +/** How a dial ended. `linked`: the server welcomed it. `code`: the server's close code, 0 when this side ended it. */ +interface Closed { + linked: boolean; + code: number; + reason: string; +} + +export class DaemonMaintenance { + private token: string; + private stopped = false; + private loop: Promise | null = null; + /** Ends the dial in progress (its probe or its socket). */ + private cancel: (() => void) | null = null; + private readonly abort = new AbortController(); + private lastLog = ""; + + constructor(private readonly login: { url: string; token: string }, private readonly deps: MaintenanceDeps) { + this.token = login.token; + } + + start(): void { + if (!this.loop && !this.stopped) this.loop = this.run(); + } + + /** Ends the redial wait, the dial and the socket; replies of requests still running are dropped. */ + async stop(): Promise { + this.stopped = true; + this.abort.abort(); + this.cancel?.(); + await this.loop; + } + + /** The token changed: redial with it. */ + rekey(token: string): void { + this.token = token; + this.cancel?.(); + } + + private say(text: string): void { + if (text === this.lastLog) return; + this.lastLog = text; + this.deps.log(text); + } + + private async run(): Promise { + let delay = maintenanceTimings.retryMs; + while (!this.stopped) { + try { + const closed = await this.connect(); + if (this.stopped) return; + if (closed.linked) delay = maintenanceTimings.retryMs; + // Another API, another host holding the token: dialing again soon changes nothing. + if (closed.code === MAINTENANCE_CLOSE.apiMismatch || closed.code === MAINTENANCE_CLOSE.replaced) delay = maintenanceTimings.retryMaxMs; + this.say(`maintenance link ${closed.linked ? "lost" : "unavailable"} (${closed.reason || `code ${closed.code}`}); redialing`); + this.deps.reload?.(); + } catch (error) { + if (this.stopped) return; + this.say(`maintenance link lost (${reason(error)}); redialing`); + } + if (this.stopped) return; + await sleep(withJitter(delay), undefined, { signal: this.abort.signal }).catch(() => {}); + delay = Math.min(delay * 2, maintenanceTimings.retryMaxMs); + } + } + + /** One dial: discovery, then the socket until it closes. */ + private async connect(): Promise { + const serverUrl = this.login.url.replace(/\/+$/, "") + MAINTENANCE_PATH; + const authorization = `Bearer ${this.token}`; + const attempt = new AbortController(); + this.cancel = () => attempt.abort(); + const unavailable = await this.discover(serverUrl, authorization, attempt.signal); + // Stopped or rekeyed meanwhile: not the server's answer. + if (attempt.signal.aborted) return { linked: false, code: 0, reason: "cancelled" }; + if (unavailable) return unavailable; + return this.dial(serverUrl.replace(/^http/, "ws"), authorization); + } + + /** + * GET the endpoint first: only a server that answers 200 `{apiVersion: 1}` (to this token) is sent a hello. A missing + * endpoint, a page, a 426 or another version is not this API, and an old server may accept any WebSocket URL and + * mistake the hello for application traffic. Null when the server speaks API 1. + */ + private async discover(url: string, authorization: string, signal: AbortSignal): Promise { + let response: Response; + try { + response = await fetch(url, { headers: { authorization }, redirect: "error", signal: AbortSignal.any([signal, AbortSignal.timeout(maintenanceTimings.helloMs)]) }); + } catch (error) { + return { linked: false, code: 0, reason: reason(error) }; + } + let body: unknown = null; + if (response.status === 200) { + try { + body = JSON.parse(await response.text()); + } catch { + // A page, or the read was cut: not this API. + } + } else void response.body?.cancel().catch(() => {}); + if (isRecord(body) && body.apiVersion === MAINTENANCE_API_VERSION) return null; + // A refused token or an unavailable server may be fixed by the next dial; anything else is not API 1. + const transient = response.status === 401 || response.status >= 500; + const why = response.status === 401 ? "the server refused this host's token" : `the server has no maintenance API ${MAINTENANCE_API_VERSION}`; + return { linked: false, code: transient ? 0 : MAINTENANCE_CLOSE.apiMismatch, reason: `${why} (HTTP ${response.status})` }; + } + + /** One socket from dial to close. */ + private dial(url: string, authorization: string): Promise { + const { promise, resolve } = Promise.withResolvers(); + const hello = this.deps.hello(); + const advertised = new Set(hello.capabilities); + const helloLine = encodeFrame({ ...hello, type: "hello", apiVersion: MAINTENANCE_API_VERSION }); + // This repo's DOM typing has no init argument; Node's and Bun's WebSocket take `{ headers }`. + const ws = new WebSocket(url, { headers: { authorization } } as unknown as string[]); + const active = new Set(); + let linked = false; + let done = false; + let watchdog: NodeJS.Timeout | undefined; + // Once ended, a socket stays inert: every handler below ignores it. + const end = (code: number, why: string) => { + if (done) return; + done = true; + clearTimeout(watchdog); + this.cancel = null; + if (linked) this.deps.onState?.(false); + resolve({ linked, code, reason: why }); + }; + const giveUp = (why: string) => { + end(0, why); + ws.close(); + }; + this.cancel = () => giveUp("cancelled"); + // The server pings every pingMs: silence for offlineMs is a dead link. + const alive = () => { + clearTimeout(watchdog); + watchdog = setTimeout(() => giveUp("no traffic from the server"), maintenanceTimings.offlineMs); + watchdog.unref?.(); + }; + alive(); + + const reply = (id: string, body: object) => { + if (done) return; + try { + ws.send(encodeFrame({ type: "res", id, ...body })); + } catch (error) { + // An oversized or unserializable result: say so. A closed socket: the server saw the link drop. + try { + ws.send(encodeFrame({ type: "res", id, ok: false, ...failure(error) })); + } catch { + // Closed. + } + } + }; + const request = (frame: Record) => { + const { id, op, args } = frame; + if (typeof id !== "string" || !REQUEST_ID.test(id)) return giveUp("invalid request id"); + const answer = async (): Promise => { + if (typeof op !== "string" || !advertised.has(op)) throw new MaintenanceError(501, "unsupported_op", "Unsupported maintenance operation"); + if (!isRecord(args)) throw new MaintenanceError(400, "bad_request", "A request's args must be an object"); + if (active.has(id)) throw new MaintenanceError(409, "duplicate_request", "A request with this id is running"); + if (active.size >= MAINTENANCE_LIMITS.inFlight) throw new MaintenanceError(429, "busy", "Too many maintenance requests are running"); + active.add(id); + try { + return { ok: true, result: (await this.deps.handleRequest(op as MaintenanceOp, args)) ?? null }; + } finally { + active.delete(id); + } + }; + void answer().catch((error: unknown) => ({ ok: false, ...failure(error) })).then((body) => reply(id, body)); + }; + const onFrame = (data: unknown) => { + const frame = decodeFrame(data); + if (frame.type === "ping") ws.send(PONG); + else if (frame.type === "welcome") { + if (frame.apiVersion !== MAINTENANCE_API_VERSION) return giveUp(`the server's maintenance API is ${String(frame.apiVersion)}`); + linked = true; + this.lastLog = ""; + this.deps.onState?.(true); + } else if (frame.type === "req") { + if (linked) request(frame); + else giveUp("request before welcome"); + } + }; + + ws.onopen = () => { + if (!done) ws.send(helloLine); + }; + ws.onmessage = ({ data }: MessageEvent) => { + if (done) return; + alive(); + try { + onFrame(data); + } catch (error) { + giveUp(reason(error)); + } + }; + ws.onclose = (event) => end(event.code, event.reason); + return promise; + } +} diff --git a/lib/hosts/daemon.test.mjs b/lib/hosts/daemon.test.mjs index 40ad4dc8f..561d832aa 100644 --- a/lib/hosts/daemon.test.mjs +++ b/lib/hosts/daemon.test.mjs @@ -1226,24 +1226,25 @@ test("uplink messages unacked when the socket drops are resent after the next we } }); -test("close 4426 exits 1 naming the side to update; close 4409 exits 0", async () => { +test("application version rejection keeps the daemon alive for maintenance and retries; replacement still exits", async () => { const log = captureLog(); try { - for (const [serverVersion, advice] of [[2, "this host's ompweb is newer than the server's: update ompweb on the server"], [10, "update ompweb on the server"], [LINK_PROTOCOL_VERSION + 1, "update ompweb on this host"]]) { - log.lines.length = 0; + for (const serverVersion of [2, 10, LINK_PROTOCOL_VERSION + 1]) { const server = scriptedServer(); const daemon = start(server); + let exited = false; + void daemon.done.then(() => { exited = true; }); await waitFor(() => server.hellos.length === 1, "the hello"); server.close(4426, JSON.stringify({ serverVersion })); - assert.equal(await daemon.done, 1); - assert.ok(log.lines.some((line) => line.includes(advice)), log.lines.join("|")); + await waitFor(() => server.hellos.length >= 2, "application redial without terminating maintenance"); + assert.equal(exited, false); + await daemon.stop(); } const server = scriptedServer(); const replaced = start(server); await waitFor(() => server.hellos.length === 1, "the hello"); server.close(4409, "replaced by a newer link"); assert.equal(await replaced.done, 0); - assert.ok(log.lines.some((line) => line.includes("another ompweb host linked with this token; exiting")), log.lines.join("|")); } finally { log.restore(); } diff --git a/lib/hosts/daemon.ts b/lib/hosts/daemon.ts index 803b3580b..726c96102 100644 --- a/lib/hosts/daemon.ts +++ b/lib/hosts/daemon.ts @@ -11,6 +11,8 @@ import { sampleHostCapacity } from "./capacity"; import type { HostControlCommand, HostControlStatus } from "./host-control"; import { readHostLinks, readHostLogin, type HostLink } from "./host-login"; import { DaemonLink, dialable, nextRedialDelay } from "./daemon-link"; +import { DaemonMaintenance } from "./daemon-maintenance"; +import { MAINTENANCE_API_VERSION, type MaintenanceOp } from "./maintenance-protocol"; import { ChannelTable } from "./daemon-channels"; import { credentialsEnv, serverFetch, startBrokerForwarder } from "./daemon-broker"; import { createClusterConfig } from "./daemon-cluster-config"; @@ -91,6 +93,7 @@ export function runHostDaemon(options: DaemonOptions): HostDaemon { for (const link of initial) if (!dialable(link.url, options.allowHttp)) throw new Error("ompweb host needs an https:// server URL (http:// only for loopback, or with --allow-http)"); const primaryUrl = (explicit?.[0]?.url ?? readHostLogin()?.url ?? initial[0]?.url)?.replace(/\/+$/, ""); const links = new Map(); + const maintenanceLinks = new Map(); const ended = new Map(); const services = new Map(); const uploadOwners = new Map(); @@ -252,13 +255,18 @@ export function runHostDaemon(options: DaemonOptions): HostDaemon { } /** Stops a removed link's workspace service (tracked so shutdown awaits it) and frees its upload ownership so other servers can take those uploads. */ function dropLink(link: DaemonLink): void { + const maintenance = maintenanceLinks.get(link.id); maintenanceLinks.delete(link.id); + if (maintenance) { + const stop = maintenance.stop().finally(() => stops.delete(stop)); + stops.add(stop); + } const service = services.get(link.id); services.delete(link.id); if (service) { const stop: Promise = service.stop().catch(error => log("could not stop a workspace service: " + reason(error))).finally(() => stops.delete(stop)); stops.add(stop); } for (const [id, owner] of uploadOwners) if (owner.link === link) uploadOwners.delete(id); for (const [key, owner] of uploadAttempts) if (owner === link) uploadAttempts.delete(key); } function makeLink(login: HostLink, primary: boolean): DaemonLink { - const link = new DaemonLink(login, "l" + ++linkId, { allowHttp: options.allowHttp, primary, log, + const link = new DaemonLink(login, "l" + ++linkId, { allowHttp: options.allowHttp, primary, log, keepOnVersionMismatch: true, hello: (link, replay) => encodeLine({ type: "hello", v: link.version, ompVersion, ompwebVersion: packageJson.version, workspaceOps: Object.keys(WORKSPACE_OP_PARAMS), platform: process.platform, home: homedir(), autoApproveExtension: localHost.autoApproveExtension(), ack: link.ackedDown, channels: channels.listing(link, replay), @@ -295,7 +303,18 @@ export function runHostDaemon(options: DaemonOptions): HostDaemon { }, }); services.set(link.id, createWorkspaceService(uploads, (id, epoch, event) => link.queue({ type: "workspace_file", id, epoch, event }, "watch:" + id))); - links.set(login.url, link); return link; + links.set(login.url, link); + const maintenance = new DaemonMaintenance(login, { + hello: () => ({ type: "hello", apiVersion: MAINTENANCE_API_VERSION, protocolVersion: LINK_PROTOCOL_VERSION, + ompwebVersion: packageJson.version, ompVersion, platform: process.platform, + capabilities: ["status", "listSessions", "checkUpdates", "updateSoftware"] }), + handleRequest: (op, args) => runMaintenance(link, op, args), + reload: () => { reconcile(link, true); }, + log, + }); + maintenanceLinks.set(link.id, maintenance); + maintenance.start(); + return link; } function reconcile(source?: DaemonLink, unauthorized = false): boolean { @@ -315,7 +334,7 @@ export function runHostDaemon(options: DaemonOptions): HostDaemon { for (const [url, link] of links) { const login = wanted.get(url); if (login?.token === link.login.token) continue; - if (login) { link.rekey(login.token); continue; } + if (login) { link.rekey(login.token); maintenanceLinks.get(link.id)?.rekey(login.token); continue; } links.delete(url); channels.linkGone(link); dropLink(link); void link.stop(); } for (const login of wanted.values()) { @@ -327,6 +346,32 @@ export function runHostDaemon(options: DaemonOptions): HostDaemon { reportConfig(); electNotifier(); return source ? links.get(source.login.url) !== source || source.login.token !== previousToken : true; } + async function runMaintenance(link: DaemonLink, op: MaintenanceOp, args: Record): Promise { + if (stopped) throw new HostOpError(503, "host_stopping", "This host is stopping"); + const update = op === "checkUpdates" || op === "updateSoftware"; + if (update) nativeUpdatesInFlight++; + try { + if (op === "status") return { + apiVersion: MAINTENANCE_API_VERSION, protocolVersion: LINK_PROTOCOL_VERSION, + ompwebVersion: packageJson.version, ompVersion, platform: process.platform, + capabilities: ["status", "listSessions", "checkUpdates", "updateSoftware"], + workspaceOps: Object.keys(WORKSPACE_OP_PARAMS), sessionProcesses: channels.size, + activeTurns: channels.activeTurns, sessions: channels.maintenanceSessions(), + drained, busyReason: busyReason(), operation: getHostUpdateOperation(), + }; + if (op === "listSessions") { + const states = new Map(channels.maintenanceSessions().map(session => [session.id, session])); + return (await sessionOps.listSessions()).map(session => ({ + id: session.id, title: session.name ?? null, cwd: session.cwd, modified: session.modified, + processRunning: states.has(session.id), active: states.get(session.id)?.active ?? false, + })); + } + return await runOp(link, op, args); + } finally { + if (update) nativeUpdatesInFlight--; + } + } + function handleDown(link: DaemonLink, message: DownMessage | UnknownMessage): void { try { switch (message.type) { @@ -453,7 +498,7 @@ export function runHostDaemon(options: DaemonOptions): HostDaemon { resume.freeze(); // The configuration first: its requests to a server are aborted at once, so no start waiting on them holds the stop. const configStopped = config.stop(); - stopping = (async () => { await Promise.all([...links.values()].map(link => link.stop())); await Promise.all([...stops, ...[...services.values()].map(service => service.stop())]); await (await hostSocket)?.close(); await channels.stop(); await configStopped; await forwarder.stop(); resume.dispose(); finish(code); })(); + stopping = (async () => { await Promise.all([...links.values()].map(link => link.stop())); await Promise.all([...maintenanceLinks.values()].map(link => link.stop())); await Promise.all([...stops, ...[...services.values()].map(service => service.stop())]); await (await hostSocket)?.close(); await channels.stop(); await configStopped; await forwarder.stop(); resume.dispose(); finish(code); })(); return stopping; } // Links first: a recovered sharing session needs its link to start. diff --git a/lib/hosts/hosts-route.test.mjs b/lib/hosts/hosts-route.test.mjs index b6ca7a668..a723e8ad3 100644 --- a/lib/hosts/hosts-route.test.mjs +++ b/lib/hosts/hosts-route.test.mjs @@ -438,7 +438,7 @@ test("caller telemetry cannot make offline or refused-version members eligible; hub.closeHostLink(ready.host.id); }); const { socket: refused } = upgradeSocket(old.token); - refused.frame(`${JSON.stringify({ type: "hello", v: 10 })}\n`); + refused.frame(`${JSON.stringify({ type: "hello", v: 9 })}\n`); assert.equal(refused.closed?.code, 4426, "a daemon below the minimum link protocol is refused"); await readyLink.dial(); const pool = seedPools(t, credentials.map(({ host }) => host.id)); diff --git a/lib/hosts/hub.test.mjs b/lib/hosts/hub.test.mjs index 3be65b2c0..723895bdf 100644 --- a/lib/hosts/hub.test.mjs +++ b/lib/hosts/hub.test.mjs @@ -171,8 +171,8 @@ test("a missing, malformed or wrong bearer token gets 401 before any upgrade or test("a hello with another version is closed 4426 with the server's version and marks the host needs-upgrade", async () => { const { host, token } = createHost("Other version"); - // 10: a prerelease daemon whose channels may still carry omp RPC frames. - for (const v of [LINK_PROTOCOL_VERSION + 1, 10]) { + // The v10/v11 shim is upgrade-only; earlier protocols remain unsupported. + for (const v of [LINK_PROTOCOL_VERSION + 1, 9]) { const socket = dial(token, hello({ v })); assert.deepEqual(socket.closed, { code: 4426, reason: JSON.stringify({ serverVersion: LINK_PROTOCOL_VERSION }) }); const listed = hub.listHosts().find((h) => h.id === host.id); diff --git a/lib/hosts/hub.ts b/lib/hosts/hub.ts index a1dd1e78d..d741006cc 100644 --- a/lib/hosts/hub.ts +++ b/lib/hosts/hub.ts @@ -55,6 +55,13 @@ import type { HostSoftwareUpdate, HostUpdateInfo, HostUpdateOperation, HostUpdat import { isUpdateDisabled } from "../update-policy"; import { WORKSPACE_OP_PARAMS, type WorkspaceFileEvent, type WorkspaceOps } from "../workspace-types"; import { validWorkspaceResult } from "./workspace-results"; +import { closeMaintenanceHost, freezeMaintenance, getMaintenanceHost, MaintenanceError, onMaintenanceChange, requestMaintenance } from "./maintenance-hub"; +import type { MaintenanceHello } from "./maintenance-protocol"; + +/** Old daemons keep their updater, never their session protocol. */ +const legacyUpgradeVersion = (version: number | null): boolean => version === 10 || version === 11; +const upgradeOnly = (link: Link): boolean => legacyUpgradeVersion(link.hostVersion); +const upgradeOp = (op: string): boolean => op === "checkUpdates" || op === "updateSoftware"; /** Liveness and op timeouts from LINK_LIMITS; tests shorten them. */ export const linkTimings = { @@ -389,6 +396,9 @@ function flush(link: Link): void { * server must not reach the children. */ function sendDown(link: Link, body: Unsequenced, key: string): void { if (hub().frozen) return; + if (upgradeOnly(link) && (body.type !== "req" || !upgradeOp(body.op))) { + throw new HostOpError(426, "host_upgrade_required", "Update this host before using its application protocol"); + } const line = encodeQueued(body); const size = Buffer.byteLength(line); // Ahead of it: what is queued or unacked on this link (counters, no scan); fairness may send it before some of that. @@ -501,6 +511,7 @@ function endSocket(link: Link): void { failAllOps(link, () => new HostOpError(503, "host_offline", `${labelOf(link.id)} is offline`)); notify(hub().uploadListeners, link.id, null); endWorkspaceWatches(link.id); + notify(hub().changeListeners); } function stopTimers(link: Link): void { @@ -563,6 +574,7 @@ const stopping = () => json({ error: "omp-web is shutting down" }, 503); */ export function freezeHostLinks(): void { hub().frozen = true; + freezeMaintenance(); for (const link of hub().links.values()) { link.capacity = null; recordDropped(link); @@ -683,7 +695,11 @@ function onFirstFrame(ws: LinkSocket, data: string | Uint8Array): void { } catch { return refuse(); } - if (versionMismatch(link, raw)) return ws.close(LINK_CLOSE.versionMismatch, JSON.stringify({ serverVersion: LINK_PROTOCOL_VERSION })); + const version = helloVersion(raw); + const legacy = legacyUpgradeVersion(version); + if ((!legacy && versionMismatch(link, raw)) || (legacy && link.socket && !upgradeOnly(link))) { + return ws.close(LINK_CLOSE.versionMismatch, JSON.stringify({ serverVersion: LINK_PROTOCOL_VERSION })); + } let hello: UpMessage | UnknownMessage; try { hello = decodeUp(data); @@ -697,6 +713,14 @@ function onFirstFrame(ws: LinkSocket, data: string | Uint8Array): void { function openLink(link: Link, ws: LinkSocket, hello: Hello): void { // One link per host: the newest wins and the previous daemon exits. + if (legacyUpgradeVersion(hello.v) || upgradeOnly(link)) { + // Nothing from another protocol, or an uncertain previous update, is replayed through the shim. + failAllOps(link, () => new HostOpError(503, "transport_lost", "The host update connection was replaced")); + link.sender = new LinkSender(); + link.outbox = new Outbox(); + link.outboxBytes = 0; + link.unacked = 0; + } const previous = link.socket; link.socket = ws; link.capacity = null; @@ -723,8 +747,8 @@ function openLink(link: Link, ws: LinkSocket, hello: Hello): void { released(link, link.outbox.ack(hello.ack)); link.unknownLog.reset(); link.hostVersion = hello.v; - link.shared = true; - link.notifier = hello.link?.notifier === true; + link.shared = !upgradeOnly(link); + link.notifier = !upgradeOnly(link) && hello.link?.notifier === true; enqueue(link, encodeLine({ type: "welcome", v: hello.v, hostId: link.id })); // Resend what the daemon has not applied, in order and paced, before anything new. link.sentSeq = hello.ack; @@ -733,7 +757,7 @@ function openLink(link: Link, ws: LinkSocket, hello: Hello): void { flush(link); link.ompVersion = hello.ompVersion; link.ompwebVersion = hello.ompwebVersion ?? null; - link.workspaceOps = new Set(hello.workspaceOps ?? []); + link.workspaceOps = new Set(upgradeOnly(link) ? [] : hello.workspaceOps ?? []); endWorkspaceWatches(link.id); link.platform = hello.platform; link.autoApprove = hello.autoApproveExtension; @@ -742,6 +766,11 @@ function openLink(link: Link, ws: LinkSocket, hello: Hello): void { clearInterval(link.pingTimer); link.pingTimer = setInterval(() => enqueue(link, encodeLine({ type: "ping" })), linkTimings.pingMs); link.pingTimer.unref?.(); + if (upgradeOnly(link)) { + link.listedSessions = new Set(); + setStatus(link, "needs-upgrade"); + return; + } setStatus(link, hello.ompVersion === null ? "omp-missing" : "online"); reattach(link, hello.channels); notify(hub().helloListeners, link.id, hello.channels); @@ -903,6 +932,10 @@ function configCurrent(link: Link): boolean { /** `whole`: for a `part` that completes a line, that line decoded. `joined`: that line itself, which gets no dedup of * its own (its parts were deduped). */ function applyUp(link: Link, message: UpMessage | UnknownMessage, whole?: UpMessage | UnknownMessage, joined = false): void { + if (upgradeOnly(link) && !["ping", "pong", "hello", "ack", "unknown", "part", "res"].includes(message.type)) { + if ("seq" in message && message.seq !== null) link.up.applied(message.seq); + return; + } if (message.type === "ping") return enqueue(link, encodeLine({ type: "pong" })); if (message.type === "pong" || message.type === "hello" || message.type === "ack") return; if (message.type === "capacity") { @@ -991,7 +1024,10 @@ function request(hostId: string, op: HostOpName, args: Record, const link = linkFor(hostId); const label = labelOf(hostId); if (hub().frozen) return Promise.reject(new HostOpError(503, "server_stopping", "omp-web is shutting down")); - if (!link.socket || !isLinkUp(link.status)) { + if (upgradeOnly(link) && !upgradeOp(op)) { + return Promise.reject(new HostOpError(426, "host_upgrade_required", "Update this host before using its application protocol")); + } + if (!link.socket || (!isLinkUp(link.status) && !upgradeOnly(link))) { return Promise.reject(new HostOpError(503, "host_offline", `${label} is offline`)); } const { promise, resolve, reject } = Promise.withResolvers(); @@ -1560,6 +1596,16 @@ export function getHost(id: string): Host | null { const updateCompatibility = () => ({ min: MIN_LINK_PROTOCOL_VERSION, max: LINK_PROTOCOL_VERSION }); +async function requestHostUpdate(id: string, op: "checkUpdates" | "updateSoftware", args: Record): Promise { + if (!getMaintenanceHost(id)) return request(id, op, args); + try { + return await requestMaintenance(id, op, args); + } catch (error) { + if (error instanceof MaintenanceError) throw new HostOpError(error.status, error.code, error.message); + throw error; + } +} + function requireUpdateHost(id: string): void { if (!getHost(id)) throw new HostOpError(404, "host_not_found", "Unknown host"); if (isUpdateDisabled()) throw new HostOpError(403, "updates_disabled", "Updates are disabled on this web server"); @@ -1581,7 +1627,7 @@ function updateOperation(value: unknown): value is HostUpdateOperation { /** Every machine, including this one, checks on its daemon. Never fall back to the web server's installation. */ export async function getHostSoftwareUpdates(id: string, force = false): Promise { requireUpdateHost(id); - const result = await request(id, "checkUpdates", { compatibility: updateCompatibility(), force }); + const result = await requestHostUpdate(id, "checkUpdates", { compatibility: updateCompatibility(), force }); if (!isRecord(result) || !softwareUpdate(result.ompweb) || !isRecord(result.omp) || !nullableString(result.omp.currentVersion) || !["npm", "bun", "binary"].includes(String(result.ompweb.installMethod)) || !(result.ompweb.protocolVersion === null || (Number.isSafeInteger(result.ompweb.protocolVersion) && Number(result.ompweb.protocolVersion) > 0)) @@ -1605,7 +1651,7 @@ export async function updateHostSoftware(id: string, target: HostUpdateTarget, v if (!candidate.updateAvailable || !candidate.canUpdate || candidate.availableVersion !== version) { throw new HostOpError(409, "host_update_blocked", candidate.blockedReason ?? "The available version changed; check for updates again"); } - const result = await request(id, "updateSoftware", { target, version, compatibility: updateCompatibility() }); + const result = await requestHostUpdate(id, "updateSoftware", { target, version, compatibility: updateCompatibility() }); if (!isRecord(result) || result.accepted !== true || !updateOperation(result.operation) || result.operation.target !== target || result.operation.version !== version) { throw new HostOpError(502, "host_bad_result", "The host returned an invalid update operation"); @@ -1619,6 +1665,9 @@ export interface LinkedHostRow { status: HostStatus; ompVersion: string | null; ompwebVersion?: string | null; + /** A connected maintenance API or legacy shim can update an incompatible application peer. */ + upgradeOnly?: boolean; + apiVersion?: number; platform: string | null; lastSeenAt: string | null; allowAgentStart: boolean; @@ -1632,20 +1681,31 @@ export interface LinkedHostRow { clusterConfig?: ConfigState & { owned: boolean; current: boolean }; } +function hostConnectionFields(link: Link | undefined, maintenance: MaintenanceHello | null): Pick { + const applicationUp = !!link?.socket && isLinkUp(link.status); + const mismatch = !!maintenance && (maintenance.protocolVersion < MIN_LINK_PROTOCOL_VERSION || maintenance.protocolVersion > LINK_PROTOCOL_VERSION); + return { + status: maintenance && !applicationUp ? mismatch ? "needs-upgrade" : "offline" : link?.status ?? "offline", + ...(!applicationUp && ((link?.socket && upgradeOnly(link)) || mismatch) ? { upgradeOnly: true } : {}), + ...(maintenance ? { apiVersion: maintenance.apiVersion } : {}), + }; +} + export function listHosts(): LinkedHostRow[] { return loadHosts().filter((record) => record.id !== LOCAL_HOST_ID).map((record) => { const link = hub().links.get(record.id); + const maintenance = getMaintenanceHost(record.id); return { id: record.id, label: record.label, - status: link?.status ?? "offline", - ompVersion: link?.ompVersion ?? null, - ompwebVersion: link?.ompwebVersion ?? null, - platform: link?.platform ?? null, + ...hostConnectionFields(link, maintenance), + ompVersion: maintenance?.ompVersion ?? link?.ompVersion ?? null, + ompwebVersion: maintenance?.ompwebVersion ?? link?.ompwebVersion ?? null, + platform: maintenance?.platform ?? link?.platform ?? null, lastSeenAt: link?.lastSeenAt ?? record.lastSeenAt, allowAgentStart: record.allowAgentStart, shareCredentials: record.shareCredentials, - hostVersion: link?.hostVersion ?? null, + hostVersion: link?.hostVersion ?? maintenance?.protocolVersion ?? null, serverVersion: LINK_PROTOCOL_VERSION, workspaceOps: [...(link?.workspaceOps ?? [])], ...(link?.configState ? { clusterConfig: { ...link.configState, owned: link.configOwned, current: configCurrent(link) } } : {}), @@ -1763,6 +1823,8 @@ export interface LocalHostRow { status: HostStatus; ompVersion: string | null; ompwebVersion?: string | null; + upgradeOnly?: boolean; + apiVersion?: number; platform: string | null; lastSeenAt: null; hostVersion: number | null; @@ -1781,15 +1843,16 @@ export type HostRow = LocalHostRow | LinkedHostRow; * the members cannot get this machine's configuration. */ export function hostsSnapshot(): { hosts: HostRow[]; pools?: HostPool[]; masterConfigError?: MasterConfigError } { const link = hub().links.get(LOCAL_HOST_ID); + const maintenance = getMaintenanceHost(LOCAL_HOST_ID); const local: LocalHostRow = { id: "local", label: customLocalLabel(), - status: link?.status ?? "offline", - ompVersion: link?.ompVersion ?? null, - ompwebVersion: link?.ompwebVersion ?? null, - platform: link?.platform ?? null, + ...hostConnectionFields(link, maintenance), + ompVersion: maintenance?.ompVersion ?? link?.ompVersion ?? null, + ompwebVersion: maintenance?.ompwebVersion ?? link?.ompwebVersion ?? null, + platform: maintenance?.platform ?? link?.platform ?? null, lastSeenAt: null, - hostVersion: link?.hostVersion ?? null, + hostVersion: link?.hostVersion ?? maintenance?.protocolVersion ?? null, serverVersion: LINK_PROTOCOL_VERSION, workspaceOps: [...(link?.workspaceOps ?? [])], serverPlatform: process.platform, @@ -1840,8 +1903,10 @@ export function getHostOmpVersion(hostId: string): string | null { export function onHostsChange(listener: () => void): () => void { const listeners = hub().changeListeners; listeners.add(listener); + const detachMaintenance = onMaintenanceChange(listener); return () => { listeners.delete(listener); + detachMaintenance(); }; } @@ -1934,7 +1999,7 @@ export function setHostUploadRetention(hostId: string, days: number | null): Pro /** Queue upload bytes (at most pieceBytes) for `hostId`'s daemon, fair with its other traffic; false when it has no socket. */ export function sendUploadFrame(hostId: string, id: string, bytes: Uint8Array): boolean { const link = hub().links.get(hostId); - if (!link?.socket || hub().frozen) return false; + if (!link?.socket || hub().frozen || upgradeOnly(link)) return false; link.sender.push(`up:${id}`, { id, blob: bytes }); flush(link); return true; @@ -1956,6 +2021,7 @@ export function onUploadEvent(listener: (hostId: string, ack: UploadAck | null) /** End a host's link now (revoke). Its daemon gets 401 when it redials and stops its children. */ export function closeHostLink(id: string): void { + closeMaintenanceHost(id); const link = hub().links.get(id); if (!link) return; hub().links.delete(id); diff --git a/lib/hosts/link-protocol.ts b/lib/hosts/link-protocol.ts index b7a24e377..a95657cc3 100644 --- a/lib/hosts/link-protocol.ts +++ b/lib/hosts/link-protocol.ts @@ -11,6 +11,8 @@ import type { SpawnRequest } from "./host"; import { isWorkspaceOp, WORKSPACE_OP_PARAMS } from "../workspace-types"; import type { WorkspaceFileEvent } from "../workspace-types"; +/** Application Protocol v12 uses maintenance API v1 on a separate authenticated connection. + * v10/v11 application links are accepted only by the upgrade shim, never by the session runtime. */ /** 11: the same wire as 10, renumbered: prerelease daemons already sent 10 with omp RPC channel lines, which this * server cannot read. 10: channel stdin/stdout lines are the runner's `{t}` frames (`cmd`/`res`/`hello`/…), no longer omp RPC frames; * `open.launch.configVersion`, the `config_state` report and the `clusterConfig` op. @@ -21,10 +23,9 @@ import type { WorkspaceFileEvent } from "../workspace-types"; * (docs/specs/2026-10-31-websocket-link-design.md, D7). 4: the link is one WebSocket: text frames carry the messages, * binary frames large results, `ack` messages replace the ack header and the POST answer. * Rules for changing it: docs/specs/2026-10-28-remote-hosts-design.md, "Protocol versions and compatibility". */ -export const LINK_PROTOCOL_VERSION = 11; -/** The oldest daemon protocol the server still links (older ones get close 4426). Dropping one is an explicit decision: - * 11 dropped all older ones, whose channel lines mean (or, for prerelease 10, may mean) omp RPC frames. */ -export const MIN_LINK_PROTOCOL_VERSION = 11; +export const LINK_PROTOCOL_VERSION = 12; +/** Oldest full application protocol; the v10/v11 update-only shim is outside this range. */ +export const MIN_LINK_PROTOCOL_VERSION = 12; /** A configuration version: the manifest's SHA-256, in lowercase hex. */ export const CONFIG_VERSION_RE = /^[0-9a-f]{64}$/; /** Safe identity shared by spawn requests and reconnect channel metadata. */ diff --git a/lib/hosts/maintenance-daemon.integration.test.mjs b/lib/hosts/maintenance-daemon.integration.test.mjs new file mode 100644 index 000000000..83f2b16a5 --- /dev/null +++ b/lib/hosts/maintenance-daemon.integration.test.mjs @@ -0,0 +1,145 @@ +import assert from "node:assert/strict"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test, { after } from "node:test"; + +const root = mkdtempSync(join(tmpdir(), "ompweb-maintenance-runtime-")); +const oldAgent = process.env.PI_CODING_AGENT_DIR, oldBin = process.env.OMP_WEB_OMP_BIN; +process.env.PI_CODING_AGENT_DIR = root; +process.env.OMP_WEB_OMP_BIN = join(root, "missing-omp"); +const { runHostDaemon } = await import("./daemon.ts"); +const hub = await import("./hub.ts"); +const { createHost, revokeHost } = await import("./registry.ts"); +const maintenance = await import("./maintenance-hub.ts"); +const { until } = await import("./link-driver.test-helper.mjs"); +after(() => { + if (oldAgent === undefined) delete process.env.PI_CODING_AGENT_DIR; else process.env.PI_CODING_AGENT_DIR = oldAgent; + if (oldBin === undefined) delete process.env.OMP_WEB_OMP_BIN; else process.env.OMP_WEB_OMP_BIN = oldBin; + rmSync(root, { recursive: true, force: true }); +}); + +test("API v1 keeps the actual daemon available after application protocol rejection", async (t) => { + const { host, token } = createHost("Maintenance only"); + const sessionId = "11111111-2222-4333-8444-5555555555cc"; + const sessions = join(root, "sessions", "-maintenance"); + mkdirSync(sessions, { recursive: true }); + writeFileSync(join(sessions, "session.jsonl"), JSON.stringify({ type: "session", version: 3, id: sessionId, cwd: root, timestamp: "2026-11-14T00:00:00.000Z" }) + "\n"); + const updateRoot = join(root, "omp-web", "host-update"); + const attempt = "api-only-health"; + mkdirSync(join(updateRoot, attempt), { recursive: true, mode: 0o700 }); + writeFileSync(join(updateRoot, "status.json"), JSON.stringify({ + operation: { id: attempt, target: "ompweb", version: "0.7.1", state: "running" }, daemonPid: 1, startedAt: Date.now(), + }), { mode: 0o600 }); + writeFileSync(join(updateRoot, attempt, "descriptor.json"), JSON.stringify({ id: attempt, version: "0.7.1", fromVersion: "0.7.0" }), { mode: 0o600 }); + let rejected = 0; + const server = Bun.serve({ + port: 0, hostname: "127.0.0.1", + fetch(request, server) { + if (new URL(request.url).pathname === "/api/hosts/maintenance") return maintenance.handleMaintenanceUpgrade(request, (data) => server.upgrade(request, { data })); + if (request.headers.get("upgrade") !== "websocket") return new Response("", { status: 426 }); + return server.upgrade(request, { data: { kind: "application" } }) ? undefined : new Response("", { status: 426 }); + }, + websocket: { + message(ws, data) { + if (ws.data.kind === "maintenance") return maintenance.maintenanceSocket.message(ws, data); + rejected++; + ws.close(4426, JSON.stringify({ serverVersion: 99 })); + }, + close(ws) { if (ws.data.kind === "maintenance") maintenance.maintenanceSocket.close(ws); }, + }, + }); + const daemon = runHostDaemon({ allowHttp: false, links: [{ url: `http://127.0.0.1:${server.port}`, token }] }); + t.after(async () => { await daemon.stop(); server.stop(true); rmSync(updateRoot, { recursive: true, force: true }); }); + await daemon.started; + await until(() => rejected > 0 && maintenance.getMaintenanceHost(host.id), "the independent API connects despite application rejection"); + const hello = maintenance.getMaintenanceHost(host.id); + assert.equal(hello.apiVersion, 1); + assert.equal(hello.protocolVersion, 12); + const status = await maintenance.requestMaintenance(host.id, "status", {}); + assert.equal(status.apiVersion, 1); + assert.equal(status.protocolVersion, 12); + assert.equal(status.sessionProcesses, 0); + assert.equal(status.busyReason, null, "an idle status read cannot count itself as pending work"); + assert.equal(existsSync(join(updateRoot, attempt, "health.json")), false, "maintenance alone cannot suppress rollback of a broken application connection"); + assert.ok(status.capabilities.includes("updateSoftware")); + const inventory = await maintenance.requestMaintenance(host.id, "listSessions", {}); + const session = inventory.find((item) => item.id === sessionId); + assert.ok(session, "maintenance lists real session metadata, not application transcript frames"); + assert.equal(session.active, false); + assert.equal(session.processRunning, false); + const row = hub.listHosts().find((item) => item.id === host.id); + assert.equal(row.status, "offline", "a compatible application endpoint outage is not a version mismatch"); + assert.equal(row.apiVersion, 1); + assert.equal(row.upgradeOnly, undefined); + assert.equal(daemon.control("status").linked, false, "maintenance must not pretend application sessions can attach"); + revokeHost(host.id); + maintenance.closeMaintenanceHost(host.id); + await until(() => !maintenance.getMaintenanceHost(host.id), "revocation ends maintenance access"); +}); + +test("maintenance reports a live prompt as active without inspecting its application transcript", async (t) => { + const { host, token } = createHost("Activity inventory"); + const id = "11111111-2222-4333-8444-5555555555dd"; + const sessionDir = join(root, "sessions", "-activity"); + mkdirSync(sessionDir, { recursive: true }); + const file = join(sessionDir, "active.jsonl"); + writeFileSync(file, JSON.stringify({ type: "session", version: 3, id, cwd: root }) + "\n"); + const fakeOmp = join(root, process.platform === "win32" ? "activity-omp.cmd" : "activity-omp"); + const source = `if (process.argv.includes("--version")) { console.log("99.0.0-fake"); process.exit(0); } +const emit = frame => process.stdout.write(JSON.stringify(frame) + "\\n"); +require("node:readline").createInterface({input:process.stdin}).on("line", line => { + const command = JSON.parse(line); + if (command.t === "cmd") emit({t:"res",id:command.id,ok:true,data:{}}); + if (command.op === "prompt") emit({t:"event",event:{type:"agent_start"}}); +}); +emit({t:"hello",proto:1,pid:process.pid,sessionId:${JSON.stringify(id)},cwd:${JSON.stringify(root)},sessionFile:${JSON.stringify(file)}}); +`; + if (process.platform === "win32") { + writeFileSync(fakeOmp + ".cjs", source); + writeFileSync(fakeOmp, `@"${process.execPath}" "${fakeOmp}.cjs" %*\r\n`); + } else writeFileSync(fakeOmp, `#!${process.execPath}\n${source}`, { mode: 0o755 }); + process.env.OMP_WEB_OMP_BIN = fakeOmp; + const { resolveOmpBin } = await import("../omp/omp-cli.ts"); + resolveOmpBin(true); + let socket, seq = 0; + const messages = []; + const server = Bun.serve({ + port: 0, hostname: "127.0.0.1", + fetch(request, server) { + if (new URL(request.url).pathname === "/api/hosts/maintenance") return maintenance.handleMaintenanceUpgrade(request, data => server.upgrade(request, { data })); + return server.upgrade(request, { data: { kind: "application" } }) ? undefined : new Response("", { status: 426 }); + }, + websocket: { + message(ws, data) { + if (ws.data.kind === "maintenance") return maintenance.maintenanceSocket.message(ws, data); + for (const line of String(data).split("\n").filter(Boolean)) { + const frame = JSON.parse(line); + if (frame.type === "hello") { socket = ws; ws.send(JSON.stringify({ type: "welcome", v: 12, hostId: host.id })); } + else if (frame.seq) { + messages.push(frame); + ws.send(JSON.stringify({ type: "ack", ack: frame.seq })); + } + } + }, + close(ws) { if (ws.data.kind === "maintenance") maintenance.maintenanceSocket.close(ws); }, + }, + }); + const daemon = runHostDaemon({ allowHttp: false, links: [{ url: `http://127.0.0.1:${server.port}`, token }] }); + t.after(async () => { await daemon.stop(); server.stop(true); }); + await daemon.started; + await until(() => socket && maintenance.getMaintenanceHost(host.id), "both connections to be ready"); + socket.send(JSON.stringify({ type: "req", seq: ++seq, id: "open-active", op: "open", args: { sessionId: id, launch: { cwd: root, recordedCwd: root, sessionFile: file, advisor: false } } })); + await until(() => messages.some(frame => frame.type === "res" && frame.id === "open-active"), "the real daemon to start its child"); + const opened = messages.find(frame => frame.type === "res" && frame.id === "open-active"); + assert.equal(opened.ok, true, JSON.stringify(opened)); + socket.send(JSON.stringify({ type: "stdin", seq: ++seq, ch: opened.result.ch, line: JSON.stringify({ t: "cmd", id: 1, op: "prompt", args: { message: "work" } }) + "\n" })); + await until(() => messages.some(frame => frame.type === "stdout" && JSON.parse(frame.line).event?.type === "agent_start"), "the prompt to become active"); + const status = await maintenance.requestMaintenance(host.id, "status", {}); + assert.equal(status.sessionProcesses, 1); + assert.equal(status.sessions.find(session => session.id === id)?.active, true); + assert.ok(status.busyReason, "an active child still blocks updating its daemon"); + const inventory = await maintenance.requestMaintenance(host.id, "listSessions", {}); + assert.equal(inventory.find(session => session.id === id)?.active, true); + assert.equal(inventory.find(session => session.id === id)?.processRunning, true); +}); diff --git a/lib/hosts/maintenance-hub.test.mjs b/lib/hosts/maintenance-hub.test.mjs new file mode 100644 index 000000000..c2164d7df --- /dev/null +++ b/lib/hosts/maintenance-hub.test.mjs @@ -0,0 +1,291 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +const dir = mkdtempSync(join(tmpdir(), "omp-maintenance-hub-")); +const previousAgentDir = process.env.PI_CODING_AGENT_DIR; +process.env.PI_CODING_AGENT_DIR = dir; +const hub = await import("./maintenance-hub.ts"); +const { MAINTENANCE_CLOSE, MAINTENANCE_LIMITS, MaintenanceError, maintenanceTimings } = await import("./maintenance-protocol.ts"); +const { createHost, revokeHost } = await import("./registry.ts"); +const timings = { ...maintenanceTimings }; + +test.after(() => { + Object.assign(maintenanceTimings, timings); + if (previousAgentDir === undefined) delete process.env.PI_CODING_AGENT_DIR; + else process.env.PI_CODING_AGENT_DIR = previousAgentDir; + rmSync(dir, { recursive: true, force: true }); +}); + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const hello = (extra = {}) => ({ type: "hello", apiVersion: 1, protocolVersion: 12, ompwebVersion: "0.7.1", ompVersion: "14.0.0", platform: "linux", capabilities: ["status", "listSessions", "checkUpdates", "updateSoftware"], ...extra }); + +/** The hub's view of a socket: what it sent and how it was closed. */ +function socket(hostId) { + return { + data: { kind: "maintenance", hostId }, + sent: [], + closed: null, + terminated: false, + send(line) { this.sent.push(JSON.parse(line)); return line.length; }, + close(code, reason) { this.closed = { code, reason }; }, + terminate() { this.terminated = true; }, + }; +} +const say = (ws, frame) => hub.maintenanceSocket.message(ws, typeof frame === "string" ? frame : JSON.stringify(frame)); +const requests = (ws) => ws.sent.filter((frame) => frame.type === "req"); +const request = (headers) => new Request("http://server/api/hosts/maintenance", { headers }); +const reject = (promise) => promise.then(() => assert.fail("expected a rejection"), (error) => error); + +function reset() { + for (const link of [...(globalThis.__ompMaintenanceHub?.links.values() ?? [])]) hub.closeMaintenanceHost(link.id); + globalThis.__ompMaintenanceHub = undefined; + Object.assign(maintenanceTimings, timings); +} +const it = (name, body) => test(name, async (t) => { t.after(reset); await body(t); }); + +/** A registered host whose daemon has said hello on a fresh socket. */ +function linked(extra) { + const { host } = createHost("h"); + const ws = socket(host.id); + say(ws, hello(extra)); + return { id: host.id, ws }; +} + +it("authenticates before discovery or upgrade", () => { + const { token } = createHost("auth"); + let upgraded = null; + const upgrade = (data) => { upgraded = data; return true; }; + assert.equal(hub.handleMaintenanceUpgrade(request({ upgrade: "websocket" }), upgrade).status, 401); + assert.equal(hub.handleMaintenanceUpgrade(request({ authorization: "Bearer wrong", upgrade: "websocket" }), upgrade).status, 401); + assert.equal(hub.handleMaintenanceUpgrade(request(), upgrade).status, 401); + assert.equal(upgraded, null); + assert.equal(hub.handleMaintenanceUpgrade(request({ authorization: `Bearer ${token}`, upgrade: "websocket" }), upgrade), undefined); + assert.equal(upgraded.kind, "maintenance"); + assert.equal(hub.handleMaintenanceUpgrade(request({ authorization: `Bearer ${token}`, upgrade: "websocket" }), () => false).status, 426); +}); + +it("answers a plain GET with the API version, never a socket", async () => { + const { token } = createHost("discovery"); + const response = hub.handleMaintenanceUpgrade(request({ authorization: `Bearer ${token}` }), () => assert.fail("no upgrade")); + assert.equal(response.status, 200); + assert.equal(response.headers.get("cache-control"), "no-store"); + assert.deepEqual(await response.json(), { apiVersion: 1, capabilities: ["status", "listSessions", "checkUpdates", "updateSoftware"] }); +}); + +it("adds the server's software to the discovery answer when given", async () => { + const { token } = createHost("server-info"); + const response = hub.handleMaintenanceUpgrade(request({ authorization: `Bearer ${token}` }), () => assert.fail("no upgrade"), { protocolVersion: 12, ompwebVersion: "0.7.1" }); + assert.deepEqual(await response.json(), { protocolVersion: 12, ompwebVersion: "0.7.1", apiVersion: 1, capabilities: ["status", "listSessions", "checkUpdates", "updateSoftware"] }); +}); + +it("welcomes a hello whatever the application protocol version", async () => { + const seen = []; + const off = hub.onMaintenanceChange(() => seen.push(1)); + const { id, ws } = linked({ protocolVersion: 99 }); + assert.deepEqual(ws.sent, [{ type: "welcome", apiVersion: 1 }]); + assert.equal(hub.getMaintenanceHost(id).protocolVersion, 99); + assert.equal(seen.length, 1); + const pending = hub.requestMaintenance(id, "status", { a: 1 }); + const [asked] = requests(ws); + assert.deepEqual({ op: asked.op, args: asked.args }, { op: "status", args: { a: 1 } }); + say(ws, { type: "res", id: asked.id, ok: true, result: { busy: false } }); + assert.deepEqual(await pending, { busy: false }); + off(); +}); + +it("refuses another API version, even when the rest of its hello is not ours", () => { + const { host } = createHost("v2"); + const ws = socket(host.id); + say(ws, { type: "hello", apiVersion: 2, whatever: [] }); + assert.deepEqual(ws.closed, { code: MAINTENANCE_CLOSE.apiMismatch, reason: JSON.stringify({ apiVersion: 1 }) }); + assert.equal(hub.getMaintenanceHost(host.id), null); +}); + +it("closes a socket whose first frame is not a valid hello", () => { + const { host } = createHost("bad"); + const closeOf = (frame) => { const ws = socket(host.id); say(ws, frame); return ws.closed.code; }; + assert.equal(closeOf({ type: "res", id: "x" }), MAINTENANCE_CLOSE.policy); + assert.equal(closeOf("not json"), MAINTENANCE_CLOSE.policy); + assert.equal(closeOf(hello({ platform: 3 })), MAINTENANCE_CLOSE.policy); + assert.equal(closeOf(JSON.stringify({ type: "hello", pad: "x".repeat(MAINTENANCE_LIMITS.frameBytes) })), MAINTENANCE_CLOSE.tooBig); + const binary = socket(host.id); + hub.maintenanceSocket.message(binary, new Uint8Array(4)); + assert.equal(binary.closed.code, MAINTENANCE_CLOSE.policy); + assert.equal(hub.getMaintenanceHost(host.id), null); +}); + +it("closes a socket that never says hello", async () => { + maintenanceTimings.helloMs = 20; + const { host } = createHost("silent"); + const ws = socket(host.id); + hub.maintenanceSocket.open(ws); + await sleep(60); + assert.equal(ws.closed.code, MAINTENANCE_CLOSE.policy); + const welcomed = socket(host.id); + hub.maintenanceSocket.open(welcomed); + say(welcomed, hello()); + await sleep(60); + assert.equal(welcomed.closed, null); +}); + +it("rechecks the registry after the hello and rejects a revoked host", () => { + const { host } = createHost("revoked"); + const ws = socket(host.id); + revokeHost(host.id); + say(ws, hello()); + assert.equal(ws.closed.code, MAINTENANCE_CLOSE.policy); + assert.equal(hub.getMaintenanceHost(host.id), null); +}); + +it("drops capabilities it does not know and never sends an op the host did not advertise", async () => { + const { id, ws } = linked({ capabilities: ["status", "future"] }); + assert.deepEqual(hub.getMaintenanceHost(id).capabilities, ["status"]); + const error = await reject(hub.requestMaintenance(id, "updateSoftware", {})); + assert.deepEqual([error.status, error.code], [501, "unsupported_op"]); + assert.equal(requests(ws).length, 0); +}); + +it("fails a request for a host without a link, an oversized one and too many at once", async () => { + assert.equal((await reject(hub.requestMaintenance("nobody", "status", {}))).code, "no_link"); + const { id, ws } = linked(); + const big = await reject(hub.requestMaintenance(id, "status", { text: "x".repeat(MAINTENANCE_LIMITS.frameBytes) })); + assert.deepEqual([big.status, big.code], [413, "too_large"]); + const open = Array.from({ length: MAINTENANCE_LIMITS.inFlight }, () => hub.requestMaintenance(id, "status", {})); + assert.equal((await reject(hub.requestMaintenance(id, "status", {}))).code, "busy"); + assert.equal(requests(ws).length, MAINTENANCE_LIMITS.inFlight); + for (const asked of requests(ws)) say(ws, { type: "res", id: asked.id, ok: true, result: null }); + assert.deepEqual(await Promise.all(open), open.map(() => null)); +}); + +it("keeps the host's native error status and code", async () => { + const { id, ws } = linked(); + const failed = hub.requestMaintenance(id, "updateSoftware", {}); + say(ws, { type: "res", id: requests(ws)[0].id, ok: false, status: 409, code: "host_busy", message: "Sessions are running" }); + const error = await reject(failed); + assert.ok(error instanceof MaintenanceError); + assert.deepEqual([error.status, error.code, error.message], [409, "host_busy", "Sessions are running"]); + const odd = hub.requestMaintenance(id, "status", {}); + say(ws, { type: "res", id: requests(ws)[1].id, ok: false, status: 200, code: "Not Snake", message: "m" }); + const normalized = await reject(odd); + assert.deepEqual([normalized.status, normalized.code], [500, "internal"]); +}); + +it("times out without repeating the request and ignores its late answer", async () => { + const { id, ws } = linked(); + const error = await reject(hub.requestMaintenance(id, "status", {}, 20)); + assert.deepEqual([error.status, error.code], [504, "timeout"]); + say(ws, { type: "res", id: requests(ws)[0].id, ok: true, result: 1 }); + assert.equal(requests(ws).length, 1); + const next = hub.requestMaintenance(id, "status", {}); + say(ws, { type: "res", id: requests(ws)[1].id, ok: true, result: 2 }); + assert.equal(await next, 2); +}); + +it("fails a request in flight when the link drops and never replays it on the next link", async () => { + const { id, ws } = linked(); + const lost = hub.requestMaintenance(id, "updateSoftware", { version: "1" }); + const lateId = requests(ws)[0].id; + hub.maintenanceSocket.close(ws); + const error = await reject(lost); + assert.deepEqual([error.status, error.code], [502, "delivery_unknown"]); + assert.equal(hub.getMaintenanceHost(id), null); + const again = socket(id); + say(again, hello()); + assert.deepEqual(again.sent, [{ type: "welcome", apiVersion: 1 }]); + say(ws, { type: "res", id: lateId, ok: true, result: "late" }); + assert.equal(requests(again).length, 0); +}); + +it("lets a newer socket replace the old one, closing and failing only the old", async () => { + const { id, ws: old } = linked(); + const inFlight = hub.requestMaintenance(id, "status", {}); + const fresh = socket(id); + say(fresh, hello({ ompwebVersion: "0.7.2" })); + assert.equal(old.closed.code, MAINTENANCE_CLOSE.replaced); + assert.equal(fresh.closed, null); + assert.equal((await reject(inFlight)).code, "delivery_unknown"); + assert.equal(hub.getMaintenanceHost(id).ompwebVersion, "0.7.2"); + // The old socket's close event and frames belong to no link. + hub.maintenanceSocket.close(old); + say(old, { type: "res", id: "x", ok: true }); + assert.equal(hub.getMaintenanceHost(id).ompwebVersion, "0.7.2"); + const next = hub.requestMaintenance(id, "status", {}); + assert.equal(requests(old).length, 1); + say(fresh, { type: "res", id: requests(fresh)[0].id, ok: true, result: "fresh" }); + assert.equal(await next, "fresh"); +}); + +it("closeMaintenanceHost closes the socket, fails what is in flight and notifies", async () => { + const { id, ws } = linked(); + const seen = []; + const off = hub.onMaintenanceChange(() => seen.push(hub.getMaintenanceHost(id))); + const inFlight = hub.requestMaintenance(id, "status", {}); + hub.closeMaintenanceHost(id); + assert.equal(ws.closed.code, MAINTENANCE_CLOSE.policy); + assert.equal((await reject(inFlight)).code, "delivery_unknown"); + assert.deepEqual(seen, [null]); + off(); + hub.closeMaintenanceHost(id); + assert.equal(seen.length, 1); +}); + +it("a throwing listener does not break the link", () => { + const off = hub.onMaintenanceChange(() => { throw new Error("listener"); }); + const { id } = linked(); + assert.ok(hub.getMaintenanceHost(id)); + off(); +}); + +it("pings, and drops a host that stays silent", async () => { + maintenanceTimings.pingMs = 10; + maintenanceTimings.offlineMs = 45; + const { id, ws } = linked(); + // Handled at once: it fails while the test sleeps. + const outcome = reject(hub.requestMaintenance(id, "status", {})); + await sleep(25); + assert.ok(ws.sent.some((frame) => frame.type === "ping")); + say(ws, { type: "pong" }); + assert.equal(ws.terminated, false); + await sleep(120); + assert.equal(ws.terminated, true); + assert.equal(hub.getMaintenanceHost(id), null); + assert.equal((await outcome).code, "delivery_unknown"); +}); + +it("closes a link whose frame is malformed or too large", () => { + const first = linked(); + say(first.ws, "{"); + assert.equal(first.ws.closed.code, MAINTENANCE_CLOSE.policy); + assert.equal(hub.getMaintenanceHost(first.id), null); + const second = linked(); + say(second.ws, JSON.stringify({ type: "res", pad: "x".repeat(MAINTENANCE_LIMITS.frameBytes) })); + assert.equal(second.ws.closed.code, MAINTENANCE_CLOSE.tooBig); + assert.equal(hub.getMaintenanceHost(second.id), null); +}); + +it("freezing refuses requests and upgrades, closes links 1001 and ignores what arrives after", async () => { + const { token } = createHost("frozen"); + const { id, ws } = linked(); + const inFlight = hub.requestMaintenance(id, "status", {}); + hub.freezeMaintenance(); + assert.equal(ws.closed.code, MAINTENANCE_CLOSE.goingAway); + assert.equal((await reject(inFlight)).code, "delivery_unknown"); + assert.equal(hub.getMaintenanceHost(id), null); + assert.equal((await reject(hub.requestMaintenance(id, "status", {}))).code, "frozen"); + const headers = { authorization: `Bearer ${token}` }; + assert.equal(hub.handleMaintenanceUpgrade(request({ ...headers, upgrade: "websocket" }), () => true).status, 503); + assert.equal(hub.handleMaintenanceUpgrade(request(headers), () => true).status, 503); + const late = socket(id); + say(late, hello()); + assert.equal(late.sent.length, 0); +}); + +it("keeps its state on globalThis across a hot reload", () => { + const { id } = linked(); + const state = globalThis.__ompMaintenanceHub; + assert.ok(state.links.has(id)); + assert.equal(hub.getMaintenanceHost(id).platform, "linux"); +}); diff --git a/lib/hosts/maintenance-hub.ts b/lib/hosts/maintenance-hub.ts new file mode 100644 index 000000000..e8ccbad04 --- /dev/null +++ b/lib/hosts/maintenance-hub.ts @@ -0,0 +1,298 @@ +// Main-server end of the maintenance API (maintenance-protocol.ts): accepts `ompweb host` daemons on +// /api/hosts/maintenance and carries the server's requests to them. Independent of the application link (hub.ts): its own +// sockets, lifecycle and version, so a host whose application protocol differs stays reachable for upgrades. +// Everything a daemon sends is untrusted input. No `next/*` or `@/` imports. +import { randomUUID } from "node:crypto"; +import { hostFromBearer } from "./host-bearer"; +import { + decodeFrame, + encodeFrame, + failure, + MAINTENANCE_API_VERSION, + MAINTENANCE_CLOSE, + MAINTENANCE_LIMITS, + MAINTENANCE_OPS, + MaintenanceError, + maintenanceTimings, + parseHello, + PING, + WELCOME, + type MaintenanceHello, + type MaintenanceOp, +} from "./maintenance-protocol"; +import { readHosts } from "./registry"; + +export { MaintenanceError }; + +/** What the hub keeps on each maintenance socket (Bun's `ws.data`): the host its token named, and whether its hello came. */ +export interface MaintenanceSocketData { + kind: "maintenance"; + hostId: string; + hello?: boolean; +} + +/** The server's software, as discovery reports it for the host to inspect. */ +export interface MaintenanceServerInfo { + protocolVersion: number; + ompwebVersion: string; +} + +/** The part of Bun's ServerWebSocket the hub uses, so tests drive it with a plain object. */ +export interface MaintenanceSocket { + readonly data: MaintenanceSocketData; + send(data: string): number; + close(code?: number, reason?: string): void; + terminate(): void; +} + +interface Pending { + resolve: (value: unknown) => void; + reject: (error: Error) => void; + timer: NodeJS.Timeout; +} + +/** A host with a welcomed socket; there is no link before its hello. */ +interface Link { + id: string; + socket: MaintenanceSocket; + hello: MaintenanceHello; + pending: Map; + /** performance.now() of the host's latest frame. */ + seen: number; + heartbeat: NodeJS.Timeout; +} + +interface HubState { + links: Map; + listeners: Set<() => void>; + frozen: boolean; +} + +declare global { + var __ompMaintenanceHub: HubState | undefined; +} + +// globalThis, like the application hub: survives a `bun --hot` re-evaluation. +function hub(): HubState { + return (globalThis.__ompMaintenanceHub ??= { links: new Map(), listeners: new Set(), frozen: false }); +} + +const refuse = (error: string, status: number) => Response.json({ error }, { status, headers: { "cache-control": "no-store" } }); + +/** + * GET /api/hosts/maintenance. The host's bearer token is checked first, so nothing unauthenticated gets an answer or a + * socket. A plain GET is discovery: 200 `{apiVersion, capabilities}` plus the caller's `server` fields (this module cannot + * know the application's versions), which a daemon needs before it dials, so a server without this API (or an old one + * that accepts any WebSocket URL) never sees a maintenance hello. An upgrade request gets the WebSocket (undefined once + * upgraded); its hello comes as the socket's first frame (maintenanceSocket). + */ +export function handleMaintenanceUpgrade(request: Request, upgrade: (data: MaintenanceSocketData) => boolean, server?: MaintenanceServerInfo): Response | undefined { + const host = hostFromBearer(request); + if (host instanceof Response) return host; + // The daemon retries a 503: it lands on the next server instead. + if (hub().frozen) return refuse("omp-web is shutting down", 503); + if (request.headers.get("upgrade")?.toLowerCase() !== "websocket") { + return Response.json({ ...server, apiVersion: MAINTENANCE_API_VERSION, capabilities: [...MAINTENANCE_OPS] }, { headers: { "cache-control": "no-store" } }); + } + return upgrade({ kind: "maintenance", hostId: host.id }) ? undefined : refuse("The maintenance link is a WebSocket", 426); +} + +/** Bun's `websocket` handlers for maintenance links (server/app.ts); `open` arms the hello deadline. */ +export const maintenanceSocket = { + open(ws: MaintenanceSocket): void { + // An authenticated socket that never says hello must not hang forever. + const timer = setTimeout(() => { + if (!ws.data.hello) ws.close(MAINTENANCE_CLOSE.policy, "No hello"); + }, maintenanceTimings.helloMs); + timer.unref?.(); + }, + message(ws: MaintenanceSocket, data: string | Uint8Array): void { + guarded(ws, () => { + // Nothing is applied once this server stops. + if (hub().frozen) return; + const link = hub().links.get(ws.data.hostId); + if (link?.socket === ws) return onFrame(link, data); + // A socket a newer one replaced, or one that was dropped: what it still sends belongs to no link. + if (ws.data.hello) return; + onHello(ws, data); + }); + }, + close(ws: MaintenanceSocket): void { + guarded(ws, () => { + const link = hub().links.get(ws.data.hostId); + if (link?.socket === ws) endLink(link); + }); + }, +}; + +/** A throw in a websocket handler would reach uncaughtException, which exits the server: it closes that socket instead. */ +function guarded(ws: MaintenanceSocket, handle: () => void): void { + try { + handle(); + } catch (error) { + console.error(`[omp-web] maintenance link of ${ws.data.hostId} failed; closing it: ${error instanceof Error ? error.stack ?? error.message : String(error)}`); + ws.close(); + } +} + +const tooBig = (error: unknown) => (error instanceof MaintenanceError && error.status === 413 ? MAINTENANCE_CLOSE.tooBig : MAINTENANCE_CLOSE.policy); + +/** A socket's first frame must be its hello. */ +function onHello(ws: MaintenanceSocket, data: string | Uint8Array): void { + let frame: Record; + try { + frame = decodeFrame(data); + } catch (error) { + return ws.close(tooBig(error), error instanceof Error ? error.message : "Bad frame"); + } + if (frame.type !== "hello") return ws.close(MAINTENANCE_CLOSE.policy, "The first frame must be a hello"); + // Compared before the rest is read: another API version may shape its hello differently. + if (frame.apiVersion !== MAINTENANCE_API_VERSION) return ws.close(MAINTENANCE_CLOSE.apiMismatch, JSON.stringify({ apiVersion: MAINTENANCE_API_VERSION })); + const hello = parseHello(frame); + if (!hello) return ws.close(MAINTENANCE_CLOSE.policy, "Invalid hello"); + // Revoked or deleted since the upgrade, or the registry cannot say. + let known: boolean; + try { + known = readHosts().some((host) => host.id === ws.data.hostId); + } catch { + return ws.close(MAINTENANCE_CLOSE.tryLater, "The host registry cannot be read"); + } + if (!known) return ws.close(MAINTENANCE_CLOSE.policy, "Unknown host"); + ws.data.hello = true; + const { links } = hub(); + const link: Link = { + id: ws.data.hostId, + socket: ws, + hello, + pending: new Map(), + seen: performance.now(), + heartbeat: setInterval(() => beat(link), maintenanceTimings.pingMs), + }; + link.heartbeat.unref?.(); + // One link per host: the newest wins; only the old socket closes. + const previous = links.get(link.id); + if (previous) { + previous.socket.close(MAINTENANCE_CLOSE.replaced, "replaced by a newer maintenance link"); + retire(previous); + } + links.set(link.id, link); + ws.send(WELCOME); + changed(); +} + +function onFrame(link: Link, data: string | Uint8Array): void { + let frame: Record; + try { + frame = decodeFrame(data); + } catch (error) { + return closeLink(link, tooBig(error), error instanceof Error ? error.message : "Bad frame"); + } + link.seen = performance.now(); + // A pong only proves liveness; types a later revision of API 1 adds are ignored. + if (frame.type !== "res" || typeof frame.id !== "string") return; + const pending = link.pending.get(frame.id); + // An answer to a request that already timed out. + if (!pending) return; + link.pending.delete(frame.id); + clearTimeout(pending.timer); + if (frame.ok === true) pending.resolve(frame.result ?? null); + else { + const { status, code, message } = failure(frame); + pending.reject(new MaintenanceError(status, code, message)); + } +} + +/** Heartbeat: a ping, or the socket dropped once the host has been silent too long (a laptop that slept, a NAT that forgot it). */ +function beat(link: Link): void { + if (performance.now() - link.seen > maintenanceTimings.offlineMs) { + link.socket.terminate(); + endLink(link); + } else link.socket.send(PING); +} + +/** Forget a link. Requests it still carried may or may not have run on the host: they fail, and none is ever repeated. */ +function retire(link: Link): void { + clearInterval(link.heartbeat); + const { links } = hub(); + if (links.get(link.id) === link) links.delete(link.id); + for (const pending of link.pending.values()) { + clearTimeout(pending.timer); + pending.reject(new MaintenanceError(502, "delivery_unknown", "The maintenance link closed before the host answered; the request may have been applied and is not repeated")); + } + link.pending.clear(); +} + +function endLink(link: Link): void { + retire(link); + changed(); +} + +function closeLink(link: Link, code: number, reason: string): void { + link.socket.close(code, reason); + endLink(link); +} + +function changed(): void { + for (const listener of hub().listeners) { + try { + listener(); + } catch (error) { + console.error(`[omp-web] maintenance listener failed: ${error instanceof Error ? error.stack ?? error.message : String(error)}`); + } + } +} + +/** The host's hello while its maintenance link is up, else null. */ +export function getMaintenanceHost(id: string): MaintenanceHello | null { + return hub().links.get(id)?.hello ?? null; +} + +/** + * One request, answered by the host's result or a MaintenanceError. Never repeated: after a timeout or a lost link + * (504 `timeout`, 502 `delivery_unknown`) the host may or may not have run it. Other failures: 503 `frozen` / `no_link`, + * 501 `unsupported_op` (the host did not advertise it), 413 `too_large`, 429 `busy`, and whatever the host reports. + */ +export async function requestMaintenance(id: string, op: MaintenanceOp, args: Record, timeoutMs: number = maintenanceTimings.requestMs): Promise { + if (hub().frozen) throw new MaintenanceError(503, "frozen", "omp-web is shutting down"); + const link = hub().links.get(id); + if (!link) throw new MaintenanceError(503, "no_link", "The host has no maintenance link"); + if (!link.hello.capabilities.includes(op)) throw new MaintenanceError(501, "unsupported_op", `The host does not support ${op}`); + if (link.pending.size >= MAINTENANCE_LIMITS.inFlight) throw new MaintenanceError(429, "busy", "Too many maintenance requests are in flight to this host"); + const requestId = randomUUID(); + const line = encodeFrame({ type: "req", id: requestId, op, args }); + const { promise, resolve, reject } = Promise.withResolvers(); + const timer = setTimeout(() => { + link.pending.delete(requestId); + reject(new MaintenanceError(504, "timeout", `The host did not answer ${op} in time; it may still be running and is not repeated`)); + }, timeoutMs); + link.pending.set(requestId, { resolve, reject, timer }); + // 0: Bun dropped it, so it never left. + if (link.socket.send(line) === 0) { + clearTimeout(timer); + link.pending.delete(requestId); + reject(new MaintenanceError(503, "no_link", "The maintenance link dropped the request; it was not sent")); + } + return promise; +} + +/** Runs on every change to which hosts have a link (a hello, a closed or replaced socket). */ +export function onMaintenanceChange(listener: () => void): () => void { + const { listeners } = hub(); + listeners.add(listener); + return () => { + listeners.delete(listener); + }; +} + +/** Close a host's maintenance link (a revoked or deleted host); its daemon redials and its token check decides. */ +export function closeMaintenanceHost(id: string): void { + const link = hub().links.get(id); + if (link) closeLink(link, MAINTENANCE_CLOSE.policy, "The host's maintenance link was closed by omp-web"); +} + +/** This server is shutting down: from here on no request or upgrade is accepted and every link closes 1001, so its + * daemon redials the next server. Host children are untouched. */ +export function freezeMaintenance(): void { + hub().frozen = true; + for (const link of [...hub().links.values()]) closeLink(link, MAINTENANCE_CLOSE.goingAway, "omp-web is restarting"); +} diff --git a/lib/hosts/maintenance-protocol.test.mjs b/lib/hosts/maintenance-protocol.test.mjs new file mode 100644 index 000000000..c0c329a42 --- /dev/null +++ b/lib/hosts/maintenance-protocol.test.mjs @@ -0,0 +1,36 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { decodeFrame, encodeFrame, failure, MAINTENANCE_LIMITS, MaintenanceError, parseHello } from "./maintenance-protocol.ts"; + +const hello = (extra = {}) => ({ type: "hello", apiVersion: 1, protocolVersion: 12, ompwebVersion: "0.7.1", ompVersion: null, platform: "linux", capabilities: ["status", "updateSoftware"], ...extra }); + +test("a hello is read whatever the application protocol version, and unknown capabilities are dropped", () => { + assert.deepEqual(parseHello(hello({ protocolVersion: 99, capabilities: ["status", "status", "future", 7] })), hello({ protocolVersion: 99, capabilities: ["status"] })); + assert.equal(parseHello(hello({ ompVersion: "14.0.0" }))?.ompVersion, "14.0.0"); +}); + +test("a hello of another API version or with a wrong field is refused", () => { + for (const bad of [{ apiVersion: 2 }, { protocolVersion: 1.5 }, { protocolVersion: "12" }, { ompwebVersion: 1 }, { ompVersion: 3 }, { platform: "x".repeat(101) }, { capabilities: "status" }]) { + assert.equal(parseHello(hello(bad)), null, JSON.stringify(bad)); + } +}); + +test("decodeFrame refuses what is not a bounded JSON object with a type", () => { + assert.deepEqual(decodeFrame('{"type":"pong"}'), { type: "pong" }); + for (const bad of [new Uint8Array(2), "nope", "[]", "null", '{"id":1}', '{"type":3}']) assert.throws(() => decodeFrame(bad), (e) => e instanceof MaintenanceError && e.status === 400, String(bad)); + assert.throws(() => decodeFrame(JSON.stringify({ type: "x", pad: "x".repeat(MAINTENANCE_LIMITS.frameBytes) })), (e) => e.status === 413 && e.code === "too_large"); +}); + +test("encodeFrame refuses a frame over the limit", () => { + assert.throws(() => encodeFrame({ type: "res", result: "x".repeat(MAINTENANCE_LIMITS.frameBytes) }), (e) => e.status === 413); +}); + +test("failure keeps a native status and code and turns anything else into a 500", () => { + assert.deepEqual(failure(new MaintenanceError(409, "update_busy", "busy")), { status: 409, code: "update_busy", message: "busy" }); + assert.deepEqual(failure({ status: 404, code: "session_not_running", message: "gone" }), { status: 404, code: "session_not_running", message: "gone" }); + assert.deepEqual(failure(Object.assign(new Error("busy"), { code: "host_busy", httpStatus: 409 })), { status: 409, code: "host_busy", message: "busy" }); + assert.deepEqual(failure(new Error("boom")), { status: 500, code: "internal", message: "boom" }); + assert.deepEqual(failure(Object.assign(new Error("x"), { code: "ENOENT", status: 200 })), { status: 500, code: "internal", message: "x" }); + assert.equal(failure({ status: 418, message: "tea" }).code, "host_error"); + assert.equal(failure("text").message, "text"); +}); diff --git a/lib/hosts/maintenance-protocol.ts b/lib/hosts/maintenance-protocol.ts new file mode 100644 index 000000000..6c6842282 --- /dev/null +++ b/lib/hosts/maintenance-protocol.ts @@ -0,0 +1,89 @@ +// Maintenance API v1: a small request/reply WebSocket between a host daemon and the server, versioned apart from the +// application link (link-protocol.ts) so an upgrade still reaches a host whose application protocol differs. +// Self-contained: no application-link imports. Everything the peer sends is untrusted input. +import { isRecord } from "../type-guards"; + +export const MAINTENANCE_API_VERSION = 1; +export const MAINTENANCE_PATH = "/api/hosts/maintenance"; +export const MAINTENANCE_OPS = ["status", "listSessions", "checkUpdates", "updateSoftware"] as const; +export type MaintenanceOp = (typeof MAINTENANCE_OPS)[number]; + +/** The host's first frame. `protocolVersion` (the application link's) is advisory: nothing here depends on it. */ +export type MaintenanceHello = { + type: "hello"; + apiVersion: number; + protocolVersion: number; + ompwebVersion: string; + ompVersion: string | null; + platform: string; + capabilities: MaintenanceOp[]; +}; + +/** `apiMismatch`: the hello's apiVersion is not the server's; its reason is JSON `{"apiVersion": }`. */ +export const MAINTENANCE_CLOSE = { goingAway: 1001, policy: 1008, tooBig: 1009, tryLater: 1013, replaced: 4409, apiMismatch: 4426 } as const; + +/** `frameBytes`: one frame, so a request's args or a result (session inventory metadata, never file contents). + * `inFlight`: requests open at once on one link. */ +export const MAINTENANCE_LIMITS = { frameBytes: 1024 * 1024, inFlight: 8 }; + +/** Mutable so tests shorten them. */ +export const maintenanceTimings = { pingMs: 15_000, offlineMs: 45_000, helloMs: 10_000, requestMs: 30_000, retryMs: 1_000, retryMaxMs: 15_000 }; + +/** A failure the peer reports, or this side raises. Statuses are HTTP's; codes are lowercase snake_case. */ +export class MaintenanceError extends Error { + readonly name = "MaintenanceError"; + readonly status: number; + readonly code: string; + + constructor(status: number, code: string, message: string) { + super(message); + this.status = status; + this.code = code; + } +} + +/** One frame as a text line; a MaintenanceError (413) when it is over the frame limit. */ +export function encodeFrame(frame: object): string { + const line = JSON.stringify(frame); + if (Buffer.byteLength(line) > MAINTENANCE_LIMITS.frameBytes) throw new MaintenanceError(413, "too_large", `A maintenance frame is limited to ${MAINTENANCE_LIMITS.frameBytes} bytes`); + return line; +} + +export const PING = encodeFrame({ type: "ping" }); +export const PONG = encodeFrame({ type: "pong" }); +export const WELCOME = encodeFrame({ type: "welcome", apiVersion: MAINTENANCE_API_VERSION }); + +/** A received frame's JSON object (which has a string `type`); anything else is a protocol violation: a MaintenanceError. */ +export function decodeFrame(data: unknown): Record { + if (typeof data !== "string") throw new MaintenanceError(400, "bad_frame", "Maintenance frames are text"); + if (Buffer.byteLength(data) > MAINTENANCE_LIMITS.frameBytes) throw new MaintenanceError(413, "too_large", `A maintenance frame is limited to ${MAINTENANCE_LIMITS.frameBytes} bytes`); + let frame: unknown; + try { + frame = JSON.parse(data); + } catch { + throw new MaintenanceError(400, "bad_frame", "A maintenance frame is not JSON"); + } + if (!isRecord(frame) || typeof frame.type !== "string") throw new MaintenanceError(400, "bad_frame", "A maintenance frame is an object with a type"); + return frame; +} + +const short = (value: unknown): value is string => typeof value === "string" && value.length <= 100; + +/** A hello's fields once its apiVersion is known to be this API's, else null. Capabilities this side does not know are + * dropped, so a later revision of API 1 may add operations without breaking the handshake. */ +export function parseHello(frame: Record): MaintenanceHello | null { + const { apiVersion, protocolVersion, ompwebVersion, ompVersion, platform, capabilities } = frame; + if (apiVersion !== MAINTENANCE_API_VERSION || typeof protocolVersion !== "number" || !Number.isInteger(protocolVersion)) return null; + if (!short(ompwebVersion) || !short(platform) || !(ompVersion === null || short(ompVersion)) || !Array.isArray(capabilities)) return null; + return { type: "hello", apiVersion: MAINTENANCE_API_VERSION, protocolVersion, ompwebVersion, ompVersion, platform, capabilities: MAINTENANCE_OPS.filter((op) => capabilities.includes(op)) }; +} + +/** The wire form of a failure, read from an Error or a `res` frame alike: a status in 400..599 (`status`, or the + * `httpStatus` of a HostUpdateError) and a lowercase code are kept; anything else is a 500 `internal`. */ +export function failure(error: unknown): { status: number; code: string; message: string } { + const e: Record = isRecord(error) ? error : {}; + const raw = e.status ?? e.httpStatus; + const status = typeof raw === "number" && Number.isInteger(raw) && raw >= 400 && raw <= 599 ? raw : 500; + const code = typeof e.code === "string" && /^[a-z0-9_]{1,64}$/.test(e.code) ? e.code : status === 500 ? "internal" : "host_error"; + return { status, code, message: (typeof e.message === "string" ? e.message : String(error)).slice(0, 500) }; +} diff --git a/lib/hosts/maintenance-transport.test.mjs b/lib/hosts/maintenance-transport.test.mjs new file mode 100644 index 000000000..bf51d6517 --- /dev/null +++ b/lib/hosts/maintenance-transport.test.mjs @@ -0,0 +1,276 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +// The hub and a real DaemonMaintenance over loopback, served through Bun.serve the way server/app.ts does. +const dir = mkdtempSync(join(tmpdir(), "omp-maintenance-transport-")); +const previousAgentDir = process.env.PI_CODING_AGENT_DIR; +process.env.PI_CODING_AGENT_DIR = dir; +const hub = await import("./maintenance-hub.ts"); +const { DaemonMaintenance } = await import("./daemon-maintenance.ts"); +const { MAINTENANCE_CLOSE, MAINTENANCE_LIMITS, MAINTENANCE_OPS, maintenanceTimings } = await import("./maintenance-protocol.ts"); +const { createHost, revokeHost } = await import("./registry.ts"); +const timings = { ...maintenanceTimings }; +Object.assign(maintenanceTimings, { retryMs: 20, retryMaxMs: 60, helloMs: 1000 }); + +const seen = { discoveries: 0, upgrades: 0 }; +const server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + fetch(request, bun) { + if (new URL(request.url).pathname !== "/api/hosts/maintenance") return new Response("app", { headers: { "content-type": "text/html" } }); + seen[request.headers.get("upgrade") === "websocket" ? "upgrades" : "discoveries"] += 1; + return hub.handleMaintenanceUpgrade(request, (data) => bun.upgrade(request, { data }), { protocolVersion: 12, ompwebVersion: "0.7.1" }); + }, + websocket: hub.maintenanceSocket, +}); + +test.after(async () => { + await server.stop(true); + Object.assign(maintenanceTimings, timings); + if (previousAgentDir === undefined) delete process.env.PI_CODING_AGENT_DIR; + else process.env.PI_CODING_AGENT_DIR = previousAgentDir; + rmSync(dir, { recursive: true, force: true }); +}); + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +async function until(predicate, what = "the condition") { + const end = Date.now() + 5000; + while (!predicate()) { + if (Date.now() > end) throw new Error(`timed out waiting for ${what}`); + await sleep(10); + } +} +/** The error a request fails with. */ +const failed = (promise) => promise.then(() => assert.fail("expected a rejection"), (error) => error); +const outcome = (promise) => promise.then((value) => ({ value }), (error) => ({ error })); + +const host = () => createHost("h"); +const helloFor = (extra) => () => ({ type: "hello", apiVersion: 1, protocolVersion: 12, ompwebVersion: "0.7.1", ompVersion: "14.0.0", platform: "linux", capabilities: [...MAINTENANCE_OPS], ...extra }); + +/** A daemon end dialing `url`; `handle` answers its requests. */ +function daemon(t, { token, url = `http://127.0.0.1:${server.port}`, handle = async () => ({}), hello } = {}) { + const log = []; + const states = []; + const maintenance = new DaemonMaintenance({ url, token }, { hello: helloFor(hello), handleRequest: handle, log: (text) => log.push(text), onState: (up) => states.push(up) }); + t.after(() => maintenance.stop()); + maintenance.start(); + return { maintenance, log, states }; +} +const linkedHost = (t, options) => { + const { host: record, token } = host(); + return { id: record.id, token, ...daemon(t, { token, ...options }) }; +}; + +/** A server that is not ours to control: `discovery()` answers the GET, `onHello(ws)` the socket's hello. */ +function fake(t, { discovery = () => Response.json({ apiVersion: 1 }), onHello = (ws) => ws.send(JSON.stringify({ type: "welcome", apiVersion: 1 })) } = {}) { + const state = { probes: 0, connections: 0, frames: [], socket: null, say: (frame) => state.socket.send(JSON.stringify(frame)) }; + const raw = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + fetch(request, bun) { + if (request.headers.get("upgrade") !== "websocket") { + state.probes += 1; + return discovery(request); + } + return bun.upgrade(request) ? undefined : new Response("no", { status: 426 }); + }, + websocket: { + open(ws) { state.connections += 1; state.socket = ws; }, + message(ws, data) { + const frame = JSON.parse(data); + state.frames.push(frame); + if (frame.type === "hello") onHello(ws); + }, + close() {}, + }, + }); + t.after(() => raw.stop(true)); + state.url = `http://127.0.0.1:${raw.port}`; + return state; +} + +test("answers status, list, check and update requests whatever the application protocol", async (t) => { + const calls = []; + const { id } = linkedHost(t, { handle: async (op, args) => { calls.push([op, args]); return { op }; }, hello: { protocolVersion: 99, apiVersion: 7 } }); + await until(() => hub.getMaintenanceHost(id), "the link"); + assert.equal(hub.getMaintenanceHost(id).protocolVersion, 99); + assert.equal(hub.getMaintenanceHost(id).apiVersion, 1); + assert.ok(seen.discoveries > 0); + for (const op of MAINTENANCE_OPS) assert.deepEqual(await hub.requestMaintenance(id, op, { n: 1 }), { op }); + assert.deepEqual(calls, MAINTENANCE_OPS.map((op) => [op, { n: 1 }])); + // A handler that returns nothing answers null. + const silent = linkedHost(t, { handle: async () => undefined }); + await until(() => hub.getMaintenanceHost(silent.id), "the link"); + assert.equal(await hub.requestMaintenance(silent.id, "status", {}), null); +}); + +test("keeps the native status and code of a host error and stays usable", async (t) => { + const errors = [ + Object.assign(new Error("A session is running"), { code: "host_busy", httpStatus: 409 }), + Object.assign(new Error("Gone"), { code: "session_not_found", status: 404 }), + new Error("boom"), + ]; + let next = 0; + const { id } = linkedHost(t, { handle: async (op) => { if (op === "status") return "x".repeat(MAINTENANCE_LIMITS.frameBytes); throw errors[next++]; } }); + await until(() => hub.getMaintenanceHost(id), "the link"); + const shape = (error) => [error.status, error.code, error.message]; + assert.deepEqual(shape(await failed(hub.requestMaintenance(id, "updateSoftware", {}))), [409, "host_busy", "A session is running"]); + assert.deepEqual(shape(await failed(hub.requestMaintenance(id, "checkUpdates", {}))), [404, "session_not_found", "Gone"]); + assert.deepEqual(shape(await failed(hub.requestMaintenance(id, "listSessions", {}))), [500, "internal", "boom"]); + assert.deepEqual((await failed(hub.requestMaintenance(id, "status", {}))).code, "too_large"); + assert.ok(hub.getMaintenanceHost(id)); +}); + +test("answers an unknown or unadvertised op, a bad args envelope and a duplicate id without dropping the link", async (t) => { + const release = Promise.withResolvers(); + const state = fake(t); + const { states } = daemon(t, { token: "t", url: state.url, hello: { capabilities: ["status"] }, handle: async () => { await release.promise; return "ok"; } }); + await until(() => states[0] === true, "the link"); + const replies = () => state.frames.filter((frame) => frame.type === "res"); + const replyTo = (id) => replies().find((frame) => frame.id === id); + state.say({ type: "req", id: "a", op: "nope", args: {} }); + state.say({ type: "req", id: "b", op: "updateSoftware", args: {} }); + state.say({ type: "req", id: "c", op: "status", args: [] }); + state.say({ type: "req", id: "d", op: "status" }); + await until(() => ["a", "b", "c", "d"].every((id) => replyTo(id)), "the replies"); + assert.deepEqual([replyTo("a"), replyTo("b")].map((r) => [r.ok, r.status, r.code]), [[false, 501, "unsupported_op"], [false, 501, "unsupported_op"]]); + assert.deepEqual([replyTo("c"), replyTo("d")].map((r) => [r.ok, r.status, r.code]), [[false, 400, "bad_request"], [false, 400, "bad_request"]]); + state.say({ type: "req", id: "e", op: "status", args: {} }); + state.say({ type: "req", id: "e", op: "status", args: {} }); + await until(() => replyTo("e"), "the duplicate's reply"); + assert.deepEqual([replyTo("e").status, replyTo("e").code], [409, "duplicate_request"]); + release.resolve(); + await until(() => replies().some((frame) => frame.id === "e" && frame.ok), "the first one's result"); + assert.deepEqual(states, [true]); + assert.equal(state.connections, 1); +}); + +test("drops a link whose server sends a request before the welcome", async (t) => { + const state = fake(t, { onHello: (ws) => ws.send(JSON.stringify({ type: "req", id: "early", op: "status", args: {} })) }); + const { states } = daemon(t, { token: "t", url: state.url }); + await until(() => state.connections >= 2, "a redial"); + assert.deepEqual(states, []); + assert.equal(state.frames.some((frame) => frame.type === "res"), false); +}); + +test("never opens a socket to a server that is not maintenance API 1, and keeps retrying", async (t) => { + const answers = { + "no endpoint": () => new Response("Not Found", { status: 404 }), + "a page": () => new Response("app", { headers: { "content-type": "text/html" } }), + "an empty 426": () => new Response(null, { status: 426 }), + "another API": () => Response.json({ apiVersion: 2 }), + "a string version": () => Response.json({ apiVersion: "1" }), + "a redirect": (request) => Response.redirect(request.url, 302), + }; + for (const [name, discovery] of Object.entries(answers)) { + const state = fake(t, { discovery }); + const { maintenance, log, states } = daemon(t, { token: "t", url: state.url }); + await until(() => state.probes >= 3, `${name} to be probed again`); + assert.equal(state.connections, 0, name); + assert.deepEqual(states, [], name); + assert.ok(log.length > 0, name); + await maintenance.stop(); + } +}); + +test("retries a refused token and a server error without ending, and links once rekeyed", async (t) => { + const { host: record, token } = host(); + const { maintenance, log, states } = daemon(t, { token: "wrong" }); + const before = seen.discoveries; + await until(() => seen.discoveries >= before + 3, "retries"); + assert.equal(hub.getMaintenanceHost(record.id), null); + assert.ok(log.some((line) => line.includes("refused this host's token")), log.join("\n")); + maintenance.rekey(token); + await until(() => hub.getMaintenanceHost(record.id) && states.includes(true), "both ends to accept the link after rekey"); + assert.deepEqual(states, [true]); +}); + +test("a hello the server refuses as another API is retried, never fatal", async (t) => { + const state = fake(t, { onHello: (ws) => ws.close(MAINTENANCE_CLOSE.apiMismatch, JSON.stringify({ apiVersion: 2 })) }); + const { log, states } = daemon(t, { token: "t", url: state.url }); + await until(() => state.connections >= 3, "redials"); + assert.deepEqual(states, []); + assert.ok(log.some((line) => line.includes("apiVersion")), log.join("\n")); + const welcome2 = fake(t, { onHello: (ws) => ws.send(JSON.stringify({ type: "welcome", apiVersion: 2 })) }); + const second = daemon(t, { token: "t", url: welcome2.url }); + await until(() => welcome2.connections >= 2, "redials"); + assert.deepEqual(second.states, []); +}); + +test("gives up on a server that goes silent and dials again", async (t) => { + maintenanceTimings.offlineMs = 80; + t.after(() => { maintenanceTimings.offlineMs = timings.offlineMs; }); + const state = fake(t); + const { states } = daemon(t, { token: "t", url: state.url }); + await until(() => states.length >= 2, "the watchdog"); + assert.deepEqual(states.slice(0, 2), [true, false]); + await until(() => state.connections >= 2, "the redial"); +}); + +test("does not repeat a request that was in flight when the link dropped", async (t) => { + const gate = Promise.withResolvers(); + let calls = 0; + const { id, states } = linkedHost(t, { handle: async () => { calls += 1; if (calls === 1) await gate.promise; return calls; } }); + await until(() => hub.getMaintenanceHost(id), "the link"); + const lost = outcome(hub.requestMaintenance(id, "updateSoftware", {})); + await until(() => calls === 1, "the handler to start"); + hub.closeMaintenanceHost(id); + assert.equal((await lost).error.code, "delivery_unknown"); + await until(() => states.join() === "true,false,true", "the redial"); + await until(() => hub.getMaintenanceHost(id), "the new link"); + await sleep(50); + assert.equal(calls, 1); + // The old connection's late result goes nowhere; the new link works. + gate.resolve(); + await sleep(30); + assert.equal(await hub.requestMaintenance(id, "status", {}), 2); + assert.equal(calls, 2); +}); + +test("stop closes the link, drops what is running and never dials again", async (t) => { + const gate = Promise.withResolvers(); + const { id, maintenance, states } = linkedHost(t, { handle: async () => { await gate.promise; return "late"; } }); + await until(() => hub.getMaintenanceHost(id), "the link"); + const pending = outcome(hub.requestMaintenance(id, "status", {})); + await sleep(30); + await maintenance.stop(); + assert.deepEqual(states, [true, false]); + assert.equal((await pending).error.code, "delivery_unknown"); + await until(() => hub.getMaintenanceHost(id) === null, "the server to see the close"); + const dials = seen.discoveries; + gate.resolve(); + await sleep(150); + assert.equal(seen.discoveries, dials); + assert.deepEqual(states, [true, false]); +}); + +test("a revoked host's daemon keeps retrying and is not let back in", async (t) => { + const { id, states } = linkedHost(t); + await until(() => hub.getMaintenanceHost(id), "the link"); + revokeHost(id); + hub.closeMaintenanceHost(id); + await until(() => states.join() === "true,false", "the drop"); + const dials = seen.discoveries; + await until(() => seen.discoveries >= dials + 3, "retries"); + assert.equal(hub.getMaintenanceHost(id), null); +}); + +test("freezing the server closes links and refuses dials until it restarts, without ending the daemon", async (t) => { + t.after(() => { globalThis.__ompMaintenanceHub = undefined; }); + const { id, states } = linkedHost(t); + await until(() => hub.getMaintenanceHost(id), "the link"); + hub.freezeMaintenance(); + await until(() => states.join() === "true,false", "the close"); + const upgrades = seen.upgrades; + const dials = seen.discoveries; + await until(() => seen.discoveries >= dials + 3, "refused dials"); + assert.equal(seen.upgrades, upgrades); + assert.equal(hub.getMaintenanceHost(id), null); + // The next server process starts with fresh state. + globalThis.__ompMaintenanceHub = undefined; + await until(() => hub.getMaintenanceHost(id), "the link on the restarted server"); + assert.deepEqual(states, [true, false, true]); +}); diff --git a/lib/hosts/update-worker.test.mjs b/lib/hosts/update-worker.test.mjs index abb855bdf..cee87cac1 100644 --- a/lib/hosts/update-worker.test.mjs +++ b/lib/hosts/update-worker.test.mjs @@ -158,6 +158,20 @@ test("owned descriptor is required and a symlink is refused before execution", { assert.equal(alive(f.daemon.pid), true); }); +test("repository metadata remains required for installation and malformed metadata cannot authorize a restart", async () => { + for (const restartOnly of [false, true]) { + const f = await fixture(`repository-boundary-${restartOnly}`, { restartOnly }); + const descriptor = read(f.descriptorPath); + if (restartOnly) descriptor.releaseRepo = "not/a/repository"; + else delete descriptor.releaseRepo; + writeFileSync(f.descriptorPath, JSON.stringify(descriptor), { mode: 0o600 }); + assert.equal(await worker.runHostUpdateWorker(f.descriptorPath), 1); + assert.match(status().operation.error, /descriptor/); + assert.equal(alive(f.daemon.pid), true); + assert.equal(versionOf(f.install), restartOnly ? "2.0.0" : "1.0.0"); + } +}); + test("binary archives reject path traversal, links and excessive sizes; a release with its omp runner fits", () => { const tar = new Uint8Array(2048); function entry(name, type = "0", size = "00000000000") { @@ -562,3 +576,14 @@ test("pruning leaves everything alone while an update is running and keeps the f assert.equal(existsSync(join(parent, ".ompweb-failed-running-a")), false); assert.equal(existsSync(join(stateRoot, "keep-b", "runtime")), false); }); + +test("a legacy restart-only descriptor needs no release repository and still verifies the new daemon", async () => { + const f = await fixture("legacy-restart-only", { restartOnly: true }); + const descriptor = read(f.descriptorPath); + delete descriptor.releaseRepo; + writeFileSync(f.descriptorPath, JSON.stringify(descriptor), { mode: 0o600 }); + assert.equal(await worker.runHostUpdateWorker(f.descriptorPath), 0, outcome(f)); + assert.equal(read(join(f.attempt, "health.json")).version, "2.0.0"); + assert.equal(existsSync(f.backup), false); + assert.equal(alive(f.daemon.pid), false); +}); diff --git a/lib/hosts/update-worker.ts b/lib/hosts/update-worker.ts index 792654027..ce32204f9 100644 --- a/lib/hosts/update-worker.ts +++ b/lib/hosts/update-worker.ts @@ -43,8 +43,8 @@ export interface HostUpdateDescriptor { managerPath?: string; restartOnly?: boolean; systemd: { unit: string; user: boolean } | null; - /** `owner/name` whose releases the binary comes from (lib/release-repo.ts; this file stays self-contained). */ - releaseRepo: string; + /** Binary release source; absent only on a legacy restart-only descriptor, which downloads nothing. */ + releaseRepo?: string; } export function hostUpdateRoot(): string { @@ -576,7 +576,9 @@ function validateDescriptor(path: string): HostUpdateDescriptor { const root = ensureHostUpdateRoot(); assertOwnedPath(path); const d = JSON.parse(readFileSync(path, "utf8")) as HostUpdateDescriptor; - if (!/^[a-zA-Z0-9-]{1,80}$/.test(d.id) || resolve(path) !== join(root, d.id, "descriptor.json") || !HOST_VERSION_PATTERN.test(d.version) || !HOST_VERSION_PATTERN.test(d.fromVersion) || d.target !== "ompweb" || !["npm", "bun", "binary"].includes(d.installMethod) || !isAbsolute(d.installDir) || !isAbsolute(d.runtime) || !isAbsolute(d.cwd) || !Number.isSafeInteger(d.daemonPid) || d.daemonPid <= 1 || !Array.isArray(d.hostArgs) || d.hostArgs.some((arg) => arg !== "--allow-http") || typeof d.releaseRepo !== "string" || !/^[\w.-]+\/[\w.-]+$/.test(d.releaseRepo)) throw new Error("Invalid host update descriptor"); + if (!/^[a-zA-Z0-9-]{1,80}$/.test(d.id) || resolve(path) !== join(root, d.id, "descriptor.json") || !HOST_VERSION_PATTERN.test(d.version) || !HOST_VERSION_PATTERN.test(d.fromVersion) || d.target !== "ompweb" || !["npm", "bun", "binary"].includes(d.installMethod) || !isAbsolute(d.installDir) || !isAbsolute(d.runtime) || !isAbsolute(d.cwd) || !Number.isSafeInteger(d.daemonPid) || d.daemonPid <= 1 || !Array.isArray(d.hostArgs) || d.hostArgs.some((arg) => arg !== "--allow-http")) throw new Error("Invalid host update descriptor"); + // A v10 daemon copies this new worker after the shared web installation has already been replaced. + if (d.releaseRepo === undefined ? d.restartOnly !== true : typeof d.releaseRepo !== "string" || !/^[\w.-]+\/[\w.-]+$/.test(d.releaseRepo)) throw new Error("Invalid host update descriptor"); if (d.restartOnly !== undefined && typeof d.restartOnly !== "boolean") throw new Error("Invalid host restart-only descriptor"); if (!d.systemd && (typeof d.daemonIdentity !== "string" || !d.daemonIdentity || d.daemonIdentity.length > 200)) throw new Error("Invalid host update descriptor: the daemon's process identity is missing"); assertOwnedPath(dirname(path), true); diff --git a/lib/hosts/upgrade-shim.test.mjs b/lib/hosts/upgrade-shim.test.mjs new file mode 100644 index 000000000..0c2891142 --- /dev/null +++ b/lib/hosts/upgrade-shim.test.mjs @@ -0,0 +1,109 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test, { after } from "node:test"; + +const dir = mkdtempSync(join(tmpdir(), "ompweb-upgrade-shim-")); +const previous = process.env.PI_CODING_AGENT_DIR; +process.env.PI_CODING_AGENT_DIR = dir; +const hub = await import("./hub.ts"); +const { linkHost, upgradeSocket, until } = await import("./link-driver.test-helper.mjs"); +const { LINK_PROTOCOL_VERSION } = await import("./link-protocol.ts"); +const { getRpcSession } = await import("../rpc-manager.ts"); +after(() => { + hub.flushLinkSamples(); + if (previous === undefined) delete process.env.PI_CODING_AGENT_DIR; + else process.env.PI_CODING_AGENT_DIR = previous; + rmSync(dir, { recursive: true, force: true }); +}); + +const sid = "11111111-2222-4333-8444-5555555555bb"; +const channels = [{ ch: "old-channel", sessionId: sid, cwd: "/w", pending: ['{"type":"agent_start"}'] }]; +const operation = { id: "update-attempt", target: "ompweb", version: "0.7.1", state: "running" }; +const info = { + ompweb: { currentVersion: "0.6.0", availableVersion: "0.7.1", updateAvailable: true, canUpdate: true, blockedReason: null, updateCommand: "npm install -g @kahme247/ompweb", installMethod: "npm", protocolVersion: 12 }, + omp: { currentVersion: "18.20.0" }, operation: null, +}; + +for (const v of [10, 11]) test(`v${v} is welcomed as itself solely for its unchanged updater`, async (t) => { + const hellos = [], sessionsChanged = [], other = [], requests = []; + t.after(hub.onHostHello((id) => hellos.push(id))); + t.after(hub.onHostSessionsChanged((id) => sessionsChanged.push(id))); + const link = await linkHost(`Legacy v${v}`, { channels, hello: { v, ompwebVersion: "0.6.0" } }); + t.after(() => link.drop()); + assert.equal(link.welcome.v, v, "the old daemon rejects a welcome above its own protocol"); + const row = () => hub.listHosts().find((host) => host.id === link.id); + assert.equal(row().status, "needs-upgrade"); + assert.equal(row().upgradeOnly, true); + assert.deepEqual(row().workspaceOps, []); + assert.equal(getRpcSession(sid), undefined, "the shim must not adopt legacy sessions"); + link.serve({ + checkUpdates: (args) => { requests.push(["checkUpdates", args]); return info; }, + updateSoftware: (args) => { requests.push(["updateSoftware", args]); return { accepted: true, operation }; }, + }, (message) => other.push(message)); + assert.equal((await hub.getHostSoftwareUpdates(link.id, true)).ompweb.canUpdate, true); + assert.deepEqual(await hub.updateHostSoftware(link.id, "ompweb", "0.7.1"), { accepted: true, operation }); + assert.ok(requests.some(([op]) => op === "updateSoftware")); + for (const [, args] of requests) assert.deepEqual(args.compatibility, { min: 12, max: 12 }); + await assert.rejects(hub.getHost(link.id).sessionFileExists("/w/session.jsonl"), { code: "host_upgrade_required" }); + await link.post( + { type: "channels_changed", channels }, + { type: "stdout", ch: "old-channel", line: '{"type":"agent_start"}' }, + { type: "sessions_changed" }, + ); + // Give deferred imports and adoption attempts a quiet window in which to expose an accidental dispatch. + await new Promise(resolve => setTimeout(resolve, 50)); + assert.equal(getRpcSession(sid), undefined, "legacy channel announcements must not adopt sessions either"); + assert.deepEqual(hellos, [], "legacy hello must not trigger application startup hooks"); + assert.deepEqual(sessionsChanged, [], "legacy events must not enter application discovery"); + assert.deepEqual(other, [], "no open, kill, config sync or application command may reach the old daemon"); + link.drop(); + await until(() => !row().upgradeOnly, "the closed shim stops advertising update capability"); +}); + +test("a legacy shim cannot replace a healthy application connection", async (t) => { + const link = await linkHost("Current host"); + t.after(() => link.drop()); + const { socket } = upgradeSocket(link.token); + socket.frame(JSON.stringify({ type: "hello", v: 10, ompVersion: null, platform: "linux", home: "/tmp", autoApproveExtension: false, ack: 0, channels: [] })); + assert.equal(socket.closed?.code, 4426); + assert.equal(link.socket.closed, null); + const row = hub.listHosts().find((host) => host.id === link.id); + assert.equal(row.status, "online"); + assert.equal(row.hostVersion, LINK_PROTOCOL_VERSION); + assert.ok(!row.upgradeOnly); +}); + +test("a shim reconnect never replays an update whose delivery is uncertain", async (t) => { + const link = await linkHost("Interrupted update", { hello: { v: 10 } }); + t.after(() => link.drop()); + const updating = hub.updateHostSoftware(link.id, "ompweb", "0.7.1"); + const rejected = assert.rejects(updating, { code: "transport_lost" }); + const check = await link.next(); + assert.equal(check.op, "checkUpdates"); + await link.answer(check, info); + const update = await link.next(); + assert.equal(update.op, "updateSoftware"); + await link.dial({ hello: { v: 10 } }); + await rejected; + assert.ok(link.socket.frames.every(line => JSON.parse(line).type !== "req"), "the old update must not run again"); +}); + +test("unacknowledged application input never crosses into a legacy shim", async (t) => { + const link = await linkHost("Protocol downgrade"); + t.after(() => link.drop()); + const spawning = hub.getHost(link.id).spawnRpc({ cwd: "/w", recordedCwd: null, sessionFile: "", advisor: false }, null); + const open = await link.next(); + assert.equal(open.op, "open"); + await link.answer(open, { ch: "full-channel", joined: false, pending: [], streaming: null, advisorSpawned: false, cwd: "/w", fellBack: false, launchAdvisor: false, autoApproveExtension: false }); + const { transport } = await spawning; + transport.write('{"t":"cmd","id":1,"op":"prompt","args":{"message":"must not replay"}}\\n', () => {}); + const stdin = await link.next(); + assert.equal(stdin.type, "stdin"); + link.drop(); + await new Promise(resolve => setImmediate(resolve)); + await link.dial({ hello: { v: 11 } }); + assert.equal(link.welcome.v, 11); + assert.ok(link.socket.frames.every(line => ["welcome", "ack", "ping", "pong"].includes(JSON.parse(line).type))); +}); diff --git a/lib/hosts/v10-upgrade.integration.test.mjs b/lib/hosts/v10-upgrade.integration.test.mjs new file mode 100644 index 000000000..a2beddb1b --- /dev/null +++ b/lib/hosts/v10-upgrade.integration.test.mjs @@ -0,0 +1,362 @@ +// Real pre-cutover daemons (link protocol 10 and 11) upgrading to this checkout through this checkout's hub. +// +// Nothing of the old side is imitated. The daemon is the published source, extracted from git at a pinned commit into +// a disposable npm-layout install and started as `node bin/omp-web-host.js`; its welcome check, its `checkUpdates` / +// `updateSoftware` handlers and its updater engine (candidate protocol check, install, swap, restart, health) all run +// unchanged. Only delivery is disposable: a fetch preload answers the npm registry and the raw-GitHub link-protocol.ts +// (the worktree's real file, parsed by the OLD updater's own regex), and a fake `npm` "installs" a copy of this +// checkout as the release. The hub is the real hub behind the real /api/hosts/link route. No user service is touched. +// +// Pinned sources (both reachable from main; the test skips, naming the commit, when a shallow clone lacks it): +// v10 546a945df0d1716a7a135d6e3221ba885b00171e package 0.6.0, link 10. The last commit before 8ec9208d (runner +// channels); published as prerelease 20261113T113829Z-546a945df0. v10 first appeared at a53494c4. Its updater +// takes target "ompweb" or "omp" and has no preflight. +// v11 3a56e79b217a3949f7180069e6769bc3fa031167 package 0.7.0, link 11: the commit tag v0.7.0 points at. +// +// Run: bun test --timeout 240000 lib/hosts/v10-upgrade.integration.test.mjs +// Needs: full git history (`git cat-file -e `), `tar` and `git` on PATH, node >= 24 on PATH, `npm ci` done in this +// checkout (daemon dependencies are symlinked from its node_modules), a POSIX machine (Linux or macOS). +// Disposable stand-ins besides delivery: one POSIX `omp` script (OMP_WEB_OMP_BIN) that answers `--version`, `--check` and +// the session start with the runner's channel hello, so the v11 updater's real `fetch-omp` and `preflight` of the candidate +// pass without the runner binary (no `bun runner/scripts/build-binary.ts` needed) or any network. +// Not covered: compiled-binary installs, systemd/launchd-supervised hosts, Windows, a daemon with live sessions (a legacy +// link carries none; the old busy checks are its own unchanged code). The server side serves only /api/hosts/link, so the +// upgraded daemon's maintenance connection is refused there; its application link and health acknowledgement are real. +// Cleanup: each scenario kills every process whose command line names its directory (daemons, detached workers, the +// stand-in sharing the install) and the file kills the rest; the run fails if any survives. A crash of the test runner +// itself can leave detached daemons running under the system temp directory (v10-upgrade-it-*): kill them by that path. +import assert from "node:assert/strict"; +import test, { after } from "node:test"; +import { execFileSync, spawn, spawnSync } from "node:child_process"; +import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { delimiter, dirname, join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { nodeBinary, repo, repoVersion, spawnNodeOutsideService } from "./update-fixture.test-helper.mjs"; + +const LEGACY = { + v10: { name: "v10", link: 10, version: "0.6.0", commit: "546a945df0d1716a7a135d6e3221ba885b00171e" }, + v11: { name: "v11", link: 11, version: "0.7.0", commit: "3a56e79b217a3949f7180069e6769bc3fa031167" }, +}; +/** The application link version this release speaks, and the only target compatibility an old updater may be given. */ +const APPLICATION = 12; + +const base = realpathSync(mkdtempSync(join(tmpdir(), "v10-upgrade-it-"))); +const previousAgentDir = process.env.PI_CODING_AGENT_DIR; +process.env.PI_CODING_AGENT_DIR = join(base, "main-agent"); + +const linkRoute = await import("../../app/api/hosts/link/route.ts"); +const hub = await import("./hub.ts"); +const { createHost } = await import("./registry.ts"); +const { closeNotificationStore } = await import("../notification-store.ts"); +const { CHANNEL_PROTO } = await import("./omp-socket.ts"); + +const wait = (ms) => new Promise((done) => setTimeout(done, ms)); +const read = (path) => JSON.parse(readFileSync(path, "utf8")); +function alive(pid) { + try { process.kill(pid, 0); return true; } catch (error) { return error.code === "EPERM"; } +} + +// Every line the hub sends each host, in order, so the old daemon's whole view of the server can be asserted. +const down = new Map(); +const wrappers = new WeakMap(); +/** The hub keys its state on socket identity: one stable wrapper per socket, recording what it sends. */ +function wrap(ws) { + let wrapper = wrappers.get(ws); + if (wrapper) return wrapper; + const send = (frame, ...rest) => { + if (typeof frame === "string") { + for (const line of frame.split("\n")) if (line) down.set(ws.data.hostId, [...(down.get(ws.data.hostId) ?? []), JSON.parse(line)]); + } + return ws.send(frame, ...rest); + }; + wrapper = new Proxy(ws, { get: (target, name) => name === "send" ? send : typeof target[name] === "function" ? target[name].bind(target) : target[name] }); + wrappers.set(ws, wrapper); + return wrapper; +} +const server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + routes: { + "/api/hosts/link": { + GET: (req) => linkRoute.GET(req, { upgrade: (data) => server.upgrade(req, { data }) }), + POST: (req) => linkRoute.POST(req), + }, + }, + websocket: { + ...hub.hostLinkSocket, + message: (ws, data) => hub.hostLinkSocket.message(wrap(ws), data), + drain: (ws) => hub.hostLinkSocket.drain(wrap(ws)), + close: (ws) => hub.hostLinkSocket.close(wrap(ws)), + }, +}); + +/** Pids of every process whose command line names `dir`: daemons, workers and stand-ins all run files from inside it. */ +function processesUnder(dir) { + const lines = process.platform === "linux" + ? readdirSync("/proc").filter((name) => /^\d+$/.test(name)).flatMap((name) => { + try { return [[Number(name), readFileSync(`/proc/${name}/cmdline`, "utf8").replaceAll("\0", " ")]]; } catch { return []; } + }) + : spawnSync("ps", ["-axo", "pid=,command="], { encoding: "utf8" }).stdout.split("\n").flatMap((line) => { + const match = /^\s*(\d+)\s+(.*)$/.exec(line); + return match ? [[Number(match[1]), match[2]]] : []; + }); + return lines.filter(([pid, command]) => pid !== process.pid && command.includes(dir)).map(([pid]) => pid); +} +async function stopAll(dir) { + for (const signal of ["SIGTERM", "SIGKILL"]) { + for (const pid of processesUnder(dir)) { try { process.kill(pid, signal); } catch { /* already gone */ } } + for (const deadline = Date.now() + 10_000; processesUnder(dir).length > 0 && Date.now() < deadline;) await wait(100); + } + assert.deepEqual(processesUnder(dir), [], "every process this test started must exit before its directory is removed"); +} + +after(async () => { + await stopAll(base); + await server.stop(true); + closeNotificationStore(); + hub.flushLinkSamples(); + if (previousAgentDir === undefined) delete process.env.PI_CODING_AGENT_DIR; + else process.env.PI_CODING_AGENT_DIR = previousAgentDir; + for (let tries = 0; ; tries++) { + try { return rmSync(base, { recursive: true, force: true }); } catch (error) { + if (tries === 50 || !["EBUSY", "ENOTEMPTY", "EPERM"].includes(error.code)) throw error; + await wait(100); + } + } +}); + +const haveCommit = (commit) => spawnSync("git", ["-C", repo, "cat-file", "-e", `${commit}^{commit}`]).status === 0; + +const sources = new Map(); +/** `bin`, `lib` and package.json of the pinned commit: everything `omp-web-host.js` loads. */ +function legacySource(legacy) { + if (sources.has(legacy.commit)) return sources.get(legacy.commit); + const dir = join(base, `source-${legacy.name}`); + mkdirSync(dir); + const archive = `${dir}.tar`; + // umask 0022: the updater refuses an install directory that is group-writable. + execFileSync("git", ["-C", repo, "-c", "tar.umask=0022", "archive", "--format=tar", "-o", archive, legacy.commit, "bin", "lib", "package.json"]); + execFileSync("tar", ["-xf", archive, "-C", dir, "--no-same-owner"]); + // The pin must be the daemon this test claims to run. + assert.equal(read(join(dir, "package.json")).version, legacy.version); + assert.match(readFileSync(join(dir, "lib", "hosts", "link-protocol.ts"), "utf8"), new RegExp(`^export const LINK_PROTOCOL_VERSION = ${legacy.link};$`, "m")); + sources.set(legacy.commit, dir); + return dir; +} + +let releaseDir; +/** This checkout as the release the fake npm delivers: what `npm install @kahme247/ompweb@` would unpack. */ +function release() { + if (releaseDir) return releaseDir; + releaseDir = join(base, "release"); + mkdirSync(releaseDir); + cpSync(join(repo, "bin"), join(releaseDir, "bin"), { recursive: true }); + cpSync(join(repo, "lib"), join(releaseDir, "lib"), { recursive: true, filter: (src) => !/\.test(-helper)?\.(mjs|ts|tsx)$/.test(src) }); + cpSync(join(repo, "package.json"), join(releaseDir, "package.json")); + return releaseDir; +} + +const preload = join(base, "release-delivery.mjs"); +// The only network the old updater sees: the npm registry's latest version and the target tag's raw link-protocol.ts. +writeFileSync(preload, `import { appendFileSync, existsSync, readFileSync } from "node:fs"; +const real = globalThis.fetch; +const dir = process.env.UPGRADE_FIXTURE_DIR; +globalThis.fetch = async (input, init) => { + const url = String(input instanceof Request ? input.url : input); + appendFileSync(dir + "/fetch.log", url + "\\n"); + if (url.startsWith("https://registry.npmjs.org/")) return Response.json({ version: process.env.UPGRADE_FIXTURE_VERSION }); + if (/^https:\\/\\/raw\\.githubusercontent\\.com\\/[^/]+\\/[^/]+\\/v[^/]+\\/lib\\/hosts\\/link-protocol\\.ts$/.test(url)) { + const override = dir + "/link-protocol.override"; + return new Response(readFileSync(existsSync(override) ? override : process.env.UPGRADE_FIXTURE_PROTOCOL_SOURCE)); + } + if (/^http:\\/\\/(127\\.0\\.0\\.1|localhost|\\[::1\\])[:/]/.test(url)) return real(input, init); + throw new Error("upgrade fixture: no network access to " + url); +}; +`); + +/** Installs the pinned daemon as a real npm-layout host, logs it into the hub and starts it. Null when no scope is available. */ +async function startLegacy(legacy, layout) { + const work = join(base, `${legacy.name}-${layout}`); + const modules = join(work, "node_modules"); + const install = join(modules, "@kahme247", "ompweb"); + const home = join(work, "home"); + const agent = join(home, ".omp", "agent"); + mkdirSync(join(work, "bin"), { recursive: true }); + mkdirSync(modules, { recursive: true }); + mkdirSync(agent, { recursive: true }); + // The daemon's dependencies, resolved from the node_modules above the install as for any hoisted global install. + for (const name of readdirSync(join(repo, "node_modules"))) { + if (!name.startsWith(".")) symlinkSync(realpathSync(join(repo, "node_modules", name)), join(modules, name), "dir"); + } + cpSync(legacySource(legacy), install, { recursive: true }); + for (let dir = install; dir !== dirname(work); dir = dirname(dir)) chmodSync(dir, 0o755); + const fakeOmp = join(work, "fake-omp"); + // One stand-in for both omp roles an update touches: the version/update probes of the old daemon, and the session omp + // that the candidate's `ompweb preflight` (run by the v11 updater before its switch) must see say hello. + writeFileSync(fakeOmp, `#!/bin/sh +if [ "$1" = "--version" ]; then echo 14.0.0-fake; exit 0; fi +case "$*" in *--check*) echo "Current version: 14.0.0-fake"; exit 0;; esac +case "$*" in *"--headless --cwd "*"--no-session"*) echo '{"t":"hello","proto":${CHANNEL_PROTO}}'; exec cat >/dev/null;; esac +exit 0 +`, { mode: 0o755 }); + writeFileSync(join(work, "idle.js"), "setInterval(() => {}, 1000);\n"); + writeFileSync(join(work, "bin", "npm"), `#!/usr/bin/env node +const fs = require("node:fs"), path = require("node:path"); +const args = process.argv.slice(2); +fs.appendFileSync(${JSON.stringify(join(work, "npm.log"))}, JSON.stringify(args) + "\\n"); +if (!args.includes(${JSON.stringify(`@kahme247/ompweb@${repoVersion}`)})) process.exit(8); +fs.cpSync(${JSON.stringify(release())}, path.join(args[args.indexOf("--prefix") + 1], "node_modules", "@kahme247", "ompweb"), { recursive: true }); +`, { mode: 0o700 }); + + const { host, token } = createHost(`${legacy.name} ${layout}`); + const env = { + ...process.env, HOME: home, USERPROFILE: home, PI_CODING_AGENT_DIR: agent, OMP_WEB_OMP_BIN: fakeOmp, OMP_WEB_PACKAGE_DIR: install, + PATH: join(work, "bin") + delimiter + process.env.PATH, + NODE_OPTIONS: [process.env.NODE_OPTIONS, `--import ${pathToFileURL(preload).href}`].filter(Boolean).join(" "), + UPGRADE_FIXTURE_DIR: work, UPGRADE_FIXTURE_VERSION: repoVersion, UPGRADE_FIXTURE_PROTOCOL_SOURCE: join(repo, "lib", "hosts", "link-protocol.ts"), + }; + delete env.OMP_WEB_DISABLE_AUTOUPDATE; + delete env.OMP_WEB_HOST_DRAINED; + const entry = join(install, "bin", "omp-web-host.js"); + const login = spawnSync(nodeBinary(), [entry, "login", "--url", `http://127.0.0.1:${server.port}`], { env, cwd: work, input: `${token}\n`, encoding: "utf8" }); + assert.equal(login.status, 0, login.stderr); + + const child = await spawnNodeOutsideService(entry, env); + if (!child) return null; + let log = ""; + child.stdout.on("data", (data) => { log += data; }); + child.stderr.on("data", (data) => { log += data; }); + const lines = (name) => existsSync(join(work, name)) ? readFileSync(join(work, name), "utf8").split("\n").filter(Boolean) : []; + return { + legacy, layout, work, install, child, hostId: host.id, stateRoot: join(agent, "omp-web", "host-update"), + log: () => log, + npmCalls: () => lines("npm.log").map((line) => JSON.parse(line)), + fetched: () => lines("fetch.log"), + status: () => read(join(agent, "omp-web", "host-update", "status.json")), + }; +} + +async function waitFor(predicate, what, run, timeoutMs = 120_000) { + for (const deadline = Date.now() + timeoutMs; ; await wait(100)) { + if (predicate()) return; + if (Date.now() > deadline) { + const status = existsSync(join(run.stateRoot, "status.json")) ? JSON.stringify(run.status()) : "none"; + assert.fail(`timed out waiting for ${what}\nupdate status: ${status}\nnpm calls: ${JSON.stringify(run.npmCalls())}\nold daemon log:\n${run.log()}`); + } + } +} + +/** checkUpdates through the hub. The daemon's own startup upload sweep briefly reports "Wait for ..."; any other block is the answer. */ +async function checkUpdates(run) { + for (const deadline = Date.now() + 30_000; ; await wait(500)) { + const info = await hub.getHostSoftwareUpdates(run.hostId, true); + if (!/^Wait for/.test(info.ompweb.blockedReason ?? "") || Date.now() > deadline) return info; + } +} + +async function upgrade(run) { + const { legacy, layout, hostId } = run; + const row = () => hub.listHosts().find((host) => host.id === hostId); + const sent = () => down.get(hostId) ?? []; + + // The real old daemon is welcomed with the version it said (it throws on any other) and linked for upgrading only. + await waitFor(() => row()?.status === "needs-upgrade" && row().upgradeOnly === true && /linked to /.test(run.log()), `the ${legacy.name} daemon to be welcomed upgrade-only`, run); + assert.equal(row().hostVersion, legacy.link); + assert.equal(row().ompwebVersion, legacy.version); + assert.deepEqual([...new Set(sent().filter((message) => message.type === "welcome").map((message) => message.v))], [legacy.link]); + + if (layout === "separate") { + // A candidate on another protocol is refused by what the old updater was told, not by a fixture shortcut. + writeFileSync(join(run.work, "link-protocol.override"), `export const LINK_PROTOCOL_VERSION = ${APPLICATION - 1};\n`); + const wrong = await checkUpdates(run); + assert.equal(wrong.ompweb.protocolVersion, APPLICATION - 1); + assert.equal(wrong.ompweb.canUpdate, false); + rmSync(join(run.work, "link-protocol.override")); + } + + let installBefore; + let web; + if (layout === "shared") { + // The web package shares this installation, and has not replaced it yet: nothing may be installed over it. + web = spawn(nodeBinary(), [join(run.work, "idle.js"), join(run.install, "package.json")], { stdio: "ignore", cwd: run.work }); + const stale = await checkUpdates(run); + assert.equal(stale.ompweb.canUpdate, false); + assert.match(stale.ompweb.blockedReason, /installation is shared/); + await assert.rejects(hub.updateHostSoftware(hostId, "ompweb", repoVersion)); + assert.deepEqual(run.npmCalls(), []); + // `npm i -g` of the web package: the shared installation now holds the candidate while the old daemon keeps running. + for (const name of readdirSync(run.install)) rmSync(join(run.install, name), { recursive: true, force: true }); + cpSync(release(), run.install, { recursive: true }); + chmodSync(run.install, 0o755); + installBefore = { directory: statSync(run.install).ino, manifest: statSync(join(run.install, "package.json")) }; + } + + const info = await checkUpdates(run); + assert.equal(info.ompweb.currentVersion, legacy.version); + assert.equal(info.ompweb.availableVersion, repoVersion); + assert.equal(info.ompweb.updateAvailable, true); + assert.equal(info.ompweb.protocolVersion, APPLICATION); + assert.equal(info.ompweb.canUpdate, true, info.ompweb.blockedReason); + + const accepted = await hub.updateHostSoftware(hostId, "ompweb", repoVersion); + assert.deepEqual({ target: accepted.operation.target, version: accepted.operation.version }, { target: "ompweb", version: repoVersion }); + await waitFor(() => existsSync(join(run.stateRoot, "status.json")) && run.status().operation.state !== "running", "the old updater to finish", run); + const { operation, daemonPid } = run.status(); + assert.equal(operation.state, "succeeded", operation.error); + await waitFor(() => row()?.status === "online" && row().hostVersion === APPLICATION && row().ompwebVersion === repoVersion, "the upgraded daemon to link at the application version", run); + assert.equal(row().upgradeOnly, undefined); + assert.equal(hub.getHostLinkProtocol(hostId), APPLICATION); + await waitFor(() => run.child.exitCode !== null || run.child.signalCode !== null, "the old daemon to stop", run); + const health = read(join(run.stateRoot, operation.id, "health.json")); + assert.equal(health.version, repoVersion); + assert.notEqual(health.pid, daemonPid); + assert.ok(alive(health.pid), "the new daemon keeps running"); + assert.equal(read(join(run.install, "package.json")).version, repoVersion); + + const backups = readdirSync(dirname(run.install)).filter((name) => name.startsWith(".ompweb-backup-")); + if (layout === "separate") { + assert.equal(run.npmCalls().length, 1); + assert.ok(run.npmCalls()[0].includes(`@kahme247/ompweb@${repoVersion}`)); + assert.deepEqual(backups, [`.ompweb-backup-${operation.id}`]); + assert.equal(read(join(dirname(run.install), backups[0], "package.json")).version, legacy.version); + } else { + // A shared install is restarted, never reinstalled: no package manager, no swap, no backup, and the other process lives. + assert.deepEqual(run.npmCalls(), []); + assert.deepEqual(backups, []); + assert.equal(read(join(run.stateRoot, operation.id, "descriptor.json")).restartOnly, true); + assert.equal(statSync(run.install).ino, installBefore.directory); + const manifest = statSync(join(run.install, "package.json")); + assert.deepEqual([manifest.ino, manifest.mtimeMs], [installBefore.manifest.ino, installBefore.manifest.mtimeMs]); + assert.ok(alive(web.pid), "the process sharing the installation is untouched"); + } + const candidateSource = new RegExp(`^https://raw\\.githubusercontent\\.com/[^/]+/[^/]+/v${repoVersion.replaceAll(".", "\\.")}/lib/hosts/link-protocol\\.ts$`); + assert.ok(run.fetched().some((url) => candidateSource.test(url)), "the old updater read the target release's link protocol"); + + // Everything the legacy connection ever received: transport and the two update requests, each with the one + // compatibility an application-12 release may be reached through. The v12 welcome ends it. + const frames = sent(); + const upgraded = frames.findIndex((message) => message.type === "welcome" && message.v === APPLICATION); + assert.ok(upgraded > 0, "the upgraded daemon is welcomed at the application version"); + const legacyFrames = frames.slice(0, upgraded); + assert.deepEqual([...new Set(legacyFrames.map((message) => message.type))].filter((type) => !["welcome", "ack", "ping", "pong", "req"].includes(type)), []); + const requests = legacyFrames.filter((message) => message.type === "req"); + assert.deepEqual([...new Set(requests.map((message) => message.op))].sort(), ["checkUpdates", "updateSoftware"]); + for (const request of requests) assert.deepEqual(request.args.compatibility, { min: APPLICATION, max: APPLICATION }); + assert.deepEqual(requests.filter((message) => message.op === "updateSoftware").map((message) => [message.args.target, message.args.version]), [["ompweb", repoVersion]]); +} + +const skipWindows = process.platform === "win32" && "this proof needs git archive, symlinked dependencies and a POSIX process table"; +const skipScope = "systemd-run cannot create the scope that keeps this test outside the service running the suite"; +async function scenario(t, legacy, layout) { + if (skipWindows) return t.skip(skipWindows); + if (!haveCommit(legacy.commit)) return t.skip(`commit ${legacy.commit} (${legacy.name} daemon) is not in this clone; fetch full history`); + const run = await startLegacy(legacy, layout); + if (!run) return t.skip(skipScope); + try { await upgrade(run); } finally { await stopAll(run.work); } +} + +test("v10 daemon (546a945d) is welcomed as v10 and upgraded by its own updater to a separate v12 install", { timeout: 240_000 }, (t) => scenario(t, LEGACY.v10, "separate")); +test("v10 daemon (546a945d) on an installation the web package already replaced is restarted, not reinstalled", { timeout: 240_000 }, (t) => scenario(t, LEGACY.v10, "shared")); +test("v11 daemon (v0.7.0) is welcomed as v11 and upgraded by its own updater to a separate v12 install", { timeout: 240_000 }, (t) => scenario(t, LEGACY.v11, "separate")); +test("v11 daemon (v0.7.0) on an installation the web package already replaced is restarted, not reinstalled", { timeout: 240_000 }, (t) => scenario(t, LEGACY.v11, "shared")); diff --git a/package-lock.json b/package-lock.json index cc6b22349..487ee1095 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@kahme247/ompweb", - "version": "0.7.0", + "version": "0.7.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@kahme247/ompweb", - "version": "0.7.0", + "version": "0.7.1", "license": "MIT", "dependencies": { "@base-ui/react": "^1.7.0", diff --git a/package.json b/package.json index 59d3b36b8..7446ec74c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@kahme247/ompweb", - "version": "0.7.0", + "version": "0.7.1", "description": "Web UI for the oh-my-pi (omp) coding agent", "homepage": "https://github.com/ompweb-dev/ompweb#readme", "repository": { diff --git a/proxy.ts b/proxy.ts index d51fb6b5b..36f5f59f9 100644 --- a/proxy.ts +++ b/proxy.ts @@ -24,7 +24,8 @@ import { // /api/hosts/cluster-config/manifest and /api/hosts/cluster-config/file/<64 hex digits> (exactly: hosts/[id]/cluster and // hosts/[id]/updates answer to id "cluster-config" and stay gated) check it before serving the master's configuration // to a cluster member (app/api/hosts/cluster-config/). -export const UNGATED = /^\/(?:assets\/|(?:favicon\.ico|icon\.svg|icon\.png|icon-192\.png|badge-96\.png|sw\.js)$|api\/hosts\/link$|api\/hosts\/broker\/v1\/|api\/hosts\/cluster-config\/(?:manifest|file\/[0-9a-f]{64})$)/; +// /api/hosts/maintenance is the independent maintenance API; its bearer gate also covers discovery. +export const UNGATED = /^\/(?:assets\/|(?:favicon\.ico|icon\.svg|icon\.png|icon-192\.png|badge-96\.png|sw\.js)$|api\/hosts\/(?:link|maintenance)$|api\/hosts\/broker\/v1\/|api\/hosts\/cluster-config\/(?:manifest|file\/[0-9a-f]{64})$)/; // Relative: request.url is always http:// under Bun, so an absolute Location would leave https behind a TLS proxy. const redirect = (location: string) => new Response(null, { status: 307, headers: { location } }); diff --git a/server/app.ts b/server/app.ts index e5169c2ca..c0a7191e8 100644 --- a/server/app.ts +++ b/server/app.ts @@ -3,6 +3,7 @@ import path from "node:path"; import { parseArgs } from "node:util"; import { browserLink, type BrowserLinkData } from "@/lib/browser-link"; import { hostLinkSocket, type LinkSocketData } from "@/lib/hosts/hub"; +import { maintenanceSocket, type MaintenanceSocketData } from "@/lib/hosts/maintenance-hub"; import { proxy } from "@/proxy"; import { ensureOmpRunner } from "@/lib/omp/runner-fetch"; import { register } from "../instrumentation.node"; @@ -26,6 +27,9 @@ type ServerSocket = Bun.ServerWebSocket; function isBrowserSocket(ws: ServerSocket): ws is Bun.ServerWebSocket { return "kind" in ws.data && ws.data.kind === "browser"; } +function isMaintenanceSocket(ws: ServerSocket): ws is Bun.ServerWebSocket { + return "kind" in ws.data && ws.data.kind === "maintenance"; +} // Not a browser link: a host link (the only other upgrade). const asLink = (ws: ServerSocket) => ws as Bun.ServerWebSocket; @@ -157,10 +161,22 @@ export function startServer(options: { port: number; hostname: string; dev: bool ...hostLinkSocket, open: (ws: ServerSocket) => { if (isBrowserSocket(ws)) browserLink.open(ws); + else if (isMaintenanceSocket(ws)) maintenanceSocket.open(ws); + }, + message: (ws: ServerSocket, data: string | Uint8Array) => { + if (isBrowserSocket(ws)) browserLink.message(ws, data); + else if (isMaintenanceSocket(ws)) maintenanceSocket.message(ws, data); + else hostLinkSocket.message(asLink(ws), data); + }, + drain: (ws: ServerSocket) => { + if (isBrowserSocket(ws)) browserLink.drain(ws); + else if (!isMaintenanceSocket(ws)) hostLinkSocket.drain(asLink(ws)); + }, + close: (ws: ServerSocket, code: number) => { + if (isBrowserSocket(ws)) browserLink.close(ws, code); + else if (isMaintenanceSocket(ws)) maintenanceSocket.close(ws); + else hostLinkSocket.close(asLink(ws)); }, - message: (ws: ServerSocket, data: string | Uint8Array) => (isBrowserSocket(ws) ? browserLink.message(ws, data) : hostLinkSocket.message(asLink(ws), data)), - drain: (ws: ServerSocket) => (isBrowserSocket(ws) ? browserLink.drain(ws) : hostLinkSocket.drain(asLink(ws))), - close: (ws: ServerSocket, code: number) => (isBrowserSocket(ws) ? browserLink.close(ws, code) : hostLinkSocket.close(asLink(ws))), }, fetch: async (request, server) => { const response = await handle(request, server, async ({ gate }) => { diff --git a/server/routes.ts b/server/routes.ts index e69bbfb3b..d6cfea353 100644 --- a/server/routes.ts +++ b/server/routes.ts @@ -26,9 +26,11 @@ import * as hostsBrokerPath from "@/app/api/hosts/broker/[...path]/route"; import * as hostsClusterConfigFileHash from "@/app/api/hosts/cluster-config/file/[hash]/route"; import * as hostsClusterConfigManifest from "@/app/api/hosts/cluster-config/manifest/route"; import * as hostsLink from "@/app/api/hosts/link/route"; +import * as hostsMaintenance from "@/app/api/hosts/maintenance/route"; import * as hosts from "@/app/api/hosts/route"; import * as hostsIdCluster from "@/app/api/hosts/[id]/cluster/route"; import * as hostsIdUpdates from "@/app/api/hosts/[id]/updates/route"; +import * as hostsIdMaintenance from "@/app/api/hosts/[id]/maintenance/route"; import * as manifest from "@/app/api/manifest/route"; import * as mcp from "@/app/api/mcp/route"; import * as mediaHash from "@/app/api/media/[hash]/route"; @@ -82,9 +84,10 @@ import * as windowsService from "@/app/api/windows-service/route"; import * as worktrees from "@/app/api/worktrees/route"; import type { BrowserLinkData } from "@/lib/browser-link"; import type { LinkSocketData } from "@/lib/hosts/hub"; +import type { MaintenanceSocketData } from "@/lib/hosts/maintenance-hub"; -/** What Bun keeps on an upgraded socket (`ws.data`): a host link's or a browser link's. */ -export type ServerSocketData = LinkSocketData | BrowserLinkData; +/** What Bun keeps on an upgraded socket: application, maintenance API, or browser transport. */ +export type ServerSocketData = LinkSocketData | MaintenanceSocketData | BrowserLinkData; type Method = "GET" | "POST" | "PUT" | "PATCH" | "DELETE"; // never: each handler narrows its own params; the router passes Record. @@ -126,8 +129,10 @@ export const ROUTE_MODULES: Record = { "hosts/cluster-config/file/[hash]": hostsClusterConfigFileHash, "hosts/cluster-config/manifest": hostsClusterConfigManifest, "hosts/link": hostsLink, + "hosts/maintenance": hostsMaintenance, "hosts": hosts, "hosts/[id]/cluster": hostsIdCluster, + "hosts/[id]/maintenance": hostsIdMaintenance, "hosts/[id]/updates": hostsIdUpdates, "manifest": manifest, "mcp": mcp, From caffbf11b2936be8ffae22f5c12ceaaa86a8ba0d Mon Sep 17 00:00:00 2001 From: Andre Brait Date: Sat, 14 Nov 2026 08:35:03 +0000 Subject: [PATCH 2/2] fix(hosts): settle malformed maintenance errors safely Reject hostile failure frames without invoking object coercion or losing request settlement. Keep skipped legacy interoperability cleanup portable and let the Windows watch fixture complete its atomic replacement after transient EPERM. --- lib/hosts/maintenance-hub.test.mjs | 9 +++++++++ lib/hosts/maintenance-protocol.test.mjs | 1 - lib/hosts/maintenance-protocol.ts | 2 +- lib/hosts/v10-upgrade.integration.test.mjs | 15 ++++++++++----- lib/hosts/workspace-watch.test.mjs | 11 ++++++++++- 5 files changed, 30 insertions(+), 8 deletions(-) diff --git a/lib/hosts/maintenance-hub.test.mjs b/lib/hosts/maintenance-hub.test.mjs index c2164d7df..86590039a 100644 --- a/lib/hosts/maintenance-hub.test.mjs +++ b/lib/hosts/maintenance-hub.test.mjs @@ -173,6 +173,15 @@ it("keeps the host's native error status and code", async () => { assert.deepEqual([normalized.status, normalized.code], [500, "internal"]); }); +it("a malformed error response rejects its request instead of losing its settlement", async () => { + const { id, ws } = linked(); + const failed = hub.requestMaintenance(id, "status", {}); + say(ws, { type: "res", id: requests(ws)[0].id, ok: false, toString: null }); + const error = await Promise.race([reject(failed), sleep(100).then(() => null)]); + assert.ok(error instanceof MaintenanceError, "the malformed response must not leave the caller pending"); + assert.deepEqual([error.status, error.code], [500, "internal"]); +}); + it("times out without repeating the request and ignores its late answer", async () => { const { id, ws } = linked(); const error = await reject(hub.requestMaintenance(id, "status", {}, 20)); diff --git a/lib/hosts/maintenance-protocol.test.mjs b/lib/hosts/maintenance-protocol.test.mjs index c0c329a42..abdbc7cfa 100644 --- a/lib/hosts/maintenance-protocol.test.mjs +++ b/lib/hosts/maintenance-protocol.test.mjs @@ -32,5 +32,4 @@ test("failure keeps a native status and code and turns anything else into a 500" assert.deepEqual(failure(new Error("boom")), { status: 500, code: "internal", message: "boom" }); assert.deepEqual(failure(Object.assign(new Error("x"), { code: "ENOENT", status: 200 })), { status: 500, code: "internal", message: "x" }); assert.equal(failure({ status: 418, message: "tea" }).code, "host_error"); - assert.equal(failure("text").message, "text"); }); diff --git a/lib/hosts/maintenance-protocol.ts b/lib/hosts/maintenance-protocol.ts index 6c6842282..80aa9dae4 100644 --- a/lib/hosts/maintenance-protocol.ts +++ b/lib/hosts/maintenance-protocol.ts @@ -85,5 +85,5 @@ export function failure(error: unknown): { status: number; code: string; message const raw = e.status ?? e.httpStatus; const status = typeof raw === "number" && Number.isInteger(raw) && raw >= 400 && raw <= 599 ? raw : 500; const code = typeof e.code === "string" && /^[a-z0-9_]{1,64}$/.test(e.code) ? e.code : status === 500 ? "internal" : "host_error"; - return { status, code, message: (typeof e.message === "string" ? e.message : String(error)).slice(0, 500) }; + return { status, code, message: (typeof e.message === "string" ? e.message : "The host reported an error").slice(0, 500) }; } diff --git a/lib/hosts/v10-upgrade.integration.test.mjs b/lib/hosts/v10-upgrade.integration.test.mjs index a2beddb1b..b8ec02f4a 100644 --- a/lib/hosts/v10-upgrade.integration.test.mjs +++ b/lib/hosts/v10-upgrade.integration.test.mjs @@ -93,14 +93,19 @@ const server = Bun.serve({ /** Pids of every process whose command line names `dir`: daemons, workers and stand-ins all run files from inside it. */ function processesUnder(dir) { - const lines = process.platform === "linux" - ? readdirSync("/proc").filter((name) => /^\d+$/.test(name)).flatMap((name) => { + let lines; + if (process.platform === "linux") { + lines = readdirSync("/proc").filter((name) => /^\d+$/.test(name)).flatMap((name) => { try { return [[Number(name), readFileSync(`/proc/${name}/cmdline`, "utf8").replaceAll("\0", " ")]]; } catch { return []; } - }) - : spawnSync("ps", ["-axo", "pid=,command="], { encoding: "utf8" }).stdout.split("\n").flatMap((line) => { + }); + } else { + const result = spawnSync("ps", ["-axo", "pid=,command="], { encoding: "utf8" }); + if (result.error || result.status !== 0 || typeof result.stdout !== "string") throw new Error("Cannot inspect test-owned processes; temporary installs retained", { cause: result.error }); + lines = result.stdout.split("\n").flatMap((line) => { const match = /^\s*(\d+)\s+(.*)$/.exec(line); return match ? [[Number(match[1]), match[2]]] : []; }); + } return lines.filter(([pid, command]) => pid !== process.pid && command.includes(dir)).map(([pid]) => pid); } async function stopAll(dir) { @@ -112,7 +117,7 @@ async function stopAll(dir) { } after(async () => { - await stopAll(base); + if (process.platform !== "win32") await stopAll(base); await server.stop(true); closeNotificationStore(); hub.flushLinkSamples(); diff --git a/lib/hosts/workspace-watch.test.mjs b/lib/hosts/workspace-watch.test.mjs index 31427fefc..ef1984683 100644 --- a/lib/hosts/workspace-watch.test.mjs +++ b/lib/hosts/workspace-watch.test.mjs @@ -21,7 +21,16 @@ test("a real daemon watch observes the final burst and atomic replacements, then writeFileSync(path, "second"); writeFileSync(path, "final burst bytes"); const until = async (size) => { const end = Date.now() + 3000; while (!events.some((e) => e.type === "change" && e.size === size)) { assert.ok(Date.now() < end, "watch must report final bytes"); await delay(10); } }; await until(17); - const replacement = join(owner.project, "replacement"); writeFileSync(replacement, "atomic save"); renameSync(replacement, path); await until(11); + const replacement = join(owner.project, "replacement"); writeFileSync(replacement, "atomic save"); + // Windows can briefly deny replacement while another process inspects the watched path. + for (const deadline = Date.now() + 2000; ;) { + try { renameSync(replacement, path); break; } + catch (error) { + if (process.platform !== "win32" || error.code !== "EPERM" || Date.now() >= deadline) throw error; + await delay(10); + } + } + await until(11); assert.equal((await owner.host.workspaceOps.unwatchFile(id, "old-epoch")).ok, true); writeFileSync(path, "newer epoch survives old close"); await until(30); await owner.host.workspaceOps.unwatchFile(id, epoch);