diff --git a/CHANGELOG.md b/CHANGELOG.md index 51f48852f..eaa3b0f03 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,16 @@ All notable changes to **omp-web** (`@kahme247/ompweb`) are documented in this f ## [Unreleased] +## [v0.7.1] - 2026-11-14 + +### Fixed + +- **Upgrade v10 and v11 hosts from Settings again.** Update the web server first, then use **Settings → Hosts → Update**. An isolated shim welcomes old daemons as their own version and speaks only their existing update contract; chat, workspace operations and old session frames remain disabled. Native busy-host, preflight and restart safeguards are unchanged. Shared installations use the existing restart-only update after the server's files are replaced. An old daemon that already exited needs starting once. + +### Added + +- **Maintenance API v1, independent of application Protocol v12.** A separate host-token-authenticated connection provides status, session inventory and activity, capabilities and software updates even when application protocols differ. Session traffic uses Protocol v12; future application incompatibility no longer needs to disconnect maintenance. Reverse proxies must forward `/api/hosts/maintenance` WebSocket upgrades as well as `/api/hosts/link`. + ## [v0.7.0] - 2026-11-14 ### Breaking changes diff --git a/README.md b/README.md index b9f241ae0..27e6bb596 100644 --- a/README.md +++ b/README.md @@ -698,10 +698,26 @@ protocol is blocked too. omp is part of each ompweb release and updates with it. Before updating a daemon, close its session processes: the updater refuses to interrupt sessions or subagents. **Settings → System** keeps the web server's own application update and service settings. -Hosts need a daemon build that supports these update operations; older builds -show an unsupported message and need one manual update first. If the daemon -shares its installation with the web server, update the server first, then use -the host's restart-only update. Unsupported service wrappers and detected restart + +**From 0.7.1 onward, maintenance API v1 is separate from application Protocol v12.** +The daemon connects to `/api/hosts/maintenance` with the same host token; status, +session inventory and activity, capabilities, update checks and installation do not +require a compatible application connection. An application mismatch still blocks +chat, workspace operations and session attachment, not maintenance. + +**Upgrading v10 and v11 daemons:** update the web server to 0.7.1 first, then use +**Settings → Hosts → Update**. An upgrade-only shim welcomes each old daemon using +its own protocol and speaks only its existing updater's requests. It never adopts +or stops its sessions, and the old updater still refuses a busy host. Close those +session processes before the cutover. If server and daemon share an installation, +the server update puts the new files in place and the host update only restarts +the daemon. A daemon that already exited after an earlier version rejection needs +starting once; an exited process cannot be reached by the shim. + +Reverse proxies must forward WebSocket upgrades for `/api/hosts/maintenance` +without an additional authentication prompt, just as for `/api/hosts/link`. +Daemons below v10 or without the existing update operations need one manual update. +Unsupported service wrappers and detected restart scripts show a manual-update reason. Stop external restart loops before updating a foreground daemon. diff --git a/app/api/hosts/[id]/maintenance/route.ts b/app/api/hosts/[id]/maintenance/route.ts new file mode 100644 index 000000000..7a5dcf114 --- /dev/null +++ b/app/api/hosts/[id]/maintenance/route.ts @@ -0,0 +1,16 @@ +import { getHost } from "@/lib/hosts/hub"; +import { MaintenanceError, requestMaintenance } from "@/lib/hosts/maintenance-hub"; + +/** Browser access stays behind the web gate; no application connection is needed. */ +export async function GET(request: Request, { params }: { params: Promise<{ id: string }> }) { + const { id } = await params; + const headers = { "cache-control": "no-store" }; + if (!getHost(id)) return Response.json({ error: "Unknown host", code: "host_not_found" }, { status: 404, headers }); + try { + const op = new URL(request.url).searchParams.get("sessions") === "1" ? "listSessions" : "status"; + return Response.json(await requestMaintenance(id, op, {}), { headers }); + } catch (error) { + if (error instanceof MaintenanceError) return Response.json({ error: error.message, code: error.code }, { status: error.status, headers }); + return Response.json({ error: "The host maintenance request failed", code: "maintenance_failed" }, { status: 500, headers }); + } +} diff --git a/app/api/hosts/maintenance/route.ts b/app/api/hosts/maintenance/route.ts new file mode 100644 index 000000000..b3174f4b6 --- /dev/null +++ b/app/api/hosts/maintenance/route.ts @@ -0,0 +1,8 @@ +import { handleMaintenanceUpgrade, type MaintenanceSocketData } from "@/lib/hosts/maintenance-hub"; +import { LINK_PROTOCOL_VERSION } from "@/lib/hosts/link-protocol"; +import packageJson from "@/package.json"; + +/** API v1 discovery and its independent host-token-authenticated WebSocket. */ +export function GET(request: Request, { upgrade }: { upgrade: (data: MaintenanceSocketData) => boolean }) { + return handleMaintenanceUpgrade(request, upgrade, { protocolVersion: LINK_PROTOCOL_VERSION, ompwebVersion: packageJson.version }); +} diff --git a/components/HostUpdates.test.mjs b/components/HostUpdates.test.mjs new file mode 100644 index 000000000..43ee3f85b --- /dev/null +++ b/components/HostUpdates.test.mjs @@ -0,0 +1,83 @@ +import "../tests/setup-dom.mjs"; +import assert from "node:assert/strict"; +import test from "node:test"; +import { act, createElement } from "react"; +import { createRoot } from "react-dom/client"; +import { HostUpdates } from "./HostUpdates"; + +const endpoint = "/api/hosts/box/updates"; +const info = { + ompweb: { currentVersion: "0.7.0", availableVersion: "0.7.1", updateAvailable: true, canUpdate: true, blockedReason: null, updateCommand: "npm install -g @kahme247/ompweb", installMethod: "npm", protocolVersion: 12 }, + omp: { currentVersion: "18.20.0" }, + operation: null, +}; + +/** Render the real host row and record every request it sends. */ +async function renderHost(t, host) { + const requests = []; + const originalFetch = globalThis.fetch; + globalThis.fetch = async (url, init) => { + requests.push({ url: String(url), method: init?.method ?? "GET", body: init?.body }); + if (init?.method === "POST") return Response.json({ accepted: true, operation: { state: "running", target: "ompweb", version: "0.7.1" } }); + return Response.json(info); + }; + const container = document.createElement("div"); + document.body.append(container); + const root = createRoot(container); + t.after(() => { + act(() => root.unmount()); + container.remove(); + globalThis.fetch = originalFetch; + }); + await act(async () => { + root.render(createElement(HostUpdates, { host })); + }); + await act(() => new Promise((resolve) => setTimeout(resolve, 20))); + return { container, requests }; +} + +const button = (root, label) => [...root.querySelectorAll("button")].find((candidate) => candidate.textContent.trim() === label); + +test("an upgrade-only legacy host can check for and submit an ompweb update", async (t) => { + const { container, requests } = await renderHost(t, { id: "box", label: "Box", status: "needs-upgrade", upgradeOnly: true, ompVersion: "omp/18.20.0", ompwebVersion: "0.7.0" }); + assert.equal(requests[0]?.url, endpoint, "the status is read on mount"); + assert.equal(button(container, "Check updates").disabled, false); + + await act(async () => button(container, "Check updates").click()); + await act(() => new Promise((resolve) => setTimeout(resolve, 20))); + assert.ok(requests.some((request) => request.url === `${endpoint}?force=1` && request.method === "GET"), requests.map((request) => request.url).join(", ")); + + act(() => button(container, "Update").click()); + const confirm = [...document.querySelectorAll(".dialog-actions button")].find((candidate) => candidate.textContent.trim() === "Update"); + assert.ok(confirm, "the update confirmation did not open"); + await act(async () => confirm.click()); + await act(() => new Promise((resolve) => setTimeout(resolve, 20))); + const post = requests.find((request) => request.method === "POST"); + assert.ok(post, "the update was not submitted"); + assert.deepEqual(JSON.parse(post.body), { target: "ompweb", version: "0.7.1" }); +}); + +test("a connected maintenance API can update a compatible host while its application connection is offline", async (t) => { + const { container, requests } = await renderHost(t, { id: "box", label: "Box", status: "offline", apiVersion: 1, hostVersion: 12, ompwebVersion: "0.7.0" }); + assert.equal(requests[0]?.url, endpoint); + assert.equal(button(container, "Check updates").disabled, false); + assert.ok(button(container, "Update")); +}); + +test("an ordinary incompatible host cannot be checked or updated", async (t) => { + const { container, requests } = await renderHost(t, { id: "box", label: "Box", status: "needs-upgrade", ompVersion: "omp/18.20.0", ompwebVersion: "0.6.0" }); + assert.deepEqual(requests, [], "an unavailable host must not be read"); + const check = button(container, "Check updates"); + assert.equal(check.disabled, true); + act(() => check.click()); + assert.equal(button(container, "Update"), undefined); + assert.ok(container.textContent.includes("This host daemon does not support updates"), container.textContent); + assert.deepEqual(requests, []); +}); + +test("an offline host stays unavailable", async (t) => { + const { container, requests } = await renderHost(t, { id: "box", label: "Box", status: "offline", ompVersion: "omp/18.20.0", ompwebVersion: "0.7.0" }); + assert.deepEqual(requests, []); + assert.equal(button(container, "Check updates").disabled, true); + assert.ok(container.textContent.includes("Updates unavailable while this host is offline."), container.textContent); +}); diff --git a/components/HostUpdates.tsx b/components/HostUpdates.tsx index ed2fd5861..ef7eb4e1d 100644 --- a/components/HostUpdates.tsx +++ b/components/HostUpdates.tsx @@ -30,7 +30,8 @@ export function HostUpdates({ host }: { host: HostRow }) { const mounted = useRef(true); const postController = useRef(null); const endpoint = `/api/hosts/${encodeURIComponent(host.id)}/updates`; - const unavailable = host.status === "offline" || host.status === "needs-upgrade"; + const upgradeOnly = host.status === "needs-upgrade" && host.upgradeOnly === true; + const unavailable = (host.status === "offline" || host.status === "needs-upgrade") && !upgradeOnly && host.apiVersion !== 1; useEffect(() => { mounted.current = true; @@ -164,7 +165,7 @@ export function HostUpdates({ host }: { host: HostRow }) { const busy = loading || posting || monitoring; const disabled = busy || unavailable || unsupported; - const unavailableReason = host.status === "needs-upgrade" || unsupported ? t("hostUpdates.unsupported") : host.status === "offline" ? t("hostUpdates.offline") : null; + const unavailableReason = (host.status === "needs-upgrade" && unavailable) || unsupported ? t("hostUpdates.unsupported") : host.status === "offline" && unavailable ? t("hostUpdates.offline") : null; const operation = info?.operation; return (
diff --git a/docs/specs/2026-10-28-remote-hosts-design.md b/docs/specs/2026-10-28-remote-hosts-design.md index 927755b17..280937c6f 100644 --- a/docs/specs/2026-10-28-remote-hosts-design.md +++ b/docs/specs/2026-10-28-remote-hosts-design.md @@ -315,15 +315,15 @@ that upload's file. ops; 9: `open.launch.shareCredentials`; 10: `open.launch.configVersion`, `config_state`, the `clusterConfig` op; 11: channel lines are the omp runner's `{t}` frames, not omp RPC frames). Main links daemons from - `MIN_LINK_PROTOCOL_VERSION` (11) up to its own `LINK_PROTOCOL_VERSION` (11); - its `welcome` carries the version the link speaks (see "Protocol versions and - compatibility"). A hello whose `v` is outside that range (checked on the raw - JSON before decoding) gets close 4426 with reason `{"serverVersion":11}`; the - daemon exits 1, telling which side to - update ("update ompweb on this host", or "this host's ompweb is newer than - the server's: update ompweb on the server" when `serverVersion` is below its - own); Settings → Hosts shows "needs upgrade (host vN, server vM)" unless the - host has a healthy socket. + `MIN_LINK_PROTOCOL_VERSION` (12) up to its own `LINK_PROTOCOL_VERSION` (12). + A v10 or v11 daemon is welcomed with its own version by an upgrade-only shim, + outside that application range. The shim only sends `checkUpdates` and + `updateSoftware`; it acknowledges and discards application activity, never + adopts or kills sessions, and never starts cluster configuration synchronization. + All other incompatible versions get close 4426 with `{"serverVersion":12}`. + New daemons retain their separate maintenance API connection while an + application mismatch is retried. Old daemons may exit on rejection; an + already-exited daemon needs starting once. - **Identity.** The host is the record whose `tokenHash` matches. Nothing the daemon sends selects or changes its identity. - **One link per host.** A hello on a new socket with the same token closes the @@ -404,21 +404,15 @@ the `spawn` op are gone). message being understood unless the version (or, later, a capability) says so. Tolerance starts with v3: a daemon older than that is strict (an unknown type or op makes it refuse the line). -- **Older versions stay supported** from the next version on, until supporting - one becomes too much of a maintenance burden; dropping one is an explicit - decision (raising `MIN_LINK_PROTOCOL_VERSION`, which is 11 today). In - practice main keeps at least version N-1. For an older daemon main decodes - its hello into today's shape and sends only messages that version - understands (to a v4 daemon: no `part`, every line whole, and no unacked - window, since it acks only inside its next uplink frame). - A daemon on a dropped version gets 426 - (a POST) or close 4426 (a hello frame). Dropping versions ends with the - decision below. -- **Upgrade order: web app first, then hosts.** Main supports older daemons; a - daemon is not required to support a newer main, and one newer than main gets - close 4426. - `hostVersion` (Settings → Hosts) is the version a linked daemon speaks, - so Settings can later show "update available" for one behind the server. +- **Application compatibility is separate from maintenance compatibility.** + Dropping an application version raises `MIN_LINK_PROTOCOL_VERSION` without + dropping maintenance API v1. The current application range is 12–12. + Legacy v10/v11 are admitted only by the update shim, not the session runtime. +- **Upgrade order: web app first, then hosts.** The shim passes the new server's + application range to the old daemon's unchanged updater, while its `welcome` + echoes the old version. The updater can install Protocol v12 without needing + to understand it. Existing safety checks still refuse busy hosts and unsafe + supervisors; a shared installation uses restart-only after updating the server. - **v4 (the WebSocket link) dropped v3** before any release; from the next version on, main keeps N-1. A v3 daemon POSTs its hello to `/api/hosts/link`; main answers that POST 426 with both versions (and never @@ -443,16 +437,26 @@ the `spawn` op are gone). a channel; both sides refuse one with 426 / close 4426 instead of hanging. Prerelease builds already shipped v10 with omp RPC frames, so the runner could not reuse 10. -- **Decided: backward compatibility becomes enforced once the WebSocket - experiments end.** Until then (the browser transport experiment and the - host-link measurements), versions may still be dropped as above. Call N the - first link protocol version released after them. From N on the guarantees - are enforced, so the web app can always update an old host from Settings → - Hosts: the handshake (`hello`/`welcome` and the version close codes), the - version report, and an update op form a fixed core that every later web app - keeps working with every daemon on N or newer. `MIN_LINK_PROTOCOL_VERSION` - never goes above N. This replaces the update-only mode that was planned - for dropped versions. +- **Maintenance API v1 is a separate, stable contract.** New daemons discover + `/api/hosts/maintenance` with the same bearer token, then open a separate + WebSocket. Its `hello`/`welcome` negotiate `apiVersion: 1` independently of the + advisory `protocolVersion`. Its codec and transport do not import the + application protocol. A protocol mismatch cannot terminate this connection. + Requests have bounded IDs and explicit replies; disconnects and timeouts never + cause automatic replay of an update. The daemon remains authoritative for + whether it is safe to update. + - `status`: software/API/protocol versions, capabilities, workspace-operation + availability, session-process inventory and activity, drained/busy state and + the durable update operation. + - `listSessions`: session metadata, including `processRunning` and `active`; + no transcript or application frame decoding. + - `checkUpdates` and `updateSoftware`: the native updater and its safeguards. + - The web-authenticated `GET /api/hosts/[id]/maintenance` exposes status; + `?sessions=1` exposes inventory. Existing host update routes prefer API v1 + when available, otherwise use the legacy shim. + - Revoke and shutdown close both connections. API v1 remains supported across + future application-protocol changes; incompatible maintenance changes need + a separately negotiated API version. ## Host ops (this sub-project) diff --git a/lib/hosts/daemon-channels.ts b/lib/hosts/daemon-channels.ts index ba0f3e28d..003735354 100644 --- a/lib/hosts/daemon-channels.ts +++ b/lib/hosts/daemon-channels.ts @@ -81,6 +81,12 @@ export class ChannelTable { for (const channel of this.channels.values()) if (channel.lifecycle?.activeTurn ?? channel.pending.activeTurn) active++; return active; } + maintenanceSessions(): { id: string | null; cwd: string; active: boolean }[] { + return [...this.channels.values()].map(channel => ({ + id: channel.pending.sessionId, cwd: channel.cwd, + active: (channel.lifecycle?.isRunning ?? channel.pending.activeTurn) || channel.resuming === true, + })); + } cwdForSession(id: string): string | undefined { for (const channel of this.channels.values()) if (channel.pending.sessionId === id) return channel.cwd; return undefined; diff --git a/lib/hosts/daemon-link.ts b/lib/hosts/daemon-link.ts index 241dfe3d7..e335bcfd8 100644 --- a/lib/hosts/daemon-link.ts +++ b/lib/hosts/daemon-link.ts @@ -54,6 +54,8 @@ export function dialable(url: string, allowHttp: boolean): boolean { } export type LinkDeps = { allowHttp: boolean; primary: boolean; + /** Maintenance has its own lifecycle, so an application mismatch must not end this host. */ + keepOnVersionMismatch?: boolean; hello(link: DaemonLink, replay: boolean): string; handleDown(link: DaemonLink, message: DownMessage | UnknownMessage): void; onEnd(link: DaemonLink, reason: "revoked" | "replaced" | "version"): void; @@ -89,6 +91,7 @@ function linkRuntime(owner: DaemonLink, deps: LinkDeps) { const linkUrl = serverUrl + "/api/hosts/link"; let authorization = "Bearer " + owner.login.token; const log = deps.log; + let mismatched = false; const channels = new Map(); // Uplink messages wait in `sender` until they go out, then here until main acks them; a dropped socket resends // them after the next welcome. @@ -338,6 +341,7 @@ function linkRuntime(owner: DaemonLink, deps: LinkDeps) { case "welcome": if (message.v < MIN_LINK_PROTOCOL_VERSION || message.v > owner.version) throw new LinkProtocolError("Invalid welcome protocol version"); owner.version = message.v; + mismatched = false; log("linked to " + serverUrl); acknowledgeHostUpdateHealth(); unknownLog.reset(); linked = socket; redialDelay = FIRST_RETRY_MS; resend = outbox.pending(); resendAt = 0; sender.dropBlobs(); @@ -454,6 +458,7 @@ function linkRuntime(owner: DaemonLink, deps: LinkDeps) { if (stopped) return; const closed = await connect(sent, hello); if (stopped) return; + if (sent !== authorization) continue; if (typeof closed !== "string" && closed.code === LINK_CLOSE.versionMismatch) { let newest: unknown; try { newest = JSON.parse(closed.reason)?.serverVersion; } catch {} if (typeof newest === "number" && Number.isInteger(newest) && newest >= MIN_LINK_PROTOCOL_VERSION && newest < owner.version) { @@ -462,13 +467,16 @@ function linkRuntime(owner: DaemonLink, deps: LinkDeps) { log(typeof newest === "number" && newest < LINK_PROTOCOL_VERSION ? "this host's ompweb is newer than the server's: update ompweb on the server" : "the server speaks a different link protocol: update ompweb on this host"); - endLink("version"); return; + if (!deps.keepOnVersionMismatch) { endLink("version"); return; } + // Rejected application replies cannot be delivered, but remain queued if compatibility returns. + mismatched = true; } if (typeof closed !== "string" && closed.code === LINK_CLOSE.replaced) { log("another ompweb host linked with this token; exiting this link"); endLink("replaced"); return; } if (typeof closed !== "string" && closed.code === 1000) { endLink("revoked"); return; } if (typeof closed !== "string" && !closed.opened) { const { status, body } = await fetch(linkUrl, { headers: { authorization: sent }, signal: AbortSignal.any([stopSignal.signal, AbortSignal.timeout(LINK_LIMITS.offlineMs)]) }); void body?.cancel().catch(() => {}); + if (sent !== authorization) continue; if (status === 401) { if (deps.reload?.(owner, true)) continue; endLink("revoked"); return; } log(status === 503 ? "the server is unavailable (503); redialing" : "the server did not accept the WebSocket upgrade (HTTP " + status + "): check that a reverse proxy forwards Upgrade and Connection, and that the server runs an ompweb at least as new as this host's"); } else log(typeof closed === "string" ? "link lost (" + closed + "); redialing" : "link lost (code " + closed.code + "); redialing"); @@ -485,7 +493,7 @@ function linkRuntime(owner: DaemonLink, deps: LinkDeps) { stopped = true; stopSignal.abort(); socket?.close(); clearTimeout(flushTimer); clearTimeout(statsTimer); await loop; } - return { queue, output, discard, result, report, busy: () => reliablePending > 0, ackedDown: () => down.acked, isUp: () => linked !== null && !stopped, + return { queue, output, discard, result, report, busy: () => !mismatched && reliablePending > 0, ackedDown: () => down.acked, isUp: () => linked !== null && !stopped, rekey: (token: string) => { owner.login.token = token; authorization = "Bearer " + token; socket?.close(1001, "credentials changed"); }, start: () => { if (!started && !stopped) { started = true; loop = keepLinked(); } }, stop }; } diff --git a/lib/hosts/daemon-links.test.mjs b/lib/hosts/daemon-links.test.mjs index cd3d573a8..7290e3b33 100644 --- a/lib/hosts/daemon-links.test.mjs +++ b/lib/hosts/daemon-links.test.mjs @@ -19,9 +19,9 @@ const until = async predicate => { const end=Date.now()+5000;while(!predicate()) function hub(t, max=LINK_PROTOCOL_VERSION){ const hellos=[],messages=[];let socket,ack=0; const server=Bun.serve({port:0,hostname:"127.0.0.1",fetch(req,s){return s.upgrade(req)?undefined:new Response("",{status:426});},websocket:{message(ws,data){if(typeof data!=="string")return;for(const line of data.split("\n").filter(Boolean)){const m=JSON.parse(line);if(m.type==="hello"){hellos.push(m);socket=ws;if(m.v>max){ws.close(4426,JSON.stringify({serverVersion:max}));continue;}ws.send(encodeLine({type:"welcome",v:max,hostId:"h"}));}else if(m.seq){messages.push(m);ack=m.seq;}}if(ws.readyState===1)ws.send(encodeLine({type:"ack",ack}));}}});t.after(()=>server.stop(true)); - return {url:`http://127.0.0.1:${server.port}`,hellos,messages,close:(code)=>socket.close(code)}; + return {url:`http://127.0.0.1:${server.port}`,hellos,messages,close:(code)=>socket.close(code),setMax:(version)=>{max=version;}}; } -function link(server,id,ends){return new DaemonLink({url:server.url,token:"test"},id,{allowHttp:false,primary:id==="l1",hello:(link)=>encodeLine({type:"hello",v:link.version,ompVersion:null,platform:"linux",home:"/tmp",autoApproveExtension:false,ack:link.ackedDown,channels:[],link:{id:link.id,notifier:false}}),handleDown(){},onEnd:(link,reason)=>ends.push([link.id,reason]),log(){}});} +function link(server,id,ends,options={}){return new DaemonLink({url:server.url,token:"test"},id,{allowHttp:false,primary:id==="l1",hello:(link)=>encodeLine({type:"hello",v:link.version,ompVersion:null,platform:"linux",home:"/tmp",autoApproveExtension:false,ack:link.ackedDown,channels:[],link:{id:link.id,notifier:false}}),handleDown(){},onEnd:(link,reason)=>ends.push([link.id,reason]),log(){},...options});} test("a server below the minimum protocol ends only its own link and leaves the other live",async t=>{ const a=hub(t),b=hub(t,LINK_PROTOCOL_VERSION-1),ends=[],la=link(a,"l1",ends),lb=link(b,"l2",ends);t.after(()=>Promise.all([la.stop(),lb.stop()]));la.start();lb.start(); @@ -66,3 +66,17 @@ test("reliable results hold their own link busy until acknowledged without block await until(() => a.messages.some(message => message.id === "held-result") && !la.busy()); assert.equal(lb.isUp(), true); }); + +test("rejected application results do not wedge maintenance and still replay when compatibility returns", async t => { + const server = hub(t, LINK_PROTOCOL_VERSION - 1); + const connection = link(server, "l1", [], { keepOnVersionMismatch: true }); + t.after(() => connection.stop()); + connection.result("pending-before-mismatch", { completed: true }); + assert.equal(connection.busy(), true, "a compatible but disconnected link still protects reliable results"); + connection.start(); + await until(() => server.hellos.length > 0 && !connection.busy()); + assert.equal(connection.isUp(), false); + server.setMax(LINK_PROTOCOL_VERSION); + await until(() => connection.isUp() && server.messages.some(message => message.id === "pending-before-mismatch")); + assert.equal(server.messages.find(message => message.id === "pending-before-mismatch").result.completed, true); +}); diff --git a/lib/hosts/daemon-maintenance.ts b/lib/hosts/daemon-maintenance.ts new file mode 100644 index 000000000..2c4eaa3d5 --- /dev/null +++ b/lib/hosts/daemon-maintenance.ts @@ -0,0 +1,237 @@ +// Host end of the maintenance API (maintenance-protocol.ts): dials the server's /api/hosts/maintenance with the host's +// bearer token and answers its requests through `deps.handleRequest`. Independent of the application link +// (daemon-link.ts): its own socket and redial loop, and nothing it meets (a server without this API, another API version, +// a refused token) ever stops the daemon or its omp children. Node-only, relative imports. +import { setTimeout as sleep } from "node:timers/promises"; +import { isRecord } from "../type-guards"; +import { + decodeFrame, + encodeFrame, + failure, + MAINTENANCE_API_VERSION, + MAINTENANCE_CLOSE, + MAINTENANCE_LIMITS, + MAINTENANCE_PATH, + MaintenanceError, + maintenanceTimings, + PONG, + type MaintenanceHello, + type MaintenanceOp, +} from "./maintenance-protocol"; + +const REQUEST_ID = /^[A-Za-z0-9_-]{1,64}$/; +// Each wait varies ±20% so many hosts do not redial in step. +const withJitter = (ms: number) => ms * (0.8 + Math.random() * 0.4); +const reason = (error: unknown) => (error instanceof Error ? error.message : String(error)); + +export interface MaintenanceDeps { + /** The host's hello, built on every dial; its apiVersion is overridden with this API's. */ + hello(): MaintenanceHello; + /** Runs one request. Throw an error with a `status` or `httpStatus` (400..599) and a lowercase `code` (HostOpError, HostUpdateError) to answer with them. */ + handleRequest(op: MaintenanceOp, args: Record): Promise; + log(text: string): void; + /** Called before each redial, like the application link's reload. */ + reload?(): void; + /** The link came up (welcomed) or went down. */ + onState?(up: boolean): void; +} + +/** How a dial ended. `linked`: the server welcomed it. `code`: the server's close code, 0 when this side ended it. */ +interface Closed { + linked: boolean; + code: number; + reason: string; +} + +export class DaemonMaintenance { + private token: string; + private stopped = false; + private loop: Promise | null = null; + /** Ends the dial in progress (its probe or its socket). */ + private cancel: (() => void) | null = null; + private readonly abort = new AbortController(); + private lastLog = ""; + + constructor(private readonly login: { url: string; token: string }, private readonly deps: MaintenanceDeps) { + this.token = login.token; + } + + start(): void { + if (!this.loop && !this.stopped) this.loop = this.run(); + } + + /** Ends the redial wait, the dial and the socket; replies of requests still running are dropped. */ + async stop(): Promise { + this.stopped = true; + this.abort.abort(); + this.cancel?.(); + await this.loop; + } + + /** The token changed: redial with it. */ + rekey(token: string): void { + this.token = token; + this.cancel?.(); + } + + private say(text: string): void { + if (text === this.lastLog) return; + this.lastLog = text; + this.deps.log(text); + } + + private async run(): Promise { + let delay = maintenanceTimings.retryMs; + while (!this.stopped) { + try { + const closed = await this.connect(); + if (this.stopped) return; + if (closed.linked) delay = maintenanceTimings.retryMs; + // Another API, another host holding the token: dialing again soon changes nothing. + if (closed.code === MAINTENANCE_CLOSE.apiMismatch || closed.code === MAINTENANCE_CLOSE.replaced) delay = maintenanceTimings.retryMaxMs; + this.say(`maintenance link ${closed.linked ? "lost" : "unavailable"} (${closed.reason || `code ${closed.code}`}); redialing`); + this.deps.reload?.(); + } catch (error) { + if (this.stopped) return; + this.say(`maintenance link lost (${reason(error)}); redialing`); + } + if (this.stopped) return; + await sleep(withJitter(delay), undefined, { signal: this.abort.signal }).catch(() => {}); + delay = Math.min(delay * 2, maintenanceTimings.retryMaxMs); + } + } + + /** One dial: discovery, then the socket until it closes. */ + private async connect(): Promise { + const serverUrl = this.login.url.replace(/\/+$/, "") + MAINTENANCE_PATH; + const authorization = `Bearer ${this.token}`; + const attempt = new AbortController(); + this.cancel = () => attempt.abort(); + const unavailable = await this.discover(serverUrl, authorization, attempt.signal); + // Stopped or rekeyed meanwhile: not the server's answer. + if (attempt.signal.aborted) return { linked: false, code: 0, reason: "cancelled" }; + if (unavailable) return unavailable; + return this.dial(serverUrl.replace(/^http/, "ws"), authorization); + } + + /** + * GET the endpoint first: only a server that answers 200 `{apiVersion: 1}` (to this token) is sent a hello. A missing + * endpoint, a page, a 426 or another version is not this API, and an old server may accept any WebSocket URL and + * mistake the hello for application traffic. Null when the server speaks API 1. + */ + private async discover(url: string, authorization: string, signal: AbortSignal): Promise { + let response: Response; + try { + response = await fetch(url, { headers: { authorization }, redirect: "error", signal: AbortSignal.any([signal, AbortSignal.timeout(maintenanceTimings.helloMs)]) }); + } catch (error) { + return { linked: false, code: 0, reason: reason(error) }; + } + let body: unknown = null; + if (response.status === 200) { + try { + body = JSON.parse(await response.text()); + } catch { + // A page, or the read was cut: not this API. + } + } else void response.body?.cancel().catch(() => {}); + if (isRecord(body) && body.apiVersion === MAINTENANCE_API_VERSION) return null; + // A refused token or an unavailable server may be fixed by the next dial; anything else is not API 1. + const transient = response.status === 401 || response.status >= 500; + const why = response.status === 401 ? "the server refused this host's token" : `the server has no maintenance API ${MAINTENANCE_API_VERSION}`; + return { linked: false, code: transient ? 0 : MAINTENANCE_CLOSE.apiMismatch, reason: `${why} (HTTP ${response.status})` }; + } + + /** One socket from dial to close. */ + private dial(url: string, authorization: string): Promise { + const { promise, resolve } = Promise.withResolvers(); + const hello = this.deps.hello(); + const advertised = new Set(hello.capabilities); + const helloLine = encodeFrame({ ...hello, type: "hello", apiVersion: MAINTENANCE_API_VERSION }); + // This repo's DOM typing has no init argument; Node's and Bun's WebSocket take `{ headers }`. + const ws = new WebSocket(url, { headers: { authorization } } as unknown as string[]); + const active = new Set(); + let linked = false; + let done = false; + let watchdog: NodeJS.Timeout | undefined; + // Once ended, a socket stays inert: every handler below ignores it. + const end = (code: number, why: string) => { + if (done) return; + done = true; + clearTimeout(watchdog); + this.cancel = null; + if (linked) this.deps.onState?.(false); + resolve({ linked, code, reason: why }); + }; + const giveUp = (why: string) => { + end(0, why); + ws.close(); + }; + this.cancel = () => giveUp("cancelled"); + // The server pings every pingMs: silence for offlineMs is a dead link. + const alive = () => { + clearTimeout(watchdog); + watchdog = setTimeout(() => giveUp("no traffic from the server"), maintenanceTimings.offlineMs); + watchdog.unref?.(); + }; + alive(); + + const reply = (id: string, body: object) => { + if (done) return; + try { + ws.send(encodeFrame({ type: "res", id, ...body })); + } catch (error) { + // An oversized or unserializable result: say so. A closed socket: the server saw the link drop. + try { + ws.send(encodeFrame({ type: "res", id, ok: false, ...failure(error) })); + } catch { + // Closed. + } + } + }; + const request = (frame: Record) => { + const { id, op, args } = frame; + if (typeof id !== "string" || !REQUEST_ID.test(id)) return giveUp("invalid request id"); + const answer = async (): Promise => { + if (typeof op !== "string" || !advertised.has(op)) throw new MaintenanceError(501, "unsupported_op", "Unsupported maintenance operation"); + if (!isRecord(args)) throw new MaintenanceError(400, "bad_request", "A request's args must be an object"); + if (active.has(id)) throw new MaintenanceError(409, "duplicate_request", "A request with this id is running"); + if (active.size >= MAINTENANCE_LIMITS.inFlight) throw new MaintenanceError(429, "busy", "Too many maintenance requests are running"); + active.add(id); + try { + return { ok: true, result: (await this.deps.handleRequest(op as MaintenanceOp, args)) ?? null }; + } finally { + active.delete(id); + } + }; + void answer().catch((error: unknown) => ({ ok: false, ...failure(error) })).then((body) => reply(id, body)); + }; + const onFrame = (data: unknown) => { + const frame = decodeFrame(data); + if (frame.type === "ping") ws.send(PONG); + else if (frame.type === "welcome") { + if (frame.apiVersion !== MAINTENANCE_API_VERSION) return giveUp(`the server's maintenance API is ${String(frame.apiVersion)}`); + linked = true; + this.lastLog = ""; + this.deps.onState?.(true); + } else if (frame.type === "req") { + if (linked) request(frame); + else giveUp("request before welcome"); + } + }; + + ws.onopen = () => { + if (!done) ws.send(helloLine); + }; + ws.onmessage = ({ data }: MessageEvent) => { + if (done) return; + alive(); + try { + onFrame(data); + } catch (error) { + giveUp(reason(error)); + } + }; + ws.onclose = (event) => end(event.code, event.reason); + return promise; + } +} diff --git a/lib/hosts/daemon.test.mjs b/lib/hosts/daemon.test.mjs index 40ad4dc8f..561d832aa 100644 --- a/lib/hosts/daemon.test.mjs +++ b/lib/hosts/daemon.test.mjs @@ -1226,24 +1226,25 @@ test("uplink messages unacked when the socket drops are resent after the next we } }); -test("close 4426 exits 1 naming the side to update; close 4409 exits 0", async () => { +test("application version rejection keeps the daemon alive for maintenance and retries; replacement still exits", async () => { const log = captureLog(); try { - for (const [serverVersion, advice] of [[2, "this host's ompweb is newer than the server's: update ompweb on the server"], [10, "update ompweb on the server"], [LINK_PROTOCOL_VERSION + 1, "update ompweb on this host"]]) { - log.lines.length = 0; + for (const serverVersion of [2, 10, LINK_PROTOCOL_VERSION + 1]) { const server = scriptedServer(); const daemon = start(server); + let exited = false; + void daemon.done.then(() => { exited = true; }); await waitFor(() => server.hellos.length === 1, "the hello"); server.close(4426, JSON.stringify({ serverVersion })); - assert.equal(await daemon.done, 1); - assert.ok(log.lines.some((line) => line.includes(advice)), log.lines.join("|")); + await waitFor(() => server.hellos.length >= 2, "application redial without terminating maintenance"); + assert.equal(exited, false); + await daemon.stop(); } const server = scriptedServer(); const replaced = start(server); await waitFor(() => server.hellos.length === 1, "the hello"); server.close(4409, "replaced by a newer link"); assert.equal(await replaced.done, 0); - assert.ok(log.lines.some((line) => line.includes("another ompweb host linked with this token; exiting")), log.lines.join("|")); } finally { log.restore(); } diff --git a/lib/hosts/daemon.ts b/lib/hosts/daemon.ts index 803b3580b..726c96102 100644 --- a/lib/hosts/daemon.ts +++ b/lib/hosts/daemon.ts @@ -11,6 +11,8 @@ import { sampleHostCapacity } from "./capacity"; import type { HostControlCommand, HostControlStatus } from "./host-control"; import { readHostLinks, readHostLogin, type HostLink } from "./host-login"; import { DaemonLink, dialable, nextRedialDelay } from "./daemon-link"; +import { DaemonMaintenance } from "./daemon-maintenance"; +import { MAINTENANCE_API_VERSION, type MaintenanceOp } from "./maintenance-protocol"; import { ChannelTable } from "./daemon-channels"; import { credentialsEnv, serverFetch, startBrokerForwarder } from "./daemon-broker"; import { createClusterConfig } from "./daemon-cluster-config"; @@ -91,6 +93,7 @@ export function runHostDaemon(options: DaemonOptions): HostDaemon { for (const link of initial) if (!dialable(link.url, options.allowHttp)) throw new Error("ompweb host needs an https:// server URL (http:// only for loopback, or with --allow-http)"); const primaryUrl = (explicit?.[0]?.url ?? readHostLogin()?.url ?? initial[0]?.url)?.replace(/\/+$/, ""); const links = new Map(); + const maintenanceLinks = new Map(); const ended = new Map(); const services = new Map(); const uploadOwners = new Map(); @@ -252,13 +255,18 @@ export function runHostDaemon(options: DaemonOptions): HostDaemon { } /** Stops a removed link's workspace service (tracked so shutdown awaits it) and frees its upload ownership so other servers can take those uploads. */ function dropLink(link: DaemonLink): void { + const maintenance = maintenanceLinks.get(link.id); maintenanceLinks.delete(link.id); + if (maintenance) { + const stop = maintenance.stop().finally(() => stops.delete(stop)); + stops.add(stop); + } const service = services.get(link.id); services.delete(link.id); if (service) { const stop: Promise = service.stop().catch(error => log("could not stop a workspace service: " + reason(error))).finally(() => stops.delete(stop)); stops.add(stop); } for (const [id, owner] of uploadOwners) if (owner.link === link) uploadOwners.delete(id); for (const [key, owner] of uploadAttempts) if (owner === link) uploadAttempts.delete(key); } function makeLink(login: HostLink, primary: boolean): DaemonLink { - const link = new DaemonLink(login, "l" + ++linkId, { allowHttp: options.allowHttp, primary, log, + const link = new DaemonLink(login, "l" + ++linkId, { allowHttp: options.allowHttp, primary, log, keepOnVersionMismatch: true, hello: (link, replay) => encodeLine({ type: "hello", v: link.version, ompVersion, ompwebVersion: packageJson.version, workspaceOps: Object.keys(WORKSPACE_OP_PARAMS), platform: process.platform, home: homedir(), autoApproveExtension: localHost.autoApproveExtension(), ack: link.ackedDown, channels: channels.listing(link, replay), @@ -295,7 +303,18 @@ export function runHostDaemon(options: DaemonOptions): HostDaemon { }, }); services.set(link.id, createWorkspaceService(uploads, (id, epoch, event) => link.queue({ type: "workspace_file", id, epoch, event }, "watch:" + id))); - links.set(login.url, link); return link; + links.set(login.url, link); + const maintenance = new DaemonMaintenance(login, { + hello: () => ({ type: "hello", apiVersion: MAINTENANCE_API_VERSION, protocolVersion: LINK_PROTOCOL_VERSION, + ompwebVersion: packageJson.version, ompVersion, platform: process.platform, + capabilities: ["status", "listSessions", "checkUpdates", "updateSoftware"] }), + handleRequest: (op, args) => runMaintenance(link, op, args), + reload: () => { reconcile(link, true); }, + log, + }); + maintenanceLinks.set(link.id, maintenance); + maintenance.start(); + return link; } function reconcile(source?: DaemonLink, unauthorized = false): boolean { @@ -315,7 +334,7 @@ export function runHostDaemon(options: DaemonOptions): HostDaemon { for (const [url, link] of links) { const login = wanted.get(url); if (login?.token === link.login.token) continue; - if (login) { link.rekey(login.token); continue; } + if (login) { link.rekey(login.token); maintenanceLinks.get(link.id)?.rekey(login.token); continue; } links.delete(url); channels.linkGone(link); dropLink(link); void link.stop(); } for (const login of wanted.values()) { @@ -327,6 +346,32 @@ export function runHostDaemon(options: DaemonOptions): HostDaemon { reportConfig(); electNotifier(); return source ? links.get(source.login.url) !== source || source.login.token !== previousToken : true; } + async function runMaintenance(link: DaemonLink, op: MaintenanceOp, args: Record): Promise { + if (stopped) throw new HostOpError(503, "host_stopping", "This host is stopping"); + const update = op === "checkUpdates" || op === "updateSoftware"; + if (update) nativeUpdatesInFlight++; + try { + if (op === "status") return { + apiVersion: MAINTENANCE_API_VERSION, protocolVersion: LINK_PROTOCOL_VERSION, + ompwebVersion: packageJson.version, ompVersion, platform: process.platform, + capabilities: ["status", "listSessions", "checkUpdates", "updateSoftware"], + workspaceOps: Object.keys(WORKSPACE_OP_PARAMS), sessionProcesses: channels.size, + activeTurns: channels.activeTurns, sessions: channels.maintenanceSessions(), + drained, busyReason: busyReason(), operation: getHostUpdateOperation(), + }; + if (op === "listSessions") { + const states = new Map(channels.maintenanceSessions().map(session => [session.id, session])); + return (await sessionOps.listSessions()).map(session => ({ + id: session.id, title: session.name ?? null, cwd: session.cwd, modified: session.modified, + processRunning: states.has(session.id), active: states.get(session.id)?.active ?? false, + })); + } + return await runOp(link, op, args); + } finally { + if (update) nativeUpdatesInFlight--; + } + } + function handleDown(link: DaemonLink, message: DownMessage | UnknownMessage): void { try { switch (message.type) { @@ -453,7 +498,7 @@ export function runHostDaemon(options: DaemonOptions): HostDaemon { resume.freeze(); // The configuration first: its requests to a server are aborted at once, so no start waiting on them holds the stop. const configStopped = config.stop(); - stopping = (async () => { await Promise.all([...links.values()].map(link => link.stop())); await Promise.all([...stops, ...[...services.values()].map(service => service.stop())]); await (await hostSocket)?.close(); await channels.stop(); await configStopped; await forwarder.stop(); resume.dispose(); finish(code); })(); + stopping = (async () => { await Promise.all([...links.values()].map(link => link.stop())); await Promise.all([...maintenanceLinks.values()].map(link => link.stop())); await Promise.all([...stops, ...[...services.values()].map(service => service.stop())]); await (await hostSocket)?.close(); await channels.stop(); await configStopped; await forwarder.stop(); resume.dispose(); finish(code); })(); return stopping; } // Links first: a recovered sharing session needs its link to start. diff --git a/lib/hosts/hosts-route.test.mjs b/lib/hosts/hosts-route.test.mjs index b6ca7a668..a723e8ad3 100644 --- a/lib/hosts/hosts-route.test.mjs +++ b/lib/hosts/hosts-route.test.mjs @@ -438,7 +438,7 @@ test("caller telemetry cannot make offline or refused-version members eligible; hub.closeHostLink(ready.host.id); }); const { socket: refused } = upgradeSocket(old.token); - refused.frame(`${JSON.stringify({ type: "hello", v: 10 })}\n`); + refused.frame(`${JSON.stringify({ type: "hello", v: 9 })}\n`); assert.equal(refused.closed?.code, 4426, "a daemon below the minimum link protocol is refused"); await readyLink.dial(); const pool = seedPools(t, credentials.map(({ host }) => host.id)); diff --git a/lib/hosts/hub.test.mjs b/lib/hosts/hub.test.mjs index 3be65b2c0..723895bdf 100644 --- a/lib/hosts/hub.test.mjs +++ b/lib/hosts/hub.test.mjs @@ -171,8 +171,8 @@ test("a missing, malformed or wrong bearer token gets 401 before any upgrade or test("a hello with another version is closed 4426 with the server's version and marks the host needs-upgrade", async () => { const { host, token } = createHost("Other version"); - // 10: a prerelease daemon whose channels may still carry omp RPC frames. - for (const v of [LINK_PROTOCOL_VERSION + 1, 10]) { + // The v10/v11 shim is upgrade-only; earlier protocols remain unsupported. + for (const v of [LINK_PROTOCOL_VERSION + 1, 9]) { const socket = dial(token, hello({ v })); assert.deepEqual(socket.closed, { code: 4426, reason: JSON.stringify({ serverVersion: LINK_PROTOCOL_VERSION }) }); const listed = hub.listHosts().find((h) => h.id === host.id); diff --git a/lib/hosts/hub.ts b/lib/hosts/hub.ts index a1dd1e78d..d741006cc 100644 --- a/lib/hosts/hub.ts +++ b/lib/hosts/hub.ts @@ -55,6 +55,13 @@ import type { HostSoftwareUpdate, HostUpdateInfo, HostUpdateOperation, HostUpdat import { isUpdateDisabled } from "../update-policy"; import { WORKSPACE_OP_PARAMS, type WorkspaceFileEvent, type WorkspaceOps } from "../workspace-types"; import { validWorkspaceResult } from "./workspace-results"; +import { closeMaintenanceHost, freezeMaintenance, getMaintenanceHost, MaintenanceError, onMaintenanceChange, requestMaintenance } from "./maintenance-hub"; +import type { MaintenanceHello } from "./maintenance-protocol"; + +/** Old daemons keep their updater, never their session protocol. */ +const legacyUpgradeVersion = (version: number | null): boolean => version === 10 || version === 11; +const upgradeOnly = (link: Link): boolean => legacyUpgradeVersion(link.hostVersion); +const upgradeOp = (op: string): boolean => op === "checkUpdates" || op === "updateSoftware"; /** Liveness and op timeouts from LINK_LIMITS; tests shorten them. */ export const linkTimings = { @@ -389,6 +396,9 @@ function flush(link: Link): void { * server must not reach the children. */ function sendDown(link: Link, body: Unsequenced, key: string): void { if (hub().frozen) return; + if (upgradeOnly(link) && (body.type !== "req" || !upgradeOp(body.op))) { + throw new HostOpError(426, "host_upgrade_required", "Update this host before using its application protocol"); + } const line = encodeQueued(body); const size = Buffer.byteLength(line); // Ahead of it: what is queued or unacked on this link (counters, no scan); fairness may send it before some of that. @@ -501,6 +511,7 @@ function endSocket(link: Link): void { failAllOps(link, () => new HostOpError(503, "host_offline", `${labelOf(link.id)} is offline`)); notify(hub().uploadListeners, link.id, null); endWorkspaceWatches(link.id); + notify(hub().changeListeners); } function stopTimers(link: Link): void { @@ -563,6 +574,7 @@ const stopping = () => json({ error: "omp-web is shutting down" }, 503); */ export function freezeHostLinks(): void { hub().frozen = true; + freezeMaintenance(); for (const link of hub().links.values()) { link.capacity = null; recordDropped(link); @@ -683,7 +695,11 @@ function onFirstFrame(ws: LinkSocket, data: string | Uint8Array): void { } catch { return refuse(); } - if (versionMismatch(link, raw)) return ws.close(LINK_CLOSE.versionMismatch, JSON.stringify({ serverVersion: LINK_PROTOCOL_VERSION })); + const version = helloVersion(raw); + const legacy = legacyUpgradeVersion(version); + if ((!legacy && versionMismatch(link, raw)) || (legacy && link.socket && !upgradeOnly(link))) { + return ws.close(LINK_CLOSE.versionMismatch, JSON.stringify({ serverVersion: LINK_PROTOCOL_VERSION })); + } let hello: UpMessage | UnknownMessage; try { hello = decodeUp(data); @@ -697,6 +713,14 @@ function onFirstFrame(ws: LinkSocket, data: string | Uint8Array): void { function openLink(link: Link, ws: LinkSocket, hello: Hello): void { // One link per host: the newest wins and the previous daemon exits. + if (legacyUpgradeVersion(hello.v) || upgradeOnly(link)) { + // Nothing from another protocol, or an uncertain previous update, is replayed through the shim. + failAllOps(link, () => new HostOpError(503, "transport_lost", "The host update connection was replaced")); + link.sender = new LinkSender(); + link.outbox = new Outbox(); + link.outboxBytes = 0; + link.unacked = 0; + } const previous = link.socket; link.socket = ws; link.capacity = null; @@ -723,8 +747,8 @@ function openLink(link: Link, ws: LinkSocket, hello: Hello): void { released(link, link.outbox.ack(hello.ack)); link.unknownLog.reset(); link.hostVersion = hello.v; - link.shared = true; - link.notifier = hello.link?.notifier === true; + link.shared = !upgradeOnly(link); + link.notifier = !upgradeOnly(link) && hello.link?.notifier === true; enqueue(link, encodeLine({ type: "welcome", v: hello.v, hostId: link.id })); // Resend what the daemon has not applied, in order and paced, before anything new. link.sentSeq = hello.ack; @@ -733,7 +757,7 @@ function openLink(link: Link, ws: LinkSocket, hello: Hello): void { flush(link); link.ompVersion = hello.ompVersion; link.ompwebVersion = hello.ompwebVersion ?? null; - link.workspaceOps = new Set(hello.workspaceOps ?? []); + link.workspaceOps = new Set(upgradeOnly(link) ? [] : hello.workspaceOps ?? []); endWorkspaceWatches(link.id); link.platform = hello.platform; link.autoApprove = hello.autoApproveExtension; @@ -742,6 +766,11 @@ function openLink(link: Link, ws: LinkSocket, hello: Hello): void { clearInterval(link.pingTimer); link.pingTimer = setInterval(() => enqueue(link, encodeLine({ type: "ping" })), linkTimings.pingMs); link.pingTimer.unref?.(); + if (upgradeOnly(link)) { + link.listedSessions = new Set(); + setStatus(link, "needs-upgrade"); + return; + } setStatus(link, hello.ompVersion === null ? "omp-missing" : "online"); reattach(link, hello.channels); notify(hub().helloListeners, link.id, hello.channels); @@ -903,6 +932,10 @@ function configCurrent(link: Link): boolean { /** `whole`: for a `part` that completes a line, that line decoded. `joined`: that line itself, which gets no dedup of * its own (its parts were deduped). */ function applyUp(link: Link, message: UpMessage | UnknownMessage, whole?: UpMessage | UnknownMessage, joined = false): void { + if (upgradeOnly(link) && !["ping", "pong", "hello", "ack", "unknown", "part", "res"].includes(message.type)) { + if ("seq" in message && message.seq !== null) link.up.applied(message.seq); + return; + } if (message.type === "ping") return enqueue(link, encodeLine({ type: "pong" })); if (message.type === "pong" || message.type === "hello" || message.type === "ack") return; if (message.type === "capacity") { @@ -991,7 +1024,10 @@ function request(hostId: string, op: HostOpName, args: Record, const link = linkFor(hostId); const label = labelOf(hostId); if (hub().frozen) return Promise.reject(new HostOpError(503, "server_stopping", "omp-web is shutting down")); - if (!link.socket || !isLinkUp(link.status)) { + if (upgradeOnly(link) && !upgradeOp(op)) { + return Promise.reject(new HostOpError(426, "host_upgrade_required", "Update this host before using its application protocol")); + } + if (!link.socket || (!isLinkUp(link.status) && !upgradeOnly(link))) { return Promise.reject(new HostOpError(503, "host_offline", `${label} is offline`)); } const { promise, resolve, reject } = Promise.withResolvers(); @@ -1560,6 +1596,16 @@ export function getHost(id: string): Host | null { const updateCompatibility = () => ({ min: MIN_LINK_PROTOCOL_VERSION, max: LINK_PROTOCOL_VERSION }); +async function requestHostUpdate(id: string, op: "checkUpdates" | "updateSoftware", args: Record): Promise { + if (!getMaintenanceHost(id)) return request(id, op, args); + try { + return await requestMaintenance(id, op, args); + } catch (error) { + if (error instanceof MaintenanceError) throw new HostOpError(error.status, error.code, error.message); + throw error; + } +} + function requireUpdateHost(id: string): void { if (!getHost(id)) throw new HostOpError(404, "host_not_found", "Unknown host"); if (isUpdateDisabled()) throw new HostOpError(403, "updates_disabled", "Updates are disabled on this web server"); @@ -1581,7 +1627,7 @@ function updateOperation(value: unknown): value is HostUpdateOperation { /** Every machine, including this one, checks on its daemon. Never fall back to the web server's installation. */ export async function getHostSoftwareUpdates(id: string, force = false): Promise { requireUpdateHost(id); - const result = await request(id, "checkUpdates", { compatibility: updateCompatibility(), force }); + const result = await requestHostUpdate(id, "checkUpdates", { compatibility: updateCompatibility(), force }); if (!isRecord(result) || !softwareUpdate(result.ompweb) || !isRecord(result.omp) || !nullableString(result.omp.currentVersion) || !["npm", "bun", "binary"].includes(String(result.ompweb.installMethod)) || !(result.ompweb.protocolVersion === null || (Number.isSafeInteger(result.ompweb.protocolVersion) && Number(result.ompweb.protocolVersion) > 0)) @@ -1605,7 +1651,7 @@ export async function updateHostSoftware(id: string, target: HostUpdateTarget, v if (!candidate.updateAvailable || !candidate.canUpdate || candidate.availableVersion !== version) { throw new HostOpError(409, "host_update_blocked", candidate.blockedReason ?? "The available version changed; check for updates again"); } - const result = await request(id, "updateSoftware", { target, version, compatibility: updateCompatibility() }); + const result = await requestHostUpdate(id, "updateSoftware", { target, version, compatibility: updateCompatibility() }); if (!isRecord(result) || result.accepted !== true || !updateOperation(result.operation) || result.operation.target !== target || result.operation.version !== version) { throw new HostOpError(502, "host_bad_result", "The host returned an invalid update operation"); @@ -1619,6 +1665,9 @@ export interface LinkedHostRow { status: HostStatus; ompVersion: string | null; ompwebVersion?: string | null; + /** A connected maintenance API or legacy shim can update an incompatible application peer. */ + upgradeOnly?: boolean; + apiVersion?: number; platform: string | null; lastSeenAt: string | null; allowAgentStart: boolean; @@ -1632,20 +1681,31 @@ export interface LinkedHostRow { clusterConfig?: ConfigState & { owned: boolean; current: boolean }; } +function hostConnectionFields(link: Link | undefined, maintenance: MaintenanceHello | null): Pick { + const applicationUp = !!link?.socket && isLinkUp(link.status); + const mismatch = !!maintenance && (maintenance.protocolVersion < MIN_LINK_PROTOCOL_VERSION || maintenance.protocolVersion > LINK_PROTOCOL_VERSION); + return { + status: maintenance && !applicationUp ? mismatch ? "needs-upgrade" : "offline" : link?.status ?? "offline", + ...(!applicationUp && ((link?.socket && upgradeOnly(link)) || mismatch) ? { upgradeOnly: true } : {}), + ...(maintenance ? { apiVersion: maintenance.apiVersion } : {}), + }; +} + export function listHosts(): LinkedHostRow[] { return loadHosts().filter((record) => record.id !== LOCAL_HOST_ID).map((record) => { const link = hub().links.get(record.id); + const maintenance = getMaintenanceHost(record.id); return { id: record.id, label: record.label, - status: link?.status ?? "offline", - ompVersion: link?.ompVersion ?? null, - ompwebVersion: link?.ompwebVersion ?? null, - platform: link?.platform ?? null, + ...hostConnectionFields(link, maintenance), + ompVersion: maintenance?.ompVersion ?? link?.ompVersion ?? null, + ompwebVersion: maintenance?.ompwebVersion ?? link?.ompwebVersion ?? null, + platform: maintenance?.platform ?? link?.platform ?? null, lastSeenAt: link?.lastSeenAt ?? record.lastSeenAt, allowAgentStart: record.allowAgentStart, shareCredentials: record.shareCredentials, - hostVersion: link?.hostVersion ?? null, + hostVersion: link?.hostVersion ?? maintenance?.protocolVersion ?? null, serverVersion: LINK_PROTOCOL_VERSION, workspaceOps: [...(link?.workspaceOps ?? [])], ...(link?.configState ? { clusterConfig: { ...link.configState, owned: link.configOwned, current: configCurrent(link) } } : {}), @@ -1763,6 +1823,8 @@ export interface LocalHostRow { status: HostStatus; ompVersion: string | null; ompwebVersion?: string | null; + upgradeOnly?: boolean; + apiVersion?: number; platform: string | null; lastSeenAt: null; hostVersion: number | null; @@ -1781,15 +1843,16 @@ export type HostRow = LocalHostRow | LinkedHostRow; * the members cannot get this machine's configuration. */ export function hostsSnapshot(): { hosts: HostRow[]; pools?: HostPool[]; masterConfigError?: MasterConfigError } { const link = hub().links.get(LOCAL_HOST_ID); + const maintenance = getMaintenanceHost(LOCAL_HOST_ID); const local: LocalHostRow = { id: "local", label: customLocalLabel(), - status: link?.status ?? "offline", - ompVersion: link?.ompVersion ?? null, - ompwebVersion: link?.ompwebVersion ?? null, - platform: link?.platform ?? null, + ...hostConnectionFields(link, maintenance), + ompVersion: maintenance?.ompVersion ?? link?.ompVersion ?? null, + ompwebVersion: maintenance?.ompwebVersion ?? link?.ompwebVersion ?? null, + platform: maintenance?.platform ?? link?.platform ?? null, lastSeenAt: null, - hostVersion: link?.hostVersion ?? null, + hostVersion: link?.hostVersion ?? maintenance?.protocolVersion ?? null, serverVersion: LINK_PROTOCOL_VERSION, workspaceOps: [...(link?.workspaceOps ?? [])], serverPlatform: process.platform, @@ -1840,8 +1903,10 @@ export function getHostOmpVersion(hostId: string): string | null { export function onHostsChange(listener: () => void): () => void { const listeners = hub().changeListeners; listeners.add(listener); + const detachMaintenance = onMaintenanceChange(listener); return () => { listeners.delete(listener); + detachMaintenance(); }; } @@ -1934,7 +1999,7 @@ export function setHostUploadRetention(hostId: string, days: number | null): Pro /** Queue upload bytes (at most pieceBytes) for `hostId`'s daemon, fair with its other traffic; false when it has no socket. */ export function sendUploadFrame(hostId: string, id: string, bytes: Uint8Array): boolean { const link = hub().links.get(hostId); - if (!link?.socket || hub().frozen) return false; + if (!link?.socket || hub().frozen || upgradeOnly(link)) return false; link.sender.push(`up:${id}`, { id, blob: bytes }); flush(link); return true; @@ -1956,6 +2021,7 @@ export function onUploadEvent(listener: (hostId: string, ack: UploadAck | null) /** End a host's link now (revoke). Its daemon gets 401 when it redials and stops its children. */ export function closeHostLink(id: string): void { + closeMaintenanceHost(id); const link = hub().links.get(id); if (!link) return; hub().links.delete(id); diff --git a/lib/hosts/link-protocol.ts b/lib/hosts/link-protocol.ts index b7a24e377..a95657cc3 100644 --- a/lib/hosts/link-protocol.ts +++ b/lib/hosts/link-protocol.ts @@ -11,6 +11,8 @@ import type { SpawnRequest } from "./host"; import { isWorkspaceOp, WORKSPACE_OP_PARAMS } from "../workspace-types"; import type { WorkspaceFileEvent } from "../workspace-types"; +/** Application Protocol v12 uses maintenance API v1 on a separate authenticated connection. + * v10/v11 application links are accepted only by the upgrade shim, never by the session runtime. */ /** 11: the same wire as 10, renumbered: prerelease daemons already sent 10 with omp RPC channel lines, which this * server cannot read. 10: channel stdin/stdout lines are the runner's `{t}` frames (`cmd`/`res`/`hello`/…), no longer omp RPC frames; * `open.launch.configVersion`, the `config_state` report and the `clusterConfig` op. @@ -21,10 +23,9 @@ import type { WorkspaceFileEvent } from "../workspace-types"; * (docs/specs/2026-10-31-websocket-link-design.md, D7). 4: the link is one WebSocket: text frames carry the messages, * binary frames large results, `ack` messages replace the ack header and the POST answer. * Rules for changing it: docs/specs/2026-10-28-remote-hosts-design.md, "Protocol versions and compatibility". */ -export const LINK_PROTOCOL_VERSION = 11; -/** The oldest daemon protocol the server still links (older ones get close 4426). Dropping one is an explicit decision: - * 11 dropped all older ones, whose channel lines mean (or, for prerelease 10, may mean) omp RPC frames. */ -export const MIN_LINK_PROTOCOL_VERSION = 11; +export const LINK_PROTOCOL_VERSION = 12; +/** Oldest full application protocol; the v10/v11 update-only shim is outside this range. */ +export const MIN_LINK_PROTOCOL_VERSION = 12; /** A configuration version: the manifest's SHA-256, in lowercase hex. */ export const CONFIG_VERSION_RE = /^[0-9a-f]{64}$/; /** Safe identity shared by spawn requests and reconnect channel metadata. */ diff --git a/lib/hosts/maintenance-daemon.integration.test.mjs b/lib/hosts/maintenance-daemon.integration.test.mjs new file mode 100644 index 000000000..83f2b16a5 --- /dev/null +++ b/lib/hosts/maintenance-daemon.integration.test.mjs @@ -0,0 +1,145 @@ +import assert from "node:assert/strict"; +import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test, { after } from "node:test"; + +const root = mkdtempSync(join(tmpdir(), "ompweb-maintenance-runtime-")); +const oldAgent = process.env.PI_CODING_AGENT_DIR, oldBin = process.env.OMP_WEB_OMP_BIN; +process.env.PI_CODING_AGENT_DIR = root; +process.env.OMP_WEB_OMP_BIN = join(root, "missing-omp"); +const { runHostDaemon } = await import("./daemon.ts"); +const hub = await import("./hub.ts"); +const { createHost, revokeHost } = await import("./registry.ts"); +const maintenance = await import("./maintenance-hub.ts"); +const { until } = await import("./link-driver.test-helper.mjs"); +after(() => { + if (oldAgent === undefined) delete process.env.PI_CODING_AGENT_DIR; else process.env.PI_CODING_AGENT_DIR = oldAgent; + if (oldBin === undefined) delete process.env.OMP_WEB_OMP_BIN; else process.env.OMP_WEB_OMP_BIN = oldBin; + rmSync(root, { recursive: true, force: true }); +}); + +test("API v1 keeps the actual daemon available after application protocol rejection", async (t) => { + const { host, token } = createHost("Maintenance only"); + const sessionId = "11111111-2222-4333-8444-5555555555cc"; + const sessions = join(root, "sessions", "-maintenance"); + mkdirSync(sessions, { recursive: true }); + writeFileSync(join(sessions, "session.jsonl"), JSON.stringify({ type: "session", version: 3, id: sessionId, cwd: root, timestamp: "2026-11-14T00:00:00.000Z" }) + "\n"); + const updateRoot = join(root, "omp-web", "host-update"); + const attempt = "api-only-health"; + mkdirSync(join(updateRoot, attempt), { recursive: true, mode: 0o700 }); + writeFileSync(join(updateRoot, "status.json"), JSON.stringify({ + operation: { id: attempt, target: "ompweb", version: "0.7.1", state: "running" }, daemonPid: 1, startedAt: Date.now(), + }), { mode: 0o600 }); + writeFileSync(join(updateRoot, attempt, "descriptor.json"), JSON.stringify({ id: attempt, version: "0.7.1", fromVersion: "0.7.0" }), { mode: 0o600 }); + let rejected = 0; + const server = Bun.serve({ + port: 0, hostname: "127.0.0.1", + fetch(request, server) { + if (new URL(request.url).pathname === "/api/hosts/maintenance") return maintenance.handleMaintenanceUpgrade(request, (data) => server.upgrade(request, { data })); + if (request.headers.get("upgrade") !== "websocket") return new Response("", { status: 426 }); + return server.upgrade(request, { data: { kind: "application" } }) ? undefined : new Response("", { status: 426 }); + }, + websocket: { + message(ws, data) { + if (ws.data.kind === "maintenance") return maintenance.maintenanceSocket.message(ws, data); + rejected++; + ws.close(4426, JSON.stringify({ serverVersion: 99 })); + }, + close(ws) { if (ws.data.kind === "maintenance") maintenance.maintenanceSocket.close(ws); }, + }, + }); + const daemon = runHostDaemon({ allowHttp: false, links: [{ url: `http://127.0.0.1:${server.port}`, token }] }); + t.after(async () => { await daemon.stop(); server.stop(true); rmSync(updateRoot, { recursive: true, force: true }); }); + await daemon.started; + await until(() => rejected > 0 && maintenance.getMaintenanceHost(host.id), "the independent API connects despite application rejection"); + const hello = maintenance.getMaintenanceHost(host.id); + assert.equal(hello.apiVersion, 1); + assert.equal(hello.protocolVersion, 12); + const status = await maintenance.requestMaintenance(host.id, "status", {}); + assert.equal(status.apiVersion, 1); + assert.equal(status.protocolVersion, 12); + assert.equal(status.sessionProcesses, 0); + assert.equal(status.busyReason, null, "an idle status read cannot count itself as pending work"); + assert.equal(existsSync(join(updateRoot, attempt, "health.json")), false, "maintenance alone cannot suppress rollback of a broken application connection"); + assert.ok(status.capabilities.includes("updateSoftware")); + const inventory = await maintenance.requestMaintenance(host.id, "listSessions", {}); + const session = inventory.find((item) => item.id === sessionId); + assert.ok(session, "maintenance lists real session metadata, not application transcript frames"); + assert.equal(session.active, false); + assert.equal(session.processRunning, false); + const row = hub.listHosts().find((item) => item.id === host.id); + assert.equal(row.status, "offline", "a compatible application endpoint outage is not a version mismatch"); + assert.equal(row.apiVersion, 1); + assert.equal(row.upgradeOnly, undefined); + assert.equal(daemon.control("status").linked, false, "maintenance must not pretend application sessions can attach"); + revokeHost(host.id); + maintenance.closeMaintenanceHost(host.id); + await until(() => !maintenance.getMaintenanceHost(host.id), "revocation ends maintenance access"); +}); + +test("maintenance reports a live prompt as active without inspecting its application transcript", async (t) => { + const { host, token } = createHost("Activity inventory"); + const id = "11111111-2222-4333-8444-5555555555dd"; + const sessionDir = join(root, "sessions", "-activity"); + mkdirSync(sessionDir, { recursive: true }); + const file = join(sessionDir, "active.jsonl"); + writeFileSync(file, JSON.stringify({ type: "session", version: 3, id, cwd: root }) + "\n"); + const fakeOmp = join(root, process.platform === "win32" ? "activity-omp.cmd" : "activity-omp"); + const source = `if (process.argv.includes("--version")) { console.log("99.0.0-fake"); process.exit(0); } +const emit = frame => process.stdout.write(JSON.stringify(frame) + "\\n"); +require("node:readline").createInterface({input:process.stdin}).on("line", line => { + const command = JSON.parse(line); + if (command.t === "cmd") emit({t:"res",id:command.id,ok:true,data:{}}); + if (command.op === "prompt") emit({t:"event",event:{type:"agent_start"}}); +}); +emit({t:"hello",proto:1,pid:process.pid,sessionId:${JSON.stringify(id)},cwd:${JSON.stringify(root)},sessionFile:${JSON.stringify(file)}}); +`; + if (process.platform === "win32") { + writeFileSync(fakeOmp + ".cjs", source); + writeFileSync(fakeOmp, `@"${process.execPath}" "${fakeOmp}.cjs" %*\r\n`); + } else writeFileSync(fakeOmp, `#!${process.execPath}\n${source}`, { mode: 0o755 }); + process.env.OMP_WEB_OMP_BIN = fakeOmp; + const { resolveOmpBin } = await import("../omp/omp-cli.ts"); + resolveOmpBin(true); + let socket, seq = 0; + const messages = []; + const server = Bun.serve({ + port: 0, hostname: "127.0.0.1", + fetch(request, server) { + if (new URL(request.url).pathname === "/api/hosts/maintenance") return maintenance.handleMaintenanceUpgrade(request, data => server.upgrade(request, { data })); + return server.upgrade(request, { data: { kind: "application" } }) ? undefined : new Response("", { status: 426 }); + }, + websocket: { + message(ws, data) { + if (ws.data.kind === "maintenance") return maintenance.maintenanceSocket.message(ws, data); + for (const line of String(data).split("\n").filter(Boolean)) { + const frame = JSON.parse(line); + if (frame.type === "hello") { socket = ws; ws.send(JSON.stringify({ type: "welcome", v: 12, hostId: host.id })); } + else if (frame.seq) { + messages.push(frame); + ws.send(JSON.stringify({ type: "ack", ack: frame.seq })); + } + } + }, + close(ws) { if (ws.data.kind === "maintenance") maintenance.maintenanceSocket.close(ws); }, + }, + }); + const daemon = runHostDaemon({ allowHttp: false, links: [{ url: `http://127.0.0.1:${server.port}`, token }] }); + t.after(async () => { await daemon.stop(); server.stop(true); }); + await daemon.started; + await until(() => socket && maintenance.getMaintenanceHost(host.id), "both connections to be ready"); + socket.send(JSON.stringify({ type: "req", seq: ++seq, id: "open-active", op: "open", args: { sessionId: id, launch: { cwd: root, recordedCwd: root, sessionFile: file, advisor: false } } })); + await until(() => messages.some(frame => frame.type === "res" && frame.id === "open-active"), "the real daemon to start its child"); + const opened = messages.find(frame => frame.type === "res" && frame.id === "open-active"); + assert.equal(opened.ok, true, JSON.stringify(opened)); + socket.send(JSON.stringify({ type: "stdin", seq: ++seq, ch: opened.result.ch, line: JSON.stringify({ t: "cmd", id: 1, op: "prompt", args: { message: "work" } }) + "\n" })); + await until(() => messages.some(frame => frame.type === "stdout" && JSON.parse(frame.line).event?.type === "agent_start"), "the prompt to become active"); + const status = await maintenance.requestMaintenance(host.id, "status", {}); + assert.equal(status.sessionProcesses, 1); + assert.equal(status.sessions.find(session => session.id === id)?.active, true); + assert.ok(status.busyReason, "an active child still blocks updating its daemon"); + const inventory = await maintenance.requestMaintenance(host.id, "listSessions", {}); + assert.equal(inventory.find(session => session.id === id)?.active, true); + assert.equal(inventory.find(session => session.id === id)?.processRunning, true); +}); diff --git a/lib/hosts/maintenance-hub.test.mjs b/lib/hosts/maintenance-hub.test.mjs new file mode 100644 index 000000000..86590039a --- /dev/null +++ b/lib/hosts/maintenance-hub.test.mjs @@ -0,0 +1,300 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +const dir = mkdtempSync(join(tmpdir(), "omp-maintenance-hub-")); +const previousAgentDir = process.env.PI_CODING_AGENT_DIR; +process.env.PI_CODING_AGENT_DIR = dir; +const hub = await import("./maintenance-hub.ts"); +const { MAINTENANCE_CLOSE, MAINTENANCE_LIMITS, MaintenanceError, maintenanceTimings } = await import("./maintenance-protocol.ts"); +const { createHost, revokeHost } = await import("./registry.ts"); +const timings = { ...maintenanceTimings }; + +test.after(() => { + Object.assign(maintenanceTimings, timings); + if (previousAgentDir === undefined) delete process.env.PI_CODING_AGENT_DIR; + else process.env.PI_CODING_AGENT_DIR = previousAgentDir; + rmSync(dir, { recursive: true, force: true }); +}); + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +const hello = (extra = {}) => ({ type: "hello", apiVersion: 1, protocolVersion: 12, ompwebVersion: "0.7.1", ompVersion: "14.0.0", platform: "linux", capabilities: ["status", "listSessions", "checkUpdates", "updateSoftware"], ...extra }); + +/** The hub's view of a socket: what it sent and how it was closed. */ +function socket(hostId) { + return { + data: { kind: "maintenance", hostId }, + sent: [], + closed: null, + terminated: false, + send(line) { this.sent.push(JSON.parse(line)); return line.length; }, + close(code, reason) { this.closed = { code, reason }; }, + terminate() { this.terminated = true; }, + }; +} +const say = (ws, frame) => hub.maintenanceSocket.message(ws, typeof frame === "string" ? frame : JSON.stringify(frame)); +const requests = (ws) => ws.sent.filter((frame) => frame.type === "req"); +const request = (headers) => new Request("http://server/api/hosts/maintenance", { headers }); +const reject = (promise) => promise.then(() => assert.fail("expected a rejection"), (error) => error); + +function reset() { + for (const link of [...(globalThis.__ompMaintenanceHub?.links.values() ?? [])]) hub.closeMaintenanceHost(link.id); + globalThis.__ompMaintenanceHub = undefined; + Object.assign(maintenanceTimings, timings); +} +const it = (name, body) => test(name, async (t) => { t.after(reset); await body(t); }); + +/** A registered host whose daemon has said hello on a fresh socket. */ +function linked(extra) { + const { host } = createHost("h"); + const ws = socket(host.id); + say(ws, hello(extra)); + return { id: host.id, ws }; +} + +it("authenticates before discovery or upgrade", () => { + const { token } = createHost("auth"); + let upgraded = null; + const upgrade = (data) => { upgraded = data; return true; }; + assert.equal(hub.handleMaintenanceUpgrade(request({ upgrade: "websocket" }), upgrade).status, 401); + assert.equal(hub.handleMaintenanceUpgrade(request({ authorization: "Bearer wrong", upgrade: "websocket" }), upgrade).status, 401); + assert.equal(hub.handleMaintenanceUpgrade(request(), upgrade).status, 401); + assert.equal(upgraded, null); + assert.equal(hub.handleMaintenanceUpgrade(request({ authorization: `Bearer ${token}`, upgrade: "websocket" }), upgrade), undefined); + assert.equal(upgraded.kind, "maintenance"); + assert.equal(hub.handleMaintenanceUpgrade(request({ authorization: `Bearer ${token}`, upgrade: "websocket" }), () => false).status, 426); +}); + +it("answers a plain GET with the API version, never a socket", async () => { + const { token } = createHost("discovery"); + const response = hub.handleMaintenanceUpgrade(request({ authorization: `Bearer ${token}` }), () => assert.fail("no upgrade")); + assert.equal(response.status, 200); + assert.equal(response.headers.get("cache-control"), "no-store"); + assert.deepEqual(await response.json(), { apiVersion: 1, capabilities: ["status", "listSessions", "checkUpdates", "updateSoftware"] }); +}); + +it("adds the server's software to the discovery answer when given", async () => { + const { token } = createHost("server-info"); + const response = hub.handleMaintenanceUpgrade(request({ authorization: `Bearer ${token}` }), () => assert.fail("no upgrade"), { protocolVersion: 12, ompwebVersion: "0.7.1" }); + assert.deepEqual(await response.json(), { protocolVersion: 12, ompwebVersion: "0.7.1", apiVersion: 1, capabilities: ["status", "listSessions", "checkUpdates", "updateSoftware"] }); +}); + +it("welcomes a hello whatever the application protocol version", async () => { + const seen = []; + const off = hub.onMaintenanceChange(() => seen.push(1)); + const { id, ws } = linked({ protocolVersion: 99 }); + assert.deepEqual(ws.sent, [{ type: "welcome", apiVersion: 1 }]); + assert.equal(hub.getMaintenanceHost(id).protocolVersion, 99); + assert.equal(seen.length, 1); + const pending = hub.requestMaintenance(id, "status", { a: 1 }); + const [asked] = requests(ws); + assert.deepEqual({ op: asked.op, args: asked.args }, { op: "status", args: { a: 1 } }); + say(ws, { type: "res", id: asked.id, ok: true, result: { busy: false } }); + assert.deepEqual(await pending, { busy: false }); + off(); +}); + +it("refuses another API version, even when the rest of its hello is not ours", () => { + const { host } = createHost("v2"); + const ws = socket(host.id); + say(ws, { type: "hello", apiVersion: 2, whatever: [] }); + assert.deepEqual(ws.closed, { code: MAINTENANCE_CLOSE.apiMismatch, reason: JSON.stringify({ apiVersion: 1 }) }); + assert.equal(hub.getMaintenanceHost(host.id), null); +}); + +it("closes a socket whose first frame is not a valid hello", () => { + const { host } = createHost("bad"); + const closeOf = (frame) => { const ws = socket(host.id); say(ws, frame); return ws.closed.code; }; + assert.equal(closeOf({ type: "res", id: "x" }), MAINTENANCE_CLOSE.policy); + assert.equal(closeOf("not json"), MAINTENANCE_CLOSE.policy); + assert.equal(closeOf(hello({ platform: 3 })), MAINTENANCE_CLOSE.policy); + assert.equal(closeOf(JSON.stringify({ type: "hello", pad: "x".repeat(MAINTENANCE_LIMITS.frameBytes) })), MAINTENANCE_CLOSE.tooBig); + const binary = socket(host.id); + hub.maintenanceSocket.message(binary, new Uint8Array(4)); + assert.equal(binary.closed.code, MAINTENANCE_CLOSE.policy); + assert.equal(hub.getMaintenanceHost(host.id), null); +}); + +it("closes a socket that never says hello", async () => { + maintenanceTimings.helloMs = 20; + const { host } = createHost("silent"); + const ws = socket(host.id); + hub.maintenanceSocket.open(ws); + await sleep(60); + assert.equal(ws.closed.code, MAINTENANCE_CLOSE.policy); + const welcomed = socket(host.id); + hub.maintenanceSocket.open(welcomed); + say(welcomed, hello()); + await sleep(60); + assert.equal(welcomed.closed, null); +}); + +it("rechecks the registry after the hello and rejects a revoked host", () => { + const { host } = createHost("revoked"); + const ws = socket(host.id); + revokeHost(host.id); + say(ws, hello()); + assert.equal(ws.closed.code, MAINTENANCE_CLOSE.policy); + assert.equal(hub.getMaintenanceHost(host.id), null); +}); + +it("drops capabilities it does not know and never sends an op the host did not advertise", async () => { + const { id, ws } = linked({ capabilities: ["status", "future"] }); + assert.deepEqual(hub.getMaintenanceHost(id).capabilities, ["status"]); + const error = await reject(hub.requestMaintenance(id, "updateSoftware", {})); + assert.deepEqual([error.status, error.code], [501, "unsupported_op"]); + assert.equal(requests(ws).length, 0); +}); + +it("fails a request for a host without a link, an oversized one and too many at once", async () => { + assert.equal((await reject(hub.requestMaintenance("nobody", "status", {}))).code, "no_link"); + const { id, ws } = linked(); + const big = await reject(hub.requestMaintenance(id, "status", { text: "x".repeat(MAINTENANCE_LIMITS.frameBytes) })); + assert.deepEqual([big.status, big.code], [413, "too_large"]); + const open = Array.from({ length: MAINTENANCE_LIMITS.inFlight }, () => hub.requestMaintenance(id, "status", {})); + assert.equal((await reject(hub.requestMaintenance(id, "status", {}))).code, "busy"); + assert.equal(requests(ws).length, MAINTENANCE_LIMITS.inFlight); + for (const asked of requests(ws)) say(ws, { type: "res", id: asked.id, ok: true, result: null }); + assert.deepEqual(await Promise.all(open), open.map(() => null)); +}); + +it("keeps the host's native error status and code", async () => { + const { id, ws } = linked(); + const failed = hub.requestMaintenance(id, "updateSoftware", {}); + say(ws, { type: "res", id: requests(ws)[0].id, ok: false, status: 409, code: "host_busy", message: "Sessions are running" }); + const error = await reject(failed); + assert.ok(error instanceof MaintenanceError); + assert.deepEqual([error.status, error.code, error.message], [409, "host_busy", "Sessions are running"]); + const odd = hub.requestMaintenance(id, "status", {}); + say(ws, { type: "res", id: requests(ws)[1].id, ok: false, status: 200, code: "Not Snake", message: "m" }); + const normalized = await reject(odd); + assert.deepEqual([normalized.status, normalized.code], [500, "internal"]); +}); + +it("a malformed error response rejects its request instead of losing its settlement", async () => { + const { id, ws } = linked(); + const failed = hub.requestMaintenance(id, "status", {}); + say(ws, { type: "res", id: requests(ws)[0].id, ok: false, toString: null }); + const error = await Promise.race([reject(failed), sleep(100).then(() => null)]); + assert.ok(error instanceof MaintenanceError, "the malformed response must not leave the caller pending"); + assert.deepEqual([error.status, error.code], [500, "internal"]); +}); + +it("times out without repeating the request and ignores its late answer", async () => { + const { id, ws } = linked(); + const error = await reject(hub.requestMaintenance(id, "status", {}, 20)); + assert.deepEqual([error.status, error.code], [504, "timeout"]); + say(ws, { type: "res", id: requests(ws)[0].id, ok: true, result: 1 }); + assert.equal(requests(ws).length, 1); + const next = hub.requestMaintenance(id, "status", {}); + say(ws, { type: "res", id: requests(ws)[1].id, ok: true, result: 2 }); + assert.equal(await next, 2); +}); + +it("fails a request in flight when the link drops and never replays it on the next link", async () => { + const { id, ws } = linked(); + const lost = hub.requestMaintenance(id, "updateSoftware", { version: "1" }); + const lateId = requests(ws)[0].id; + hub.maintenanceSocket.close(ws); + const error = await reject(lost); + assert.deepEqual([error.status, error.code], [502, "delivery_unknown"]); + assert.equal(hub.getMaintenanceHost(id), null); + const again = socket(id); + say(again, hello()); + assert.deepEqual(again.sent, [{ type: "welcome", apiVersion: 1 }]); + say(ws, { type: "res", id: lateId, ok: true, result: "late" }); + assert.equal(requests(again).length, 0); +}); + +it("lets a newer socket replace the old one, closing and failing only the old", async () => { + const { id, ws: old } = linked(); + const inFlight = hub.requestMaintenance(id, "status", {}); + const fresh = socket(id); + say(fresh, hello({ ompwebVersion: "0.7.2" })); + assert.equal(old.closed.code, MAINTENANCE_CLOSE.replaced); + assert.equal(fresh.closed, null); + assert.equal((await reject(inFlight)).code, "delivery_unknown"); + assert.equal(hub.getMaintenanceHost(id).ompwebVersion, "0.7.2"); + // The old socket's close event and frames belong to no link. + hub.maintenanceSocket.close(old); + say(old, { type: "res", id: "x", ok: true }); + assert.equal(hub.getMaintenanceHost(id).ompwebVersion, "0.7.2"); + const next = hub.requestMaintenance(id, "status", {}); + assert.equal(requests(old).length, 1); + say(fresh, { type: "res", id: requests(fresh)[0].id, ok: true, result: "fresh" }); + assert.equal(await next, "fresh"); +}); + +it("closeMaintenanceHost closes the socket, fails what is in flight and notifies", async () => { + const { id, ws } = linked(); + const seen = []; + const off = hub.onMaintenanceChange(() => seen.push(hub.getMaintenanceHost(id))); + const inFlight = hub.requestMaintenance(id, "status", {}); + hub.closeMaintenanceHost(id); + assert.equal(ws.closed.code, MAINTENANCE_CLOSE.policy); + assert.equal((await reject(inFlight)).code, "delivery_unknown"); + assert.deepEqual(seen, [null]); + off(); + hub.closeMaintenanceHost(id); + assert.equal(seen.length, 1); +}); + +it("a throwing listener does not break the link", () => { + const off = hub.onMaintenanceChange(() => { throw new Error("listener"); }); + const { id } = linked(); + assert.ok(hub.getMaintenanceHost(id)); + off(); +}); + +it("pings, and drops a host that stays silent", async () => { + maintenanceTimings.pingMs = 10; + maintenanceTimings.offlineMs = 45; + const { id, ws } = linked(); + // Handled at once: it fails while the test sleeps. + const outcome = reject(hub.requestMaintenance(id, "status", {})); + await sleep(25); + assert.ok(ws.sent.some((frame) => frame.type === "ping")); + say(ws, { type: "pong" }); + assert.equal(ws.terminated, false); + await sleep(120); + assert.equal(ws.terminated, true); + assert.equal(hub.getMaintenanceHost(id), null); + assert.equal((await outcome).code, "delivery_unknown"); +}); + +it("closes a link whose frame is malformed or too large", () => { + const first = linked(); + say(first.ws, "{"); + assert.equal(first.ws.closed.code, MAINTENANCE_CLOSE.policy); + assert.equal(hub.getMaintenanceHost(first.id), null); + const second = linked(); + say(second.ws, JSON.stringify({ type: "res", pad: "x".repeat(MAINTENANCE_LIMITS.frameBytes) })); + assert.equal(second.ws.closed.code, MAINTENANCE_CLOSE.tooBig); + assert.equal(hub.getMaintenanceHost(second.id), null); +}); + +it("freezing refuses requests and upgrades, closes links 1001 and ignores what arrives after", async () => { + const { token } = createHost("frozen"); + const { id, ws } = linked(); + const inFlight = hub.requestMaintenance(id, "status", {}); + hub.freezeMaintenance(); + assert.equal(ws.closed.code, MAINTENANCE_CLOSE.goingAway); + assert.equal((await reject(inFlight)).code, "delivery_unknown"); + assert.equal(hub.getMaintenanceHost(id), null); + assert.equal((await reject(hub.requestMaintenance(id, "status", {}))).code, "frozen"); + const headers = { authorization: `Bearer ${token}` }; + assert.equal(hub.handleMaintenanceUpgrade(request({ ...headers, upgrade: "websocket" }), () => true).status, 503); + assert.equal(hub.handleMaintenanceUpgrade(request(headers), () => true).status, 503); + const late = socket(id); + say(late, hello()); + assert.equal(late.sent.length, 0); +}); + +it("keeps its state on globalThis across a hot reload", () => { + const { id } = linked(); + const state = globalThis.__ompMaintenanceHub; + assert.ok(state.links.has(id)); + assert.equal(hub.getMaintenanceHost(id).platform, "linux"); +}); diff --git a/lib/hosts/maintenance-hub.ts b/lib/hosts/maintenance-hub.ts new file mode 100644 index 000000000..e8ccbad04 --- /dev/null +++ b/lib/hosts/maintenance-hub.ts @@ -0,0 +1,298 @@ +// Main-server end of the maintenance API (maintenance-protocol.ts): accepts `ompweb host` daemons on +// /api/hosts/maintenance and carries the server's requests to them. Independent of the application link (hub.ts): its own +// sockets, lifecycle and version, so a host whose application protocol differs stays reachable for upgrades. +// Everything a daemon sends is untrusted input. No `next/*` or `@/` imports. +import { randomUUID } from "node:crypto"; +import { hostFromBearer } from "./host-bearer"; +import { + decodeFrame, + encodeFrame, + failure, + MAINTENANCE_API_VERSION, + MAINTENANCE_CLOSE, + MAINTENANCE_LIMITS, + MAINTENANCE_OPS, + MaintenanceError, + maintenanceTimings, + parseHello, + PING, + WELCOME, + type MaintenanceHello, + type MaintenanceOp, +} from "./maintenance-protocol"; +import { readHosts } from "./registry"; + +export { MaintenanceError }; + +/** What the hub keeps on each maintenance socket (Bun's `ws.data`): the host its token named, and whether its hello came. */ +export interface MaintenanceSocketData { + kind: "maintenance"; + hostId: string; + hello?: boolean; +} + +/** The server's software, as discovery reports it for the host to inspect. */ +export interface MaintenanceServerInfo { + protocolVersion: number; + ompwebVersion: string; +} + +/** The part of Bun's ServerWebSocket the hub uses, so tests drive it with a plain object. */ +export interface MaintenanceSocket { + readonly data: MaintenanceSocketData; + send(data: string): number; + close(code?: number, reason?: string): void; + terminate(): void; +} + +interface Pending { + resolve: (value: unknown) => void; + reject: (error: Error) => void; + timer: NodeJS.Timeout; +} + +/** A host with a welcomed socket; there is no link before its hello. */ +interface Link { + id: string; + socket: MaintenanceSocket; + hello: MaintenanceHello; + pending: Map; + /** performance.now() of the host's latest frame. */ + seen: number; + heartbeat: NodeJS.Timeout; +} + +interface HubState { + links: Map; + listeners: Set<() => void>; + frozen: boolean; +} + +declare global { + var __ompMaintenanceHub: HubState | undefined; +} + +// globalThis, like the application hub: survives a `bun --hot` re-evaluation. +function hub(): HubState { + return (globalThis.__ompMaintenanceHub ??= { links: new Map(), listeners: new Set(), frozen: false }); +} + +const refuse = (error: string, status: number) => Response.json({ error }, { status, headers: { "cache-control": "no-store" } }); + +/** + * GET /api/hosts/maintenance. The host's bearer token is checked first, so nothing unauthenticated gets an answer or a + * socket. A plain GET is discovery: 200 `{apiVersion, capabilities}` plus the caller's `server` fields (this module cannot + * know the application's versions), which a daemon needs before it dials, so a server without this API (or an old one + * that accepts any WebSocket URL) never sees a maintenance hello. An upgrade request gets the WebSocket (undefined once + * upgraded); its hello comes as the socket's first frame (maintenanceSocket). + */ +export function handleMaintenanceUpgrade(request: Request, upgrade: (data: MaintenanceSocketData) => boolean, server?: MaintenanceServerInfo): Response | undefined { + const host = hostFromBearer(request); + if (host instanceof Response) return host; + // The daemon retries a 503: it lands on the next server instead. + if (hub().frozen) return refuse("omp-web is shutting down", 503); + if (request.headers.get("upgrade")?.toLowerCase() !== "websocket") { + return Response.json({ ...server, apiVersion: MAINTENANCE_API_VERSION, capabilities: [...MAINTENANCE_OPS] }, { headers: { "cache-control": "no-store" } }); + } + return upgrade({ kind: "maintenance", hostId: host.id }) ? undefined : refuse("The maintenance link is a WebSocket", 426); +} + +/** Bun's `websocket` handlers for maintenance links (server/app.ts); `open` arms the hello deadline. */ +export const maintenanceSocket = { + open(ws: MaintenanceSocket): void { + // An authenticated socket that never says hello must not hang forever. + const timer = setTimeout(() => { + if (!ws.data.hello) ws.close(MAINTENANCE_CLOSE.policy, "No hello"); + }, maintenanceTimings.helloMs); + timer.unref?.(); + }, + message(ws: MaintenanceSocket, data: string | Uint8Array): void { + guarded(ws, () => { + // Nothing is applied once this server stops. + if (hub().frozen) return; + const link = hub().links.get(ws.data.hostId); + if (link?.socket === ws) return onFrame(link, data); + // A socket a newer one replaced, or one that was dropped: what it still sends belongs to no link. + if (ws.data.hello) return; + onHello(ws, data); + }); + }, + close(ws: MaintenanceSocket): void { + guarded(ws, () => { + const link = hub().links.get(ws.data.hostId); + if (link?.socket === ws) endLink(link); + }); + }, +}; + +/** A throw in a websocket handler would reach uncaughtException, which exits the server: it closes that socket instead. */ +function guarded(ws: MaintenanceSocket, handle: () => void): void { + try { + handle(); + } catch (error) { + console.error(`[omp-web] maintenance link of ${ws.data.hostId} failed; closing it: ${error instanceof Error ? error.stack ?? error.message : String(error)}`); + ws.close(); + } +} + +const tooBig = (error: unknown) => (error instanceof MaintenanceError && error.status === 413 ? MAINTENANCE_CLOSE.tooBig : MAINTENANCE_CLOSE.policy); + +/** A socket's first frame must be its hello. */ +function onHello(ws: MaintenanceSocket, data: string | Uint8Array): void { + let frame: Record; + try { + frame = decodeFrame(data); + } catch (error) { + return ws.close(tooBig(error), error instanceof Error ? error.message : "Bad frame"); + } + if (frame.type !== "hello") return ws.close(MAINTENANCE_CLOSE.policy, "The first frame must be a hello"); + // Compared before the rest is read: another API version may shape its hello differently. + if (frame.apiVersion !== MAINTENANCE_API_VERSION) return ws.close(MAINTENANCE_CLOSE.apiMismatch, JSON.stringify({ apiVersion: MAINTENANCE_API_VERSION })); + const hello = parseHello(frame); + if (!hello) return ws.close(MAINTENANCE_CLOSE.policy, "Invalid hello"); + // Revoked or deleted since the upgrade, or the registry cannot say. + let known: boolean; + try { + known = readHosts().some((host) => host.id === ws.data.hostId); + } catch { + return ws.close(MAINTENANCE_CLOSE.tryLater, "The host registry cannot be read"); + } + if (!known) return ws.close(MAINTENANCE_CLOSE.policy, "Unknown host"); + ws.data.hello = true; + const { links } = hub(); + const link: Link = { + id: ws.data.hostId, + socket: ws, + hello, + pending: new Map(), + seen: performance.now(), + heartbeat: setInterval(() => beat(link), maintenanceTimings.pingMs), + }; + link.heartbeat.unref?.(); + // One link per host: the newest wins; only the old socket closes. + const previous = links.get(link.id); + if (previous) { + previous.socket.close(MAINTENANCE_CLOSE.replaced, "replaced by a newer maintenance link"); + retire(previous); + } + links.set(link.id, link); + ws.send(WELCOME); + changed(); +} + +function onFrame(link: Link, data: string | Uint8Array): void { + let frame: Record; + try { + frame = decodeFrame(data); + } catch (error) { + return closeLink(link, tooBig(error), error instanceof Error ? error.message : "Bad frame"); + } + link.seen = performance.now(); + // A pong only proves liveness; types a later revision of API 1 adds are ignored. + if (frame.type !== "res" || typeof frame.id !== "string") return; + const pending = link.pending.get(frame.id); + // An answer to a request that already timed out. + if (!pending) return; + link.pending.delete(frame.id); + clearTimeout(pending.timer); + if (frame.ok === true) pending.resolve(frame.result ?? null); + else { + const { status, code, message } = failure(frame); + pending.reject(new MaintenanceError(status, code, message)); + } +} + +/** Heartbeat: a ping, or the socket dropped once the host has been silent too long (a laptop that slept, a NAT that forgot it). */ +function beat(link: Link): void { + if (performance.now() - link.seen > maintenanceTimings.offlineMs) { + link.socket.terminate(); + endLink(link); + } else link.socket.send(PING); +} + +/** Forget a link. Requests it still carried may or may not have run on the host: they fail, and none is ever repeated. */ +function retire(link: Link): void { + clearInterval(link.heartbeat); + const { links } = hub(); + if (links.get(link.id) === link) links.delete(link.id); + for (const pending of link.pending.values()) { + clearTimeout(pending.timer); + pending.reject(new MaintenanceError(502, "delivery_unknown", "The maintenance link closed before the host answered; the request may have been applied and is not repeated")); + } + link.pending.clear(); +} + +function endLink(link: Link): void { + retire(link); + changed(); +} + +function closeLink(link: Link, code: number, reason: string): void { + link.socket.close(code, reason); + endLink(link); +} + +function changed(): void { + for (const listener of hub().listeners) { + try { + listener(); + } catch (error) { + console.error(`[omp-web] maintenance listener failed: ${error instanceof Error ? error.stack ?? error.message : String(error)}`); + } + } +} + +/** The host's hello while its maintenance link is up, else null. */ +export function getMaintenanceHost(id: string): MaintenanceHello | null { + return hub().links.get(id)?.hello ?? null; +} + +/** + * One request, answered by the host's result or a MaintenanceError. Never repeated: after a timeout or a lost link + * (504 `timeout`, 502 `delivery_unknown`) the host may or may not have run it. Other failures: 503 `frozen` / `no_link`, + * 501 `unsupported_op` (the host did not advertise it), 413 `too_large`, 429 `busy`, and whatever the host reports. + */ +export async function requestMaintenance(id: string, op: MaintenanceOp, args: Record, timeoutMs: number = maintenanceTimings.requestMs): Promise { + if (hub().frozen) throw new MaintenanceError(503, "frozen", "omp-web is shutting down"); + const link = hub().links.get(id); + if (!link) throw new MaintenanceError(503, "no_link", "The host has no maintenance link"); + if (!link.hello.capabilities.includes(op)) throw new MaintenanceError(501, "unsupported_op", `The host does not support ${op}`); + if (link.pending.size >= MAINTENANCE_LIMITS.inFlight) throw new MaintenanceError(429, "busy", "Too many maintenance requests are in flight to this host"); + const requestId = randomUUID(); + const line = encodeFrame({ type: "req", id: requestId, op, args }); + const { promise, resolve, reject } = Promise.withResolvers(); + const timer = setTimeout(() => { + link.pending.delete(requestId); + reject(new MaintenanceError(504, "timeout", `The host did not answer ${op} in time; it may still be running and is not repeated`)); + }, timeoutMs); + link.pending.set(requestId, { resolve, reject, timer }); + // 0: Bun dropped it, so it never left. + if (link.socket.send(line) === 0) { + clearTimeout(timer); + link.pending.delete(requestId); + reject(new MaintenanceError(503, "no_link", "The maintenance link dropped the request; it was not sent")); + } + return promise; +} + +/** Runs on every change to which hosts have a link (a hello, a closed or replaced socket). */ +export function onMaintenanceChange(listener: () => void): () => void { + const { listeners } = hub(); + listeners.add(listener); + return () => { + listeners.delete(listener); + }; +} + +/** Close a host's maintenance link (a revoked or deleted host); its daemon redials and its token check decides. */ +export function closeMaintenanceHost(id: string): void { + const link = hub().links.get(id); + if (link) closeLink(link, MAINTENANCE_CLOSE.policy, "The host's maintenance link was closed by omp-web"); +} + +/** This server is shutting down: from here on no request or upgrade is accepted and every link closes 1001, so its + * daemon redials the next server. Host children are untouched. */ +export function freezeMaintenance(): void { + hub().frozen = true; + for (const link of [...hub().links.values()]) closeLink(link, MAINTENANCE_CLOSE.goingAway, "omp-web is restarting"); +} diff --git a/lib/hosts/maintenance-protocol.test.mjs b/lib/hosts/maintenance-protocol.test.mjs new file mode 100644 index 000000000..abdbc7cfa --- /dev/null +++ b/lib/hosts/maintenance-protocol.test.mjs @@ -0,0 +1,35 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { decodeFrame, encodeFrame, failure, MAINTENANCE_LIMITS, MaintenanceError, parseHello } from "./maintenance-protocol.ts"; + +const hello = (extra = {}) => ({ type: "hello", apiVersion: 1, protocolVersion: 12, ompwebVersion: "0.7.1", ompVersion: null, platform: "linux", capabilities: ["status", "updateSoftware"], ...extra }); + +test("a hello is read whatever the application protocol version, and unknown capabilities are dropped", () => { + assert.deepEqual(parseHello(hello({ protocolVersion: 99, capabilities: ["status", "status", "future", 7] })), hello({ protocolVersion: 99, capabilities: ["status"] })); + assert.equal(parseHello(hello({ ompVersion: "14.0.0" }))?.ompVersion, "14.0.0"); +}); + +test("a hello of another API version or with a wrong field is refused", () => { + for (const bad of [{ apiVersion: 2 }, { protocolVersion: 1.5 }, { protocolVersion: "12" }, { ompwebVersion: 1 }, { ompVersion: 3 }, { platform: "x".repeat(101) }, { capabilities: "status" }]) { + assert.equal(parseHello(hello(bad)), null, JSON.stringify(bad)); + } +}); + +test("decodeFrame refuses what is not a bounded JSON object with a type", () => { + assert.deepEqual(decodeFrame('{"type":"pong"}'), { type: "pong" }); + for (const bad of [new Uint8Array(2), "nope", "[]", "null", '{"id":1}', '{"type":3}']) assert.throws(() => decodeFrame(bad), (e) => e instanceof MaintenanceError && e.status === 400, String(bad)); + assert.throws(() => decodeFrame(JSON.stringify({ type: "x", pad: "x".repeat(MAINTENANCE_LIMITS.frameBytes) })), (e) => e.status === 413 && e.code === "too_large"); +}); + +test("encodeFrame refuses a frame over the limit", () => { + assert.throws(() => encodeFrame({ type: "res", result: "x".repeat(MAINTENANCE_LIMITS.frameBytes) }), (e) => e.status === 413); +}); + +test("failure keeps a native status and code and turns anything else into a 500", () => { + assert.deepEqual(failure(new MaintenanceError(409, "update_busy", "busy")), { status: 409, code: "update_busy", message: "busy" }); + assert.deepEqual(failure({ status: 404, code: "session_not_running", message: "gone" }), { status: 404, code: "session_not_running", message: "gone" }); + assert.deepEqual(failure(Object.assign(new Error("busy"), { code: "host_busy", httpStatus: 409 })), { status: 409, code: "host_busy", message: "busy" }); + assert.deepEqual(failure(new Error("boom")), { status: 500, code: "internal", message: "boom" }); + assert.deepEqual(failure(Object.assign(new Error("x"), { code: "ENOENT", status: 200 })), { status: 500, code: "internal", message: "x" }); + assert.equal(failure({ status: 418, message: "tea" }).code, "host_error"); +}); diff --git a/lib/hosts/maintenance-protocol.ts b/lib/hosts/maintenance-protocol.ts new file mode 100644 index 000000000..80aa9dae4 --- /dev/null +++ b/lib/hosts/maintenance-protocol.ts @@ -0,0 +1,89 @@ +// Maintenance API v1: a small request/reply WebSocket between a host daemon and the server, versioned apart from the +// application link (link-protocol.ts) so an upgrade still reaches a host whose application protocol differs. +// Self-contained: no application-link imports. Everything the peer sends is untrusted input. +import { isRecord } from "../type-guards"; + +export const MAINTENANCE_API_VERSION = 1; +export const MAINTENANCE_PATH = "/api/hosts/maintenance"; +export const MAINTENANCE_OPS = ["status", "listSessions", "checkUpdates", "updateSoftware"] as const; +export type MaintenanceOp = (typeof MAINTENANCE_OPS)[number]; + +/** The host's first frame. `protocolVersion` (the application link's) is advisory: nothing here depends on it. */ +export type MaintenanceHello = { + type: "hello"; + apiVersion: number; + protocolVersion: number; + ompwebVersion: string; + ompVersion: string | null; + platform: string; + capabilities: MaintenanceOp[]; +}; + +/** `apiMismatch`: the hello's apiVersion is not the server's; its reason is JSON `{"apiVersion": }`. */ +export const MAINTENANCE_CLOSE = { goingAway: 1001, policy: 1008, tooBig: 1009, tryLater: 1013, replaced: 4409, apiMismatch: 4426 } as const; + +/** `frameBytes`: one frame, so a request's args or a result (session inventory metadata, never file contents). + * `inFlight`: requests open at once on one link. */ +export const MAINTENANCE_LIMITS = { frameBytes: 1024 * 1024, inFlight: 8 }; + +/** Mutable so tests shorten them. */ +export const maintenanceTimings = { pingMs: 15_000, offlineMs: 45_000, helloMs: 10_000, requestMs: 30_000, retryMs: 1_000, retryMaxMs: 15_000 }; + +/** A failure the peer reports, or this side raises. Statuses are HTTP's; codes are lowercase snake_case. */ +export class MaintenanceError extends Error { + readonly name = "MaintenanceError"; + readonly status: number; + readonly code: string; + + constructor(status: number, code: string, message: string) { + super(message); + this.status = status; + this.code = code; + } +} + +/** One frame as a text line; a MaintenanceError (413) when it is over the frame limit. */ +export function encodeFrame(frame: object): string { + const line = JSON.stringify(frame); + if (Buffer.byteLength(line) > MAINTENANCE_LIMITS.frameBytes) throw new MaintenanceError(413, "too_large", `A maintenance frame is limited to ${MAINTENANCE_LIMITS.frameBytes} bytes`); + return line; +} + +export const PING = encodeFrame({ type: "ping" }); +export const PONG = encodeFrame({ type: "pong" }); +export const WELCOME = encodeFrame({ type: "welcome", apiVersion: MAINTENANCE_API_VERSION }); + +/** A received frame's JSON object (which has a string `type`); anything else is a protocol violation: a MaintenanceError. */ +export function decodeFrame(data: unknown): Record { + if (typeof data !== "string") throw new MaintenanceError(400, "bad_frame", "Maintenance frames are text"); + if (Buffer.byteLength(data) > MAINTENANCE_LIMITS.frameBytes) throw new MaintenanceError(413, "too_large", `A maintenance frame is limited to ${MAINTENANCE_LIMITS.frameBytes} bytes`); + let frame: unknown; + try { + frame = JSON.parse(data); + } catch { + throw new MaintenanceError(400, "bad_frame", "A maintenance frame is not JSON"); + } + if (!isRecord(frame) || typeof frame.type !== "string") throw new MaintenanceError(400, "bad_frame", "A maintenance frame is an object with a type"); + return frame; +} + +const short = (value: unknown): value is string => typeof value === "string" && value.length <= 100; + +/** A hello's fields once its apiVersion is known to be this API's, else null. Capabilities this side does not know are + * dropped, so a later revision of API 1 may add operations without breaking the handshake. */ +export function parseHello(frame: Record): MaintenanceHello | null { + const { apiVersion, protocolVersion, ompwebVersion, ompVersion, platform, capabilities } = frame; + if (apiVersion !== MAINTENANCE_API_VERSION || typeof protocolVersion !== "number" || !Number.isInteger(protocolVersion)) return null; + if (!short(ompwebVersion) || !short(platform) || !(ompVersion === null || short(ompVersion)) || !Array.isArray(capabilities)) return null; + return { type: "hello", apiVersion: MAINTENANCE_API_VERSION, protocolVersion, ompwebVersion, ompVersion, platform, capabilities: MAINTENANCE_OPS.filter((op) => capabilities.includes(op)) }; +} + +/** The wire form of a failure, read from an Error or a `res` frame alike: a status in 400..599 (`status`, or the + * `httpStatus` of a HostUpdateError) and a lowercase code are kept; anything else is a 500 `internal`. */ +export function failure(error: unknown): { status: number; code: string; message: string } { + const e: Record = isRecord(error) ? error : {}; + const raw = e.status ?? e.httpStatus; + const status = typeof raw === "number" && Number.isInteger(raw) && raw >= 400 && raw <= 599 ? raw : 500; + const code = typeof e.code === "string" && /^[a-z0-9_]{1,64}$/.test(e.code) ? e.code : status === 500 ? "internal" : "host_error"; + return { status, code, message: (typeof e.message === "string" ? e.message : "The host reported an error").slice(0, 500) }; +} diff --git a/lib/hosts/maintenance-transport.test.mjs b/lib/hosts/maintenance-transport.test.mjs new file mode 100644 index 000000000..bf51d6517 --- /dev/null +++ b/lib/hosts/maintenance-transport.test.mjs @@ -0,0 +1,276 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +// The hub and a real DaemonMaintenance over loopback, served through Bun.serve the way server/app.ts does. +const dir = mkdtempSync(join(tmpdir(), "omp-maintenance-transport-")); +const previousAgentDir = process.env.PI_CODING_AGENT_DIR; +process.env.PI_CODING_AGENT_DIR = dir; +const hub = await import("./maintenance-hub.ts"); +const { DaemonMaintenance } = await import("./daemon-maintenance.ts"); +const { MAINTENANCE_CLOSE, MAINTENANCE_LIMITS, MAINTENANCE_OPS, maintenanceTimings } = await import("./maintenance-protocol.ts"); +const { createHost, revokeHost } = await import("./registry.ts"); +const timings = { ...maintenanceTimings }; +Object.assign(maintenanceTimings, { retryMs: 20, retryMaxMs: 60, helloMs: 1000 }); + +const seen = { discoveries: 0, upgrades: 0 }; +const server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + fetch(request, bun) { + if (new URL(request.url).pathname !== "/api/hosts/maintenance") return new Response("app", { headers: { "content-type": "text/html" } }); + seen[request.headers.get("upgrade") === "websocket" ? "upgrades" : "discoveries"] += 1; + return hub.handleMaintenanceUpgrade(request, (data) => bun.upgrade(request, { data }), { protocolVersion: 12, ompwebVersion: "0.7.1" }); + }, + websocket: hub.maintenanceSocket, +}); + +test.after(async () => { + await server.stop(true); + Object.assign(maintenanceTimings, timings); + if (previousAgentDir === undefined) delete process.env.PI_CODING_AGENT_DIR; + else process.env.PI_CODING_AGENT_DIR = previousAgentDir; + rmSync(dir, { recursive: true, force: true }); +}); + +const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms)); +async function until(predicate, what = "the condition") { + const end = Date.now() + 5000; + while (!predicate()) { + if (Date.now() > end) throw new Error(`timed out waiting for ${what}`); + await sleep(10); + } +} +/** The error a request fails with. */ +const failed = (promise) => promise.then(() => assert.fail("expected a rejection"), (error) => error); +const outcome = (promise) => promise.then((value) => ({ value }), (error) => ({ error })); + +const host = () => createHost("h"); +const helloFor = (extra) => () => ({ type: "hello", apiVersion: 1, protocolVersion: 12, ompwebVersion: "0.7.1", ompVersion: "14.0.0", platform: "linux", capabilities: [...MAINTENANCE_OPS], ...extra }); + +/** A daemon end dialing `url`; `handle` answers its requests. */ +function daemon(t, { token, url = `http://127.0.0.1:${server.port}`, handle = async () => ({}), hello } = {}) { + const log = []; + const states = []; + const maintenance = new DaemonMaintenance({ url, token }, { hello: helloFor(hello), handleRequest: handle, log: (text) => log.push(text), onState: (up) => states.push(up) }); + t.after(() => maintenance.stop()); + maintenance.start(); + return { maintenance, log, states }; +} +const linkedHost = (t, options) => { + const { host: record, token } = host(); + return { id: record.id, token, ...daemon(t, { token, ...options }) }; +}; + +/** A server that is not ours to control: `discovery()` answers the GET, `onHello(ws)` the socket's hello. */ +function fake(t, { discovery = () => Response.json({ apiVersion: 1 }), onHello = (ws) => ws.send(JSON.stringify({ type: "welcome", apiVersion: 1 })) } = {}) { + const state = { probes: 0, connections: 0, frames: [], socket: null, say: (frame) => state.socket.send(JSON.stringify(frame)) }; + const raw = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + fetch(request, bun) { + if (request.headers.get("upgrade") !== "websocket") { + state.probes += 1; + return discovery(request); + } + return bun.upgrade(request) ? undefined : new Response("no", { status: 426 }); + }, + websocket: { + open(ws) { state.connections += 1; state.socket = ws; }, + message(ws, data) { + const frame = JSON.parse(data); + state.frames.push(frame); + if (frame.type === "hello") onHello(ws); + }, + close() {}, + }, + }); + t.after(() => raw.stop(true)); + state.url = `http://127.0.0.1:${raw.port}`; + return state; +} + +test("answers status, list, check and update requests whatever the application protocol", async (t) => { + const calls = []; + const { id } = linkedHost(t, { handle: async (op, args) => { calls.push([op, args]); return { op }; }, hello: { protocolVersion: 99, apiVersion: 7 } }); + await until(() => hub.getMaintenanceHost(id), "the link"); + assert.equal(hub.getMaintenanceHost(id).protocolVersion, 99); + assert.equal(hub.getMaintenanceHost(id).apiVersion, 1); + assert.ok(seen.discoveries > 0); + for (const op of MAINTENANCE_OPS) assert.deepEqual(await hub.requestMaintenance(id, op, { n: 1 }), { op }); + assert.deepEqual(calls, MAINTENANCE_OPS.map((op) => [op, { n: 1 }])); + // A handler that returns nothing answers null. + const silent = linkedHost(t, { handle: async () => undefined }); + await until(() => hub.getMaintenanceHost(silent.id), "the link"); + assert.equal(await hub.requestMaintenance(silent.id, "status", {}), null); +}); + +test("keeps the native status and code of a host error and stays usable", async (t) => { + const errors = [ + Object.assign(new Error("A session is running"), { code: "host_busy", httpStatus: 409 }), + Object.assign(new Error("Gone"), { code: "session_not_found", status: 404 }), + new Error("boom"), + ]; + let next = 0; + const { id } = linkedHost(t, { handle: async (op) => { if (op === "status") return "x".repeat(MAINTENANCE_LIMITS.frameBytes); throw errors[next++]; } }); + await until(() => hub.getMaintenanceHost(id), "the link"); + const shape = (error) => [error.status, error.code, error.message]; + assert.deepEqual(shape(await failed(hub.requestMaintenance(id, "updateSoftware", {}))), [409, "host_busy", "A session is running"]); + assert.deepEqual(shape(await failed(hub.requestMaintenance(id, "checkUpdates", {}))), [404, "session_not_found", "Gone"]); + assert.deepEqual(shape(await failed(hub.requestMaintenance(id, "listSessions", {}))), [500, "internal", "boom"]); + assert.deepEqual((await failed(hub.requestMaintenance(id, "status", {}))).code, "too_large"); + assert.ok(hub.getMaintenanceHost(id)); +}); + +test("answers an unknown or unadvertised op, a bad args envelope and a duplicate id without dropping the link", async (t) => { + const release = Promise.withResolvers(); + const state = fake(t); + const { states } = daemon(t, { token: "t", url: state.url, hello: { capabilities: ["status"] }, handle: async () => { await release.promise; return "ok"; } }); + await until(() => states[0] === true, "the link"); + const replies = () => state.frames.filter((frame) => frame.type === "res"); + const replyTo = (id) => replies().find((frame) => frame.id === id); + state.say({ type: "req", id: "a", op: "nope", args: {} }); + state.say({ type: "req", id: "b", op: "updateSoftware", args: {} }); + state.say({ type: "req", id: "c", op: "status", args: [] }); + state.say({ type: "req", id: "d", op: "status" }); + await until(() => ["a", "b", "c", "d"].every((id) => replyTo(id)), "the replies"); + assert.deepEqual([replyTo("a"), replyTo("b")].map((r) => [r.ok, r.status, r.code]), [[false, 501, "unsupported_op"], [false, 501, "unsupported_op"]]); + assert.deepEqual([replyTo("c"), replyTo("d")].map((r) => [r.ok, r.status, r.code]), [[false, 400, "bad_request"], [false, 400, "bad_request"]]); + state.say({ type: "req", id: "e", op: "status", args: {} }); + state.say({ type: "req", id: "e", op: "status", args: {} }); + await until(() => replyTo("e"), "the duplicate's reply"); + assert.deepEqual([replyTo("e").status, replyTo("e").code], [409, "duplicate_request"]); + release.resolve(); + await until(() => replies().some((frame) => frame.id === "e" && frame.ok), "the first one's result"); + assert.deepEqual(states, [true]); + assert.equal(state.connections, 1); +}); + +test("drops a link whose server sends a request before the welcome", async (t) => { + const state = fake(t, { onHello: (ws) => ws.send(JSON.stringify({ type: "req", id: "early", op: "status", args: {} })) }); + const { states } = daemon(t, { token: "t", url: state.url }); + await until(() => state.connections >= 2, "a redial"); + assert.deepEqual(states, []); + assert.equal(state.frames.some((frame) => frame.type === "res"), false); +}); + +test("never opens a socket to a server that is not maintenance API 1, and keeps retrying", async (t) => { + const answers = { + "no endpoint": () => new Response("Not Found", { status: 404 }), + "a page": () => new Response("app", { headers: { "content-type": "text/html" } }), + "an empty 426": () => new Response(null, { status: 426 }), + "another API": () => Response.json({ apiVersion: 2 }), + "a string version": () => Response.json({ apiVersion: "1" }), + "a redirect": (request) => Response.redirect(request.url, 302), + }; + for (const [name, discovery] of Object.entries(answers)) { + const state = fake(t, { discovery }); + const { maintenance, log, states } = daemon(t, { token: "t", url: state.url }); + await until(() => state.probes >= 3, `${name} to be probed again`); + assert.equal(state.connections, 0, name); + assert.deepEqual(states, [], name); + assert.ok(log.length > 0, name); + await maintenance.stop(); + } +}); + +test("retries a refused token and a server error without ending, and links once rekeyed", async (t) => { + const { host: record, token } = host(); + const { maintenance, log, states } = daemon(t, { token: "wrong" }); + const before = seen.discoveries; + await until(() => seen.discoveries >= before + 3, "retries"); + assert.equal(hub.getMaintenanceHost(record.id), null); + assert.ok(log.some((line) => line.includes("refused this host's token")), log.join("\n")); + maintenance.rekey(token); + await until(() => hub.getMaintenanceHost(record.id) && states.includes(true), "both ends to accept the link after rekey"); + assert.deepEqual(states, [true]); +}); + +test("a hello the server refuses as another API is retried, never fatal", async (t) => { + const state = fake(t, { onHello: (ws) => ws.close(MAINTENANCE_CLOSE.apiMismatch, JSON.stringify({ apiVersion: 2 })) }); + const { log, states } = daemon(t, { token: "t", url: state.url }); + await until(() => state.connections >= 3, "redials"); + assert.deepEqual(states, []); + assert.ok(log.some((line) => line.includes("apiVersion")), log.join("\n")); + const welcome2 = fake(t, { onHello: (ws) => ws.send(JSON.stringify({ type: "welcome", apiVersion: 2 })) }); + const second = daemon(t, { token: "t", url: welcome2.url }); + await until(() => welcome2.connections >= 2, "redials"); + assert.deepEqual(second.states, []); +}); + +test("gives up on a server that goes silent and dials again", async (t) => { + maintenanceTimings.offlineMs = 80; + t.after(() => { maintenanceTimings.offlineMs = timings.offlineMs; }); + const state = fake(t); + const { states } = daemon(t, { token: "t", url: state.url }); + await until(() => states.length >= 2, "the watchdog"); + assert.deepEqual(states.slice(0, 2), [true, false]); + await until(() => state.connections >= 2, "the redial"); +}); + +test("does not repeat a request that was in flight when the link dropped", async (t) => { + const gate = Promise.withResolvers(); + let calls = 0; + const { id, states } = linkedHost(t, { handle: async () => { calls += 1; if (calls === 1) await gate.promise; return calls; } }); + await until(() => hub.getMaintenanceHost(id), "the link"); + const lost = outcome(hub.requestMaintenance(id, "updateSoftware", {})); + await until(() => calls === 1, "the handler to start"); + hub.closeMaintenanceHost(id); + assert.equal((await lost).error.code, "delivery_unknown"); + await until(() => states.join() === "true,false,true", "the redial"); + await until(() => hub.getMaintenanceHost(id), "the new link"); + await sleep(50); + assert.equal(calls, 1); + // The old connection's late result goes nowhere; the new link works. + gate.resolve(); + await sleep(30); + assert.equal(await hub.requestMaintenance(id, "status", {}), 2); + assert.equal(calls, 2); +}); + +test("stop closes the link, drops what is running and never dials again", async (t) => { + const gate = Promise.withResolvers(); + const { id, maintenance, states } = linkedHost(t, { handle: async () => { await gate.promise; return "late"; } }); + await until(() => hub.getMaintenanceHost(id), "the link"); + const pending = outcome(hub.requestMaintenance(id, "status", {})); + await sleep(30); + await maintenance.stop(); + assert.deepEqual(states, [true, false]); + assert.equal((await pending).error.code, "delivery_unknown"); + await until(() => hub.getMaintenanceHost(id) === null, "the server to see the close"); + const dials = seen.discoveries; + gate.resolve(); + await sleep(150); + assert.equal(seen.discoveries, dials); + assert.deepEqual(states, [true, false]); +}); + +test("a revoked host's daemon keeps retrying and is not let back in", async (t) => { + const { id, states } = linkedHost(t); + await until(() => hub.getMaintenanceHost(id), "the link"); + revokeHost(id); + hub.closeMaintenanceHost(id); + await until(() => states.join() === "true,false", "the drop"); + const dials = seen.discoveries; + await until(() => seen.discoveries >= dials + 3, "retries"); + assert.equal(hub.getMaintenanceHost(id), null); +}); + +test("freezing the server closes links and refuses dials until it restarts, without ending the daemon", async (t) => { + t.after(() => { globalThis.__ompMaintenanceHub = undefined; }); + const { id, states } = linkedHost(t); + await until(() => hub.getMaintenanceHost(id), "the link"); + hub.freezeMaintenance(); + await until(() => states.join() === "true,false", "the close"); + const upgrades = seen.upgrades; + const dials = seen.discoveries; + await until(() => seen.discoveries >= dials + 3, "refused dials"); + assert.equal(seen.upgrades, upgrades); + assert.equal(hub.getMaintenanceHost(id), null); + // The next server process starts with fresh state. + globalThis.__ompMaintenanceHub = undefined; + await until(() => hub.getMaintenanceHost(id), "the link on the restarted server"); + assert.deepEqual(states, [true, false, true]); +}); diff --git a/lib/hosts/update-worker.test.mjs b/lib/hosts/update-worker.test.mjs index abb855bdf..cee87cac1 100644 --- a/lib/hosts/update-worker.test.mjs +++ b/lib/hosts/update-worker.test.mjs @@ -158,6 +158,20 @@ test("owned descriptor is required and a symlink is refused before execution", { assert.equal(alive(f.daemon.pid), true); }); +test("repository metadata remains required for installation and malformed metadata cannot authorize a restart", async () => { + for (const restartOnly of [false, true]) { + const f = await fixture(`repository-boundary-${restartOnly}`, { restartOnly }); + const descriptor = read(f.descriptorPath); + if (restartOnly) descriptor.releaseRepo = "not/a/repository"; + else delete descriptor.releaseRepo; + writeFileSync(f.descriptorPath, JSON.stringify(descriptor), { mode: 0o600 }); + assert.equal(await worker.runHostUpdateWorker(f.descriptorPath), 1); + assert.match(status().operation.error, /descriptor/); + assert.equal(alive(f.daemon.pid), true); + assert.equal(versionOf(f.install), restartOnly ? "2.0.0" : "1.0.0"); + } +}); + test("binary archives reject path traversal, links and excessive sizes; a release with its omp runner fits", () => { const tar = new Uint8Array(2048); function entry(name, type = "0", size = "00000000000") { @@ -562,3 +576,14 @@ test("pruning leaves everything alone while an update is running and keeps the f assert.equal(existsSync(join(parent, ".ompweb-failed-running-a")), false); assert.equal(existsSync(join(stateRoot, "keep-b", "runtime")), false); }); + +test("a legacy restart-only descriptor needs no release repository and still verifies the new daemon", async () => { + const f = await fixture("legacy-restart-only", { restartOnly: true }); + const descriptor = read(f.descriptorPath); + delete descriptor.releaseRepo; + writeFileSync(f.descriptorPath, JSON.stringify(descriptor), { mode: 0o600 }); + assert.equal(await worker.runHostUpdateWorker(f.descriptorPath), 0, outcome(f)); + assert.equal(read(join(f.attempt, "health.json")).version, "2.0.0"); + assert.equal(existsSync(f.backup), false); + assert.equal(alive(f.daemon.pid), false); +}); diff --git a/lib/hosts/update-worker.ts b/lib/hosts/update-worker.ts index 792654027..ce32204f9 100644 --- a/lib/hosts/update-worker.ts +++ b/lib/hosts/update-worker.ts @@ -43,8 +43,8 @@ export interface HostUpdateDescriptor { managerPath?: string; restartOnly?: boolean; systemd: { unit: string; user: boolean } | null; - /** `owner/name` whose releases the binary comes from (lib/release-repo.ts; this file stays self-contained). */ - releaseRepo: string; + /** Binary release source; absent only on a legacy restart-only descriptor, which downloads nothing. */ + releaseRepo?: string; } export function hostUpdateRoot(): string { @@ -576,7 +576,9 @@ function validateDescriptor(path: string): HostUpdateDescriptor { const root = ensureHostUpdateRoot(); assertOwnedPath(path); const d = JSON.parse(readFileSync(path, "utf8")) as HostUpdateDescriptor; - if (!/^[a-zA-Z0-9-]{1,80}$/.test(d.id) || resolve(path) !== join(root, d.id, "descriptor.json") || !HOST_VERSION_PATTERN.test(d.version) || !HOST_VERSION_PATTERN.test(d.fromVersion) || d.target !== "ompweb" || !["npm", "bun", "binary"].includes(d.installMethod) || !isAbsolute(d.installDir) || !isAbsolute(d.runtime) || !isAbsolute(d.cwd) || !Number.isSafeInteger(d.daemonPid) || d.daemonPid <= 1 || !Array.isArray(d.hostArgs) || d.hostArgs.some((arg) => arg !== "--allow-http") || typeof d.releaseRepo !== "string" || !/^[\w.-]+\/[\w.-]+$/.test(d.releaseRepo)) throw new Error("Invalid host update descriptor"); + if (!/^[a-zA-Z0-9-]{1,80}$/.test(d.id) || resolve(path) !== join(root, d.id, "descriptor.json") || !HOST_VERSION_PATTERN.test(d.version) || !HOST_VERSION_PATTERN.test(d.fromVersion) || d.target !== "ompweb" || !["npm", "bun", "binary"].includes(d.installMethod) || !isAbsolute(d.installDir) || !isAbsolute(d.runtime) || !isAbsolute(d.cwd) || !Number.isSafeInteger(d.daemonPid) || d.daemonPid <= 1 || !Array.isArray(d.hostArgs) || d.hostArgs.some((arg) => arg !== "--allow-http")) throw new Error("Invalid host update descriptor"); + // A v10 daemon copies this new worker after the shared web installation has already been replaced. + if (d.releaseRepo === undefined ? d.restartOnly !== true : typeof d.releaseRepo !== "string" || !/^[\w.-]+\/[\w.-]+$/.test(d.releaseRepo)) throw new Error("Invalid host update descriptor"); if (d.restartOnly !== undefined && typeof d.restartOnly !== "boolean") throw new Error("Invalid host restart-only descriptor"); if (!d.systemd && (typeof d.daemonIdentity !== "string" || !d.daemonIdentity || d.daemonIdentity.length > 200)) throw new Error("Invalid host update descriptor: the daemon's process identity is missing"); assertOwnedPath(dirname(path), true); diff --git a/lib/hosts/upgrade-shim.test.mjs b/lib/hosts/upgrade-shim.test.mjs new file mode 100644 index 000000000..0c2891142 --- /dev/null +++ b/lib/hosts/upgrade-shim.test.mjs @@ -0,0 +1,109 @@ +import assert from "node:assert/strict"; +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test, { after } from "node:test"; + +const dir = mkdtempSync(join(tmpdir(), "ompweb-upgrade-shim-")); +const previous = process.env.PI_CODING_AGENT_DIR; +process.env.PI_CODING_AGENT_DIR = dir; +const hub = await import("./hub.ts"); +const { linkHost, upgradeSocket, until } = await import("./link-driver.test-helper.mjs"); +const { LINK_PROTOCOL_VERSION } = await import("./link-protocol.ts"); +const { getRpcSession } = await import("../rpc-manager.ts"); +after(() => { + hub.flushLinkSamples(); + if (previous === undefined) delete process.env.PI_CODING_AGENT_DIR; + else process.env.PI_CODING_AGENT_DIR = previous; + rmSync(dir, { recursive: true, force: true }); +}); + +const sid = "11111111-2222-4333-8444-5555555555bb"; +const channels = [{ ch: "old-channel", sessionId: sid, cwd: "/w", pending: ['{"type":"agent_start"}'] }]; +const operation = { id: "update-attempt", target: "ompweb", version: "0.7.1", state: "running" }; +const info = { + ompweb: { currentVersion: "0.6.0", availableVersion: "0.7.1", updateAvailable: true, canUpdate: true, blockedReason: null, updateCommand: "npm install -g @kahme247/ompweb", installMethod: "npm", protocolVersion: 12 }, + omp: { currentVersion: "18.20.0" }, operation: null, +}; + +for (const v of [10, 11]) test(`v${v} is welcomed as itself solely for its unchanged updater`, async (t) => { + const hellos = [], sessionsChanged = [], other = [], requests = []; + t.after(hub.onHostHello((id) => hellos.push(id))); + t.after(hub.onHostSessionsChanged((id) => sessionsChanged.push(id))); + const link = await linkHost(`Legacy v${v}`, { channels, hello: { v, ompwebVersion: "0.6.0" } }); + t.after(() => link.drop()); + assert.equal(link.welcome.v, v, "the old daemon rejects a welcome above its own protocol"); + const row = () => hub.listHosts().find((host) => host.id === link.id); + assert.equal(row().status, "needs-upgrade"); + assert.equal(row().upgradeOnly, true); + assert.deepEqual(row().workspaceOps, []); + assert.equal(getRpcSession(sid), undefined, "the shim must not adopt legacy sessions"); + link.serve({ + checkUpdates: (args) => { requests.push(["checkUpdates", args]); return info; }, + updateSoftware: (args) => { requests.push(["updateSoftware", args]); return { accepted: true, operation }; }, + }, (message) => other.push(message)); + assert.equal((await hub.getHostSoftwareUpdates(link.id, true)).ompweb.canUpdate, true); + assert.deepEqual(await hub.updateHostSoftware(link.id, "ompweb", "0.7.1"), { accepted: true, operation }); + assert.ok(requests.some(([op]) => op === "updateSoftware")); + for (const [, args] of requests) assert.deepEqual(args.compatibility, { min: 12, max: 12 }); + await assert.rejects(hub.getHost(link.id).sessionFileExists("/w/session.jsonl"), { code: "host_upgrade_required" }); + await link.post( + { type: "channels_changed", channels }, + { type: "stdout", ch: "old-channel", line: '{"type":"agent_start"}' }, + { type: "sessions_changed" }, + ); + // Give deferred imports and adoption attempts a quiet window in which to expose an accidental dispatch. + await new Promise(resolve => setTimeout(resolve, 50)); + assert.equal(getRpcSession(sid), undefined, "legacy channel announcements must not adopt sessions either"); + assert.deepEqual(hellos, [], "legacy hello must not trigger application startup hooks"); + assert.deepEqual(sessionsChanged, [], "legacy events must not enter application discovery"); + assert.deepEqual(other, [], "no open, kill, config sync or application command may reach the old daemon"); + link.drop(); + await until(() => !row().upgradeOnly, "the closed shim stops advertising update capability"); +}); + +test("a legacy shim cannot replace a healthy application connection", async (t) => { + const link = await linkHost("Current host"); + t.after(() => link.drop()); + const { socket } = upgradeSocket(link.token); + socket.frame(JSON.stringify({ type: "hello", v: 10, ompVersion: null, platform: "linux", home: "/tmp", autoApproveExtension: false, ack: 0, channels: [] })); + assert.equal(socket.closed?.code, 4426); + assert.equal(link.socket.closed, null); + const row = hub.listHosts().find((host) => host.id === link.id); + assert.equal(row.status, "online"); + assert.equal(row.hostVersion, LINK_PROTOCOL_VERSION); + assert.ok(!row.upgradeOnly); +}); + +test("a shim reconnect never replays an update whose delivery is uncertain", async (t) => { + const link = await linkHost("Interrupted update", { hello: { v: 10 } }); + t.after(() => link.drop()); + const updating = hub.updateHostSoftware(link.id, "ompweb", "0.7.1"); + const rejected = assert.rejects(updating, { code: "transport_lost" }); + const check = await link.next(); + assert.equal(check.op, "checkUpdates"); + await link.answer(check, info); + const update = await link.next(); + assert.equal(update.op, "updateSoftware"); + await link.dial({ hello: { v: 10 } }); + await rejected; + assert.ok(link.socket.frames.every(line => JSON.parse(line).type !== "req"), "the old update must not run again"); +}); + +test("unacknowledged application input never crosses into a legacy shim", async (t) => { + const link = await linkHost("Protocol downgrade"); + t.after(() => link.drop()); + const spawning = hub.getHost(link.id).spawnRpc({ cwd: "/w", recordedCwd: null, sessionFile: "", advisor: false }, null); + const open = await link.next(); + assert.equal(open.op, "open"); + await link.answer(open, { ch: "full-channel", joined: false, pending: [], streaming: null, advisorSpawned: false, cwd: "/w", fellBack: false, launchAdvisor: false, autoApproveExtension: false }); + const { transport } = await spawning; + transport.write('{"t":"cmd","id":1,"op":"prompt","args":{"message":"must not replay"}}\\n', () => {}); + const stdin = await link.next(); + assert.equal(stdin.type, "stdin"); + link.drop(); + await new Promise(resolve => setImmediate(resolve)); + await link.dial({ hello: { v: 11 } }); + assert.equal(link.welcome.v, 11); + assert.ok(link.socket.frames.every(line => ["welcome", "ack", "ping", "pong"].includes(JSON.parse(line).type))); +}); diff --git a/lib/hosts/v10-upgrade.integration.test.mjs b/lib/hosts/v10-upgrade.integration.test.mjs new file mode 100644 index 000000000..b8ec02f4a --- /dev/null +++ b/lib/hosts/v10-upgrade.integration.test.mjs @@ -0,0 +1,367 @@ +// Real pre-cutover daemons (link protocol 10 and 11) upgrading to this checkout through this checkout's hub. +// +// Nothing of the old side is imitated. The daemon is the published source, extracted from git at a pinned commit into +// a disposable npm-layout install and started as `node bin/omp-web-host.js`; its welcome check, its `checkUpdates` / +// `updateSoftware` handlers and its updater engine (candidate protocol check, install, swap, restart, health) all run +// unchanged. Only delivery is disposable: a fetch preload answers the npm registry and the raw-GitHub link-protocol.ts +// (the worktree's real file, parsed by the OLD updater's own regex), and a fake `npm` "installs" a copy of this +// checkout as the release. The hub is the real hub behind the real /api/hosts/link route. No user service is touched. +// +// Pinned sources (both reachable from main; the test skips, naming the commit, when a shallow clone lacks it): +// v10 546a945df0d1716a7a135d6e3221ba885b00171e package 0.6.0, link 10. The last commit before 8ec9208d (runner +// channels); published as prerelease 20261113T113829Z-546a945df0. v10 first appeared at a53494c4. Its updater +// takes target "ompweb" or "omp" and has no preflight. +// v11 3a56e79b217a3949f7180069e6769bc3fa031167 package 0.7.0, link 11: the commit tag v0.7.0 points at. +// +// Run: bun test --timeout 240000 lib/hosts/v10-upgrade.integration.test.mjs +// Needs: full git history (`git cat-file -e `), `tar` and `git` on PATH, node >= 24 on PATH, `npm ci` done in this +// checkout (daemon dependencies are symlinked from its node_modules), a POSIX machine (Linux or macOS). +// Disposable stand-ins besides delivery: one POSIX `omp` script (OMP_WEB_OMP_BIN) that answers `--version`, `--check` and +// the session start with the runner's channel hello, so the v11 updater's real `fetch-omp` and `preflight` of the candidate +// pass without the runner binary (no `bun runner/scripts/build-binary.ts` needed) or any network. +// Not covered: compiled-binary installs, systemd/launchd-supervised hosts, Windows, a daemon with live sessions (a legacy +// link carries none; the old busy checks are its own unchanged code). The server side serves only /api/hosts/link, so the +// upgraded daemon's maintenance connection is refused there; its application link and health acknowledgement are real. +// Cleanup: each scenario kills every process whose command line names its directory (daemons, detached workers, the +// stand-in sharing the install) and the file kills the rest; the run fails if any survives. A crash of the test runner +// itself can leave detached daemons running under the system temp directory (v10-upgrade-it-*): kill them by that path. +import assert from "node:assert/strict"; +import test, { after } from "node:test"; +import { execFileSync, spawn, spawnSync } from "node:child_process"; +import { chmodSync, cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, rmSync, statSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { delimiter, dirname, join } from "node:path"; +import { pathToFileURL } from "node:url"; +import { nodeBinary, repo, repoVersion, spawnNodeOutsideService } from "./update-fixture.test-helper.mjs"; + +const LEGACY = { + v10: { name: "v10", link: 10, version: "0.6.0", commit: "546a945df0d1716a7a135d6e3221ba885b00171e" }, + v11: { name: "v11", link: 11, version: "0.7.0", commit: "3a56e79b217a3949f7180069e6769bc3fa031167" }, +}; +/** The application link version this release speaks, and the only target compatibility an old updater may be given. */ +const APPLICATION = 12; + +const base = realpathSync(mkdtempSync(join(tmpdir(), "v10-upgrade-it-"))); +const previousAgentDir = process.env.PI_CODING_AGENT_DIR; +process.env.PI_CODING_AGENT_DIR = join(base, "main-agent"); + +const linkRoute = await import("../../app/api/hosts/link/route.ts"); +const hub = await import("./hub.ts"); +const { createHost } = await import("./registry.ts"); +const { closeNotificationStore } = await import("../notification-store.ts"); +const { CHANNEL_PROTO } = await import("./omp-socket.ts"); + +const wait = (ms) => new Promise((done) => setTimeout(done, ms)); +const read = (path) => JSON.parse(readFileSync(path, "utf8")); +function alive(pid) { + try { process.kill(pid, 0); return true; } catch (error) { return error.code === "EPERM"; } +} + +// Every line the hub sends each host, in order, so the old daemon's whole view of the server can be asserted. +const down = new Map(); +const wrappers = new WeakMap(); +/** The hub keys its state on socket identity: one stable wrapper per socket, recording what it sends. */ +function wrap(ws) { + let wrapper = wrappers.get(ws); + if (wrapper) return wrapper; + const send = (frame, ...rest) => { + if (typeof frame === "string") { + for (const line of frame.split("\n")) if (line) down.set(ws.data.hostId, [...(down.get(ws.data.hostId) ?? []), JSON.parse(line)]); + } + return ws.send(frame, ...rest); + }; + wrapper = new Proxy(ws, { get: (target, name) => name === "send" ? send : typeof target[name] === "function" ? target[name].bind(target) : target[name] }); + wrappers.set(ws, wrapper); + return wrapper; +} +const server = Bun.serve({ + port: 0, + hostname: "127.0.0.1", + routes: { + "/api/hosts/link": { + GET: (req) => linkRoute.GET(req, { upgrade: (data) => server.upgrade(req, { data }) }), + POST: (req) => linkRoute.POST(req), + }, + }, + websocket: { + ...hub.hostLinkSocket, + message: (ws, data) => hub.hostLinkSocket.message(wrap(ws), data), + drain: (ws) => hub.hostLinkSocket.drain(wrap(ws)), + close: (ws) => hub.hostLinkSocket.close(wrap(ws)), + }, +}); + +/** Pids of every process whose command line names `dir`: daemons, workers and stand-ins all run files from inside it. */ +function processesUnder(dir) { + let lines; + if (process.platform === "linux") { + lines = readdirSync("/proc").filter((name) => /^\d+$/.test(name)).flatMap((name) => { + try { return [[Number(name), readFileSync(`/proc/${name}/cmdline`, "utf8").replaceAll("\0", " ")]]; } catch { return []; } + }); + } else { + const result = spawnSync("ps", ["-axo", "pid=,command="], { encoding: "utf8" }); + if (result.error || result.status !== 0 || typeof result.stdout !== "string") throw new Error("Cannot inspect test-owned processes; temporary installs retained", { cause: result.error }); + lines = result.stdout.split("\n").flatMap((line) => { + const match = /^\s*(\d+)\s+(.*)$/.exec(line); + return match ? [[Number(match[1]), match[2]]] : []; + }); + } + return lines.filter(([pid, command]) => pid !== process.pid && command.includes(dir)).map(([pid]) => pid); +} +async function stopAll(dir) { + for (const signal of ["SIGTERM", "SIGKILL"]) { + for (const pid of processesUnder(dir)) { try { process.kill(pid, signal); } catch { /* already gone */ } } + for (const deadline = Date.now() + 10_000; processesUnder(dir).length > 0 && Date.now() < deadline;) await wait(100); + } + assert.deepEqual(processesUnder(dir), [], "every process this test started must exit before its directory is removed"); +} + +after(async () => { + if (process.platform !== "win32") await stopAll(base); + await server.stop(true); + closeNotificationStore(); + hub.flushLinkSamples(); + if (previousAgentDir === undefined) delete process.env.PI_CODING_AGENT_DIR; + else process.env.PI_CODING_AGENT_DIR = previousAgentDir; + for (let tries = 0; ; tries++) { + try { return rmSync(base, { recursive: true, force: true }); } catch (error) { + if (tries === 50 || !["EBUSY", "ENOTEMPTY", "EPERM"].includes(error.code)) throw error; + await wait(100); + } + } +}); + +const haveCommit = (commit) => spawnSync("git", ["-C", repo, "cat-file", "-e", `${commit}^{commit}`]).status === 0; + +const sources = new Map(); +/** `bin`, `lib` and package.json of the pinned commit: everything `omp-web-host.js` loads. */ +function legacySource(legacy) { + if (sources.has(legacy.commit)) return sources.get(legacy.commit); + const dir = join(base, `source-${legacy.name}`); + mkdirSync(dir); + const archive = `${dir}.tar`; + // umask 0022: the updater refuses an install directory that is group-writable. + execFileSync("git", ["-C", repo, "-c", "tar.umask=0022", "archive", "--format=tar", "-o", archive, legacy.commit, "bin", "lib", "package.json"]); + execFileSync("tar", ["-xf", archive, "-C", dir, "--no-same-owner"]); + // The pin must be the daemon this test claims to run. + assert.equal(read(join(dir, "package.json")).version, legacy.version); + assert.match(readFileSync(join(dir, "lib", "hosts", "link-protocol.ts"), "utf8"), new RegExp(`^export const LINK_PROTOCOL_VERSION = ${legacy.link};$`, "m")); + sources.set(legacy.commit, dir); + return dir; +} + +let releaseDir; +/** This checkout as the release the fake npm delivers: what `npm install @kahme247/ompweb@` would unpack. */ +function release() { + if (releaseDir) return releaseDir; + releaseDir = join(base, "release"); + mkdirSync(releaseDir); + cpSync(join(repo, "bin"), join(releaseDir, "bin"), { recursive: true }); + cpSync(join(repo, "lib"), join(releaseDir, "lib"), { recursive: true, filter: (src) => !/\.test(-helper)?\.(mjs|ts|tsx)$/.test(src) }); + cpSync(join(repo, "package.json"), join(releaseDir, "package.json")); + return releaseDir; +} + +const preload = join(base, "release-delivery.mjs"); +// The only network the old updater sees: the npm registry's latest version and the target tag's raw link-protocol.ts. +writeFileSync(preload, `import { appendFileSync, existsSync, readFileSync } from "node:fs"; +const real = globalThis.fetch; +const dir = process.env.UPGRADE_FIXTURE_DIR; +globalThis.fetch = async (input, init) => { + const url = String(input instanceof Request ? input.url : input); + appendFileSync(dir + "/fetch.log", url + "\\n"); + if (url.startsWith("https://registry.npmjs.org/")) return Response.json({ version: process.env.UPGRADE_FIXTURE_VERSION }); + if (/^https:\\/\\/raw\\.githubusercontent\\.com\\/[^/]+\\/[^/]+\\/v[^/]+\\/lib\\/hosts\\/link-protocol\\.ts$/.test(url)) { + const override = dir + "/link-protocol.override"; + return new Response(readFileSync(existsSync(override) ? override : process.env.UPGRADE_FIXTURE_PROTOCOL_SOURCE)); + } + if (/^http:\\/\\/(127\\.0\\.0\\.1|localhost|\\[::1\\])[:/]/.test(url)) return real(input, init); + throw new Error("upgrade fixture: no network access to " + url); +}; +`); + +/** Installs the pinned daemon as a real npm-layout host, logs it into the hub and starts it. Null when no scope is available. */ +async function startLegacy(legacy, layout) { + const work = join(base, `${legacy.name}-${layout}`); + const modules = join(work, "node_modules"); + const install = join(modules, "@kahme247", "ompweb"); + const home = join(work, "home"); + const agent = join(home, ".omp", "agent"); + mkdirSync(join(work, "bin"), { recursive: true }); + mkdirSync(modules, { recursive: true }); + mkdirSync(agent, { recursive: true }); + // The daemon's dependencies, resolved from the node_modules above the install as for any hoisted global install. + for (const name of readdirSync(join(repo, "node_modules"))) { + if (!name.startsWith(".")) symlinkSync(realpathSync(join(repo, "node_modules", name)), join(modules, name), "dir"); + } + cpSync(legacySource(legacy), install, { recursive: true }); + for (let dir = install; dir !== dirname(work); dir = dirname(dir)) chmodSync(dir, 0o755); + const fakeOmp = join(work, "fake-omp"); + // One stand-in for both omp roles an update touches: the version/update probes of the old daemon, and the session omp + // that the candidate's `ompweb preflight` (run by the v11 updater before its switch) must see say hello. + writeFileSync(fakeOmp, `#!/bin/sh +if [ "$1" = "--version" ]; then echo 14.0.0-fake; exit 0; fi +case "$*" in *--check*) echo "Current version: 14.0.0-fake"; exit 0;; esac +case "$*" in *"--headless --cwd "*"--no-session"*) echo '{"t":"hello","proto":${CHANNEL_PROTO}}'; exec cat >/dev/null;; esac +exit 0 +`, { mode: 0o755 }); + writeFileSync(join(work, "idle.js"), "setInterval(() => {}, 1000);\n"); + writeFileSync(join(work, "bin", "npm"), `#!/usr/bin/env node +const fs = require("node:fs"), path = require("node:path"); +const args = process.argv.slice(2); +fs.appendFileSync(${JSON.stringify(join(work, "npm.log"))}, JSON.stringify(args) + "\\n"); +if (!args.includes(${JSON.stringify(`@kahme247/ompweb@${repoVersion}`)})) process.exit(8); +fs.cpSync(${JSON.stringify(release())}, path.join(args[args.indexOf("--prefix") + 1], "node_modules", "@kahme247", "ompweb"), { recursive: true }); +`, { mode: 0o700 }); + + const { host, token } = createHost(`${legacy.name} ${layout}`); + const env = { + ...process.env, HOME: home, USERPROFILE: home, PI_CODING_AGENT_DIR: agent, OMP_WEB_OMP_BIN: fakeOmp, OMP_WEB_PACKAGE_DIR: install, + PATH: join(work, "bin") + delimiter + process.env.PATH, + NODE_OPTIONS: [process.env.NODE_OPTIONS, `--import ${pathToFileURL(preload).href}`].filter(Boolean).join(" "), + UPGRADE_FIXTURE_DIR: work, UPGRADE_FIXTURE_VERSION: repoVersion, UPGRADE_FIXTURE_PROTOCOL_SOURCE: join(repo, "lib", "hosts", "link-protocol.ts"), + }; + delete env.OMP_WEB_DISABLE_AUTOUPDATE; + delete env.OMP_WEB_HOST_DRAINED; + const entry = join(install, "bin", "omp-web-host.js"); + const login = spawnSync(nodeBinary(), [entry, "login", "--url", `http://127.0.0.1:${server.port}`], { env, cwd: work, input: `${token}\n`, encoding: "utf8" }); + assert.equal(login.status, 0, login.stderr); + + const child = await spawnNodeOutsideService(entry, env); + if (!child) return null; + let log = ""; + child.stdout.on("data", (data) => { log += data; }); + child.stderr.on("data", (data) => { log += data; }); + const lines = (name) => existsSync(join(work, name)) ? readFileSync(join(work, name), "utf8").split("\n").filter(Boolean) : []; + return { + legacy, layout, work, install, child, hostId: host.id, stateRoot: join(agent, "omp-web", "host-update"), + log: () => log, + npmCalls: () => lines("npm.log").map((line) => JSON.parse(line)), + fetched: () => lines("fetch.log"), + status: () => read(join(agent, "omp-web", "host-update", "status.json")), + }; +} + +async function waitFor(predicate, what, run, timeoutMs = 120_000) { + for (const deadline = Date.now() + timeoutMs; ; await wait(100)) { + if (predicate()) return; + if (Date.now() > deadline) { + const status = existsSync(join(run.stateRoot, "status.json")) ? JSON.stringify(run.status()) : "none"; + assert.fail(`timed out waiting for ${what}\nupdate status: ${status}\nnpm calls: ${JSON.stringify(run.npmCalls())}\nold daemon log:\n${run.log()}`); + } + } +} + +/** checkUpdates through the hub. The daemon's own startup upload sweep briefly reports "Wait for ..."; any other block is the answer. */ +async function checkUpdates(run) { + for (const deadline = Date.now() + 30_000; ; await wait(500)) { + const info = await hub.getHostSoftwareUpdates(run.hostId, true); + if (!/^Wait for/.test(info.ompweb.blockedReason ?? "") || Date.now() > deadline) return info; + } +} + +async function upgrade(run) { + const { legacy, layout, hostId } = run; + const row = () => hub.listHosts().find((host) => host.id === hostId); + const sent = () => down.get(hostId) ?? []; + + // The real old daemon is welcomed with the version it said (it throws on any other) and linked for upgrading only. + await waitFor(() => row()?.status === "needs-upgrade" && row().upgradeOnly === true && /linked to /.test(run.log()), `the ${legacy.name} daemon to be welcomed upgrade-only`, run); + assert.equal(row().hostVersion, legacy.link); + assert.equal(row().ompwebVersion, legacy.version); + assert.deepEqual([...new Set(sent().filter((message) => message.type === "welcome").map((message) => message.v))], [legacy.link]); + + if (layout === "separate") { + // A candidate on another protocol is refused by what the old updater was told, not by a fixture shortcut. + writeFileSync(join(run.work, "link-protocol.override"), `export const LINK_PROTOCOL_VERSION = ${APPLICATION - 1};\n`); + const wrong = await checkUpdates(run); + assert.equal(wrong.ompweb.protocolVersion, APPLICATION - 1); + assert.equal(wrong.ompweb.canUpdate, false); + rmSync(join(run.work, "link-protocol.override")); + } + + let installBefore; + let web; + if (layout === "shared") { + // The web package shares this installation, and has not replaced it yet: nothing may be installed over it. + web = spawn(nodeBinary(), [join(run.work, "idle.js"), join(run.install, "package.json")], { stdio: "ignore", cwd: run.work }); + const stale = await checkUpdates(run); + assert.equal(stale.ompweb.canUpdate, false); + assert.match(stale.ompweb.blockedReason, /installation is shared/); + await assert.rejects(hub.updateHostSoftware(hostId, "ompweb", repoVersion)); + assert.deepEqual(run.npmCalls(), []); + // `npm i -g` of the web package: the shared installation now holds the candidate while the old daemon keeps running. + for (const name of readdirSync(run.install)) rmSync(join(run.install, name), { recursive: true, force: true }); + cpSync(release(), run.install, { recursive: true }); + chmodSync(run.install, 0o755); + installBefore = { directory: statSync(run.install).ino, manifest: statSync(join(run.install, "package.json")) }; + } + + const info = await checkUpdates(run); + assert.equal(info.ompweb.currentVersion, legacy.version); + assert.equal(info.ompweb.availableVersion, repoVersion); + assert.equal(info.ompweb.updateAvailable, true); + assert.equal(info.ompweb.protocolVersion, APPLICATION); + assert.equal(info.ompweb.canUpdate, true, info.ompweb.blockedReason); + + const accepted = await hub.updateHostSoftware(hostId, "ompweb", repoVersion); + assert.deepEqual({ target: accepted.operation.target, version: accepted.operation.version }, { target: "ompweb", version: repoVersion }); + await waitFor(() => existsSync(join(run.stateRoot, "status.json")) && run.status().operation.state !== "running", "the old updater to finish", run); + const { operation, daemonPid } = run.status(); + assert.equal(operation.state, "succeeded", operation.error); + await waitFor(() => row()?.status === "online" && row().hostVersion === APPLICATION && row().ompwebVersion === repoVersion, "the upgraded daemon to link at the application version", run); + assert.equal(row().upgradeOnly, undefined); + assert.equal(hub.getHostLinkProtocol(hostId), APPLICATION); + await waitFor(() => run.child.exitCode !== null || run.child.signalCode !== null, "the old daemon to stop", run); + const health = read(join(run.stateRoot, operation.id, "health.json")); + assert.equal(health.version, repoVersion); + assert.notEqual(health.pid, daemonPid); + assert.ok(alive(health.pid), "the new daemon keeps running"); + assert.equal(read(join(run.install, "package.json")).version, repoVersion); + + const backups = readdirSync(dirname(run.install)).filter((name) => name.startsWith(".ompweb-backup-")); + if (layout === "separate") { + assert.equal(run.npmCalls().length, 1); + assert.ok(run.npmCalls()[0].includes(`@kahme247/ompweb@${repoVersion}`)); + assert.deepEqual(backups, [`.ompweb-backup-${operation.id}`]); + assert.equal(read(join(dirname(run.install), backups[0], "package.json")).version, legacy.version); + } else { + // A shared install is restarted, never reinstalled: no package manager, no swap, no backup, and the other process lives. + assert.deepEqual(run.npmCalls(), []); + assert.deepEqual(backups, []); + assert.equal(read(join(run.stateRoot, operation.id, "descriptor.json")).restartOnly, true); + assert.equal(statSync(run.install).ino, installBefore.directory); + const manifest = statSync(join(run.install, "package.json")); + assert.deepEqual([manifest.ino, manifest.mtimeMs], [installBefore.manifest.ino, installBefore.manifest.mtimeMs]); + assert.ok(alive(web.pid), "the process sharing the installation is untouched"); + } + const candidateSource = new RegExp(`^https://raw\\.githubusercontent\\.com/[^/]+/[^/]+/v${repoVersion.replaceAll(".", "\\.")}/lib/hosts/link-protocol\\.ts$`); + assert.ok(run.fetched().some((url) => candidateSource.test(url)), "the old updater read the target release's link protocol"); + + // Everything the legacy connection ever received: transport and the two update requests, each with the one + // compatibility an application-12 release may be reached through. The v12 welcome ends it. + const frames = sent(); + const upgraded = frames.findIndex((message) => message.type === "welcome" && message.v === APPLICATION); + assert.ok(upgraded > 0, "the upgraded daemon is welcomed at the application version"); + const legacyFrames = frames.slice(0, upgraded); + assert.deepEqual([...new Set(legacyFrames.map((message) => message.type))].filter((type) => !["welcome", "ack", "ping", "pong", "req"].includes(type)), []); + const requests = legacyFrames.filter((message) => message.type === "req"); + assert.deepEqual([...new Set(requests.map((message) => message.op))].sort(), ["checkUpdates", "updateSoftware"]); + for (const request of requests) assert.deepEqual(request.args.compatibility, { min: APPLICATION, max: APPLICATION }); + assert.deepEqual(requests.filter((message) => message.op === "updateSoftware").map((message) => [message.args.target, message.args.version]), [["ompweb", repoVersion]]); +} + +const skipWindows = process.platform === "win32" && "this proof needs git archive, symlinked dependencies and a POSIX process table"; +const skipScope = "systemd-run cannot create the scope that keeps this test outside the service running the suite"; +async function scenario(t, legacy, layout) { + if (skipWindows) return t.skip(skipWindows); + if (!haveCommit(legacy.commit)) return t.skip(`commit ${legacy.commit} (${legacy.name} daemon) is not in this clone; fetch full history`); + const run = await startLegacy(legacy, layout); + if (!run) return t.skip(skipScope); + try { await upgrade(run); } finally { await stopAll(run.work); } +} + +test("v10 daemon (546a945d) is welcomed as v10 and upgraded by its own updater to a separate v12 install", { timeout: 240_000 }, (t) => scenario(t, LEGACY.v10, "separate")); +test("v10 daemon (546a945d) on an installation the web package already replaced is restarted, not reinstalled", { timeout: 240_000 }, (t) => scenario(t, LEGACY.v10, "shared")); +test("v11 daemon (v0.7.0) is welcomed as v11 and upgraded by its own updater to a separate v12 install", { timeout: 240_000 }, (t) => scenario(t, LEGACY.v11, "separate")); +test("v11 daemon (v0.7.0) on an installation the web package already replaced is restarted, not reinstalled", { timeout: 240_000 }, (t) => scenario(t, LEGACY.v11, "shared")); diff --git a/lib/hosts/workspace-watch.test.mjs b/lib/hosts/workspace-watch.test.mjs index 31427fefc..ef1984683 100644 --- a/lib/hosts/workspace-watch.test.mjs +++ b/lib/hosts/workspace-watch.test.mjs @@ -21,7 +21,16 @@ test("a real daemon watch observes the final burst and atomic replacements, then writeFileSync(path, "second"); writeFileSync(path, "final burst bytes"); const until = async (size) => { const end = Date.now() + 3000; while (!events.some((e) => e.type === "change" && e.size === size)) { assert.ok(Date.now() < end, "watch must report final bytes"); await delay(10); } }; await until(17); - const replacement = join(owner.project, "replacement"); writeFileSync(replacement, "atomic save"); renameSync(replacement, path); await until(11); + const replacement = join(owner.project, "replacement"); writeFileSync(replacement, "atomic save"); + // Windows can briefly deny replacement while another process inspects the watched path. + for (const deadline = Date.now() + 2000; ;) { + try { renameSync(replacement, path); break; } + catch (error) { + if (process.platform !== "win32" || error.code !== "EPERM" || Date.now() >= deadline) throw error; + await delay(10); + } + } + await until(11); assert.equal((await owner.host.workspaceOps.unwatchFile(id, "old-epoch")).ok, true); writeFileSync(path, "newer epoch survives old close"); await until(30); await owner.host.workspaceOps.unwatchFile(id, epoch); diff --git a/package-lock.json b/package-lock.json index cc6b22349..487ee1095 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@kahme247/ompweb", - "version": "0.7.0", + "version": "0.7.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@kahme247/ompweb", - "version": "0.7.0", + "version": "0.7.1", "license": "MIT", "dependencies": { "@base-ui/react": "^1.7.0", diff --git a/package.json b/package.json index 59d3b36b8..7446ec74c 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@kahme247/ompweb", - "version": "0.7.0", + "version": "0.7.1", "description": "Web UI for the oh-my-pi (omp) coding agent", "homepage": "https://github.com/ompweb-dev/ompweb#readme", "repository": { diff --git a/proxy.ts b/proxy.ts index d51fb6b5b..36f5f59f9 100644 --- a/proxy.ts +++ b/proxy.ts @@ -24,7 +24,8 @@ import { // /api/hosts/cluster-config/manifest and /api/hosts/cluster-config/file/<64 hex digits> (exactly: hosts/[id]/cluster and // hosts/[id]/updates answer to id "cluster-config" and stay gated) check it before serving the master's configuration // to a cluster member (app/api/hosts/cluster-config/). -export const UNGATED = /^\/(?:assets\/|(?:favicon\.ico|icon\.svg|icon\.png|icon-192\.png|badge-96\.png|sw\.js)$|api\/hosts\/link$|api\/hosts\/broker\/v1\/|api\/hosts\/cluster-config\/(?:manifest|file\/[0-9a-f]{64})$)/; +// /api/hosts/maintenance is the independent maintenance API; its bearer gate also covers discovery. +export const UNGATED = /^\/(?:assets\/|(?:favicon\.ico|icon\.svg|icon\.png|icon-192\.png|badge-96\.png|sw\.js)$|api\/hosts\/(?:link|maintenance)$|api\/hosts\/broker\/v1\/|api\/hosts\/cluster-config\/(?:manifest|file\/[0-9a-f]{64})$)/; // Relative: request.url is always http:// under Bun, so an absolute Location would leave https behind a TLS proxy. const redirect = (location: string) => new Response(null, { status: 307, headers: { location } }); diff --git a/server/app.ts b/server/app.ts index e5169c2ca..c0a7191e8 100644 --- a/server/app.ts +++ b/server/app.ts @@ -3,6 +3,7 @@ import path from "node:path"; import { parseArgs } from "node:util"; import { browserLink, type BrowserLinkData } from "@/lib/browser-link"; import { hostLinkSocket, type LinkSocketData } from "@/lib/hosts/hub"; +import { maintenanceSocket, type MaintenanceSocketData } from "@/lib/hosts/maintenance-hub"; import { proxy } from "@/proxy"; import { ensureOmpRunner } from "@/lib/omp/runner-fetch"; import { register } from "../instrumentation.node"; @@ -26,6 +27,9 @@ type ServerSocket = Bun.ServerWebSocket; function isBrowserSocket(ws: ServerSocket): ws is Bun.ServerWebSocket { return "kind" in ws.data && ws.data.kind === "browser"; } +function isMaintenanceSocket(ws: ServerSocket): ws is Bun.ServerWebSocket { + return "kind" in ws.data && ws.data.kind === "maintenance"; +} // Not a browser link: a host link (the only other upgrade). const asLink = (ws: ServerSocket) => ws as Bun.ServerWebSocket; @@ -157,10 +161,22 @@ export function startServer(options: { port: number; hostname: string; dev: bool ...hostLinkSocket, open: (ws: ServerSocket) => { if (isBrowserSocket(ws)) browserLink.open(ws); + else if (isMaintenanceSocket(ws)) maintenanceSocket.open(ws); + }, + message: (ws: ServerSocket, data: string | Uint8Array) => { + if (isBrowserSocket(ws)) browserLink.message(ws, data); + else if (isMaintenanceSocket(ws)) maintenanceSocket.message(ws, data); + else hostLinkSocket.message(asLink(ws), data); + }, + drain: (ws: ServerSocket) => { + if (isBrowserSocket(ws)) browserLink.drain(ws); + else if (!isMaintenanceSocket(ws)) hostLinkSocket.drain(asLink(ws)); + }, + close: (ws: ServerSocket, code: number) => { + if (isBrowserSocket(ws)) browserLink.close(ws, code); + else if (isMaintenanceSocket(ws)) maintenanceSocket.close(ws); + else hostLinkSocket.close(asLink(ws)); }, - message: (ws: ServerSocket, data: string | Uint8Array) => (isBrowserSocket(ws) ? browserLink.message(ws, data) : hostLinkSocket.message(asLink(ws), data)), - drain: (ws: ServerSocket) => (isBrowserSocket(ws) ? browserLink.drain(ws) : hostLinkSocket.drain(asLink(ws))), - close: (ws: ServerSocket, code: number) => (isBrowserSocket(ws) ? browserLink.close(ws, code) : hostLinkSocket.close(asLink(ws))), }, fetch: async (request, server) => { const response = await handle(request, server, async ({ gate }) => { diff --git a/server/routes.ts b/server/routes.ts index e69bbfb3b..d6cfea353 100644 --- a/server/routes.ts +++ b/server/routes.ts @@ -26,9 +26,11 @@ import * as hostsBrokerPath from "@/app/api/hosts/broker/[...path]/route"; import * as hostsClusterConfigFileHash from "@/app/api/hosts/cluster-config/file/[hash]/route"; import * as hostsClusterConfigManifest from "@/app/api/hosts/cluster-config/manifest/route"; import * as hostsLink from "@/app/api/hosts/link/route"; +import * as hostsMaintenance from "@/app/api/hosts/maintenance/route"; import * as hosts from "@/app/api/hosts/route"; import * as hostsIdCluster from "@/app/api/hosts/[id]/cluster/route"; import * as hostsIdUpdates from "@/app/api/hosts/[id]/updates/route"; +import * as hostsIdMaintenance from "@/app/api/hosts/[id]/maintenance/route"; import * as manifest from "@/app/api/manifest/route"; import * as mcp from "@/app/api/mcp/route"; import * as mediaHash from "@/app/api/media/[hash]/route"; @@ -82,9 +84,10 @@ import * as windowsService from "@/app/api/windows-service/route"; import * as worktrees from "@/app/api/worktrees/route"; import type { BrowserLinkData } from "@/lib/browser-link"; import type { LinkSocketData } from "@/lib/hosts/hub"; +import type { MaintenanceSocketData } from "@/lib/hosts/maintenance-hub"; -/** What Bun keeps on an upgraded socket (`ws.data`): a host link's or a browser link's. */ -export type ServerSocketData = LinkSocketData | BrowserLinkData; +/** What Bun keeps on an upgraded socket: application, maintenance API, or browser transport. */ +export type ServerSocketData = LinkSocketData | MaintenanceSocketData | BrowserLinkData; type Method = "GET" | "POST" | "PUT" | "PATCH" | "DELETE"; // never: each handler narrows its own params; the router passes Record. @@ -126,8 +129,10 @@ export const ROUTE_MODULES: Record = { "hosts/cluster-config/file/[hash]": hostsClusterConfigFileHash, "hosts/cluster-config/manifest": hostsClusterConfigManifest, "hosts/link": hostsLink, + "hosts/maintenance": hostsMaintenance, "hosts": hosts, "hosts/[id]/cluster": hostsIdCluster, + "hosts/[id]/maintenance": hostsIdMaintenance, "hosts/[id]/updates": hostsIdUpdates, "manifest": manifest, "mcp": mcp,