From abb1ddefbfe2660f4cee1af36ab78becb00f4339 Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Sat, 7 Nov 2026 08:38:54 -0600 Subject: [PATCH 1/2] Make backend shutdown safe on Python 3.13 and report a stuck backend - The stdin pump stops reading once it has delivered `quit` (#3). A read still in flight when the interpreter shuts down holds stdin's buffer lock; Python 3.13 parks that thread forever and closing stdin at exit aborts the process. - backend_exited is reported when the backend process exits, if its output has not closed within a second, with stdio_open: true (#35). In the packaged layout the interpreter behind the venv launcher can keep the pipes open long after the launcher has gone. Reported once, whichever of the spawn error, exit and close comes first. - After the Windows backstop's taskkill, a backend_stuck message (pid, kill status) reports a tree whose output is still open 3 s later. --- de_shell/ipc.py | 18 +++++++ de_shell/js/main/backendProcess.test.ts | 34 +++++++++++++ de_shell/js/main/backendProcess.ts | 66 +++++++++++++++++++++---- de_shell/js/renderer/index.ts | 2 +- de_shell/js/renderer/protocol.ts | 17 ++++++- tests/test_ipc_read_messages.py | 42 ++++++++++++++++ 6 files changed, 167 insertions(+), 12 deletions(-) create mode 100644 tests/test_ipc_read_messages.py diff --git a/de_shell/ipc.py b/de_shell/ipc.py index 61db345..7791019 100644 --- a/de_shell/ipc.py +++ b/de_shell/ipc.py @@ -174,6 +174,17 @@ def emit_window_computing(window_id: int | None, computing: bool) -> None: emit({"type": "window_computing", "window_id": window_id, "computing": bool(computing)}) +def _is_quit(line: str) -> bool: + """True for a ``{"type": "quit"}`` line; cheap for every other line.""" + if '"quit"' not in line: + return False + try: + message = json.loads(line) + except json.JSONDecodeError: + return False + return isinstance(message, dict) and message.get("type") == "quit" + + async def read_messages(loop: asyncio.AbstractEventLoop | None = None): """ Async generator that yields parsed JSON dicts from stdin. @@ -216,6 +227,13 @@ def _pump() -> None: if binary: raw = raw.decode("utf-8", errors="replace") loop.call_soon_threadsafe(q.put_nowait, raw) + if _is_quit(raw): + # Stop reading: the frontend may keep writing after `quit`, + # and a readline still in flight when the interpreter shuts + # down holds stdin's buffer lock. Python 3.13 then parks this + # thread forever, lock held, and closing stdin at exit aborts + # the process (SIGABRT, "could not acquire lock"). + break except Exception as e: log.debug("stdin pump stopped: %s", e) finally: diff --git a/de_shell/js/main/backendProcess.test.ts b/de_shell/js/main/backendProcess.test.ts index 8abc8a4..3538e25 100644 --- a/de_shell/js/main/backendProcess.test.ts +++ b/de_shell/js/main/backendProcess.test.ts @@ -68,3 +68,37 @@ test('a missing uv names uv and says how to fix it', async () => { const text = streams.join('') assert.match(text, /PATH/i) }) + +test('a clean exit is reported once, after the output closes', async () => { + const { messages } = collectBackend([process.execPath, '-e', 'process.exit(3)']) + try { + await waitFor(() => messages.some((m) => m.type === 'backend_exited')) + await new Promise((r) => setTimeout(r, 1500)) + } finally { + stopBackend() + } + const exits = messages.filter((m) => m.type === 'backend_exited') + assert.deepEqual(exits, [{ type: 'backend_exited', code: 3 }]) +}) + +// On Windows a child's pipes report closed when it exits, even while a +// process it started still writes to them, so there is no case to construct. +const outputOutlivesExit = process.platform !== 'win32' + +test('an exit whose output a child keeps open is reported without waiting for it', + { skip: !outputOutlivesExit && 'pipes close at exit on Windows' }, async () => { + // The packaged layout: the spawned launcher exits while the interpreter it + // started still holds the same stdout and stderr. + const holdsOutput = "require('child_process').spawn(process.execPath, " + + "['-e', 'setTimeout(() => {}, 6000)'], { stdio: 'inherit' }); process.exit(0)" + const started = Date.now() + const { messages } = collectBackend([process.execPath, '-e', holdsOutput]) + try { + await waitFor(() => messages.some((m) => m.type === 'backend_exited')) + } finally { + stopBackend() + } + assert.ok(Date.now() - started < 5000, 'the exit waited for the output to close') + const exited = messages.find((m) => m.type === 'backend_exited') + assert.deepEqual(exited, { type: 'backend_exited', code: 0, stdio_open: true }) +}) diff --git a/de_shell/js/main/backendProcess.ts b/de_shell/js/main/backendProcess.ts index f64a668..4ef38b0 100644 --- a/de_shell/js/main/backendProcess.ts +++ b/de_shell/js/main/backendProcess.ts @@ -22,6 +22,14 @@ export interface BackendHandlers { let proc: ChildProcess | null = null let tickTimer: ReturnType | null = null +/** The running backend's handlers, so stopBackend can report a stuck tree. */ +let procHandlers: BackendHandlers | null = null +/** How long after the backstop's kill the backend's output may stay open + * before it is reported as stuck. */ +const STUCK_AFTER_KILL_MS = 3000 +/** How long after the backend process exits its output may take to close + * before the exit is reported without it. */ +const CLOSE_AFTER_EXIT_MS = 1000 /** * The sidecar's last few hundred output lines, for a problem report. @@ -77,6 +85,7 @@ export function startBackend( stdio: ['pipe', 'pipe', 'pipe'], }) proc = child + procHandlers = handlers // BACKEND TICK (0.5 Hz): Windows throttles timer delivery to the hidden // Python child so aggressively that its timer waits (time.sleep, @@ -114,6 +123,18 @@ export function startBackend( handlers.onStream(d.toString(), 'stderr') }) + // Surface the death to the renderer so the UI doesn't silently freeze — + // every sendAction() after this no-ops (proc is null), so without this the + // user gets no indication the analysis backend stopped. Routed through the + // same onMessage path as a synthetic message (not a PLOTAPP: line). Once per + // backend: a failed spawn, 'exit' and 'close' can each arrive. + let exitReported = false + const reportExit = (details: Record) => { + if (exitReported) return + exitReported = true + handlers.onMessage({ type: 'backend_exited', ...details }) + } + // SPAWN-ERROR TRAP: a command that cannot be started (uv missing from PATH is // the recurring case) emits 'error' on the ChildProcess — and an 'error' event // with no listener CRASHES the Electron main process. 'close' never fires for @@ -128,21 +149,28 @@ export function startBackend( (enoent ? ` — ${cmd} was not found. Is it installed and on PATH?` : '') + '\n', 'stderr') - handlers.onMessage({ type: 'backend_exited', code: null, error: String(err.message ?? err) }) + reportExit({ code: null, error: String(err.message ?? err) }) }) - child.on('close', (code) => { + // 'close' also waits for stdout and stderr to close, and in the packaged + // layout the spawned command is the venv's launcher while the real + // interpreter is its child holding the same pipes — so the launcher can be + // gone long before 'close' arrives (23.7 s in one case, the interpreter's + // exit pending on a DLL load). So 'exit' reports too, once 'close' has had + // a moment to follow, with `stdio_open`: the launcher exited, its output is + // still open. + child.on('exit', (code) => { // Only forget THIS child. After stopBackend() + startBackend(), the old - // process's late close event must not null the new backend's handle — + // process's late exit event must not null the new backend's handle — // every sendAction() after that would silently no-op. + if (proc === child) proc = null + setTimeout(() => reportExit({ code, stdio_open: true }), CLOSE_AFTER_EXIT_MS).unref() + }) + child.on('close', (code) => { if (proc === child) proc = null rememberOutput(`[exited with code ${code}]`) handlers.onStream(`[${shellConfig().appName} exited with code ${code}]\n`, 'stderr') - // Surface the death to the renderer so the UI doesn't silently freeze — - // every sendAction() after this no-ops (proc is null), so without this the - // user gets no indication the analysis backend stopped. Routed through the - // same onMessage path as a synthetic message (not a PLOTAPP: line). - handlers.onMessage({ type: 'backend_exited', code }) + reportExit({ code }) }) } @@ -211,6 +239,7 @@ export function stopBackend(options: { immediate?: boolean } = {}): void { } stopping = true proc = null // every sendAction() after this no-ops; prevents re-entrant kills + const handlers = procHandlers // KNOWN BUG (open): quitting while a find-vectors batch is still streaming can // WEDGE shutdown on Windows. Clearing the tick timer here stops the stdin tick // that keeps the hidden backend scheduled (the very starvation the tick was @@ -241,7 +270,9 @@ export function stopBackend(options: { immediate?: boolean } = {}): void { setTimeout(() => { if (p.exitCode !== null || p.signalCode !== null) return // already gone try { - spawn('taskkill', ['/pid', String(pid), '/T', '/F'], { stdio: 'ignore' }) + const kill = spawn('taskkill', ['/pid', String(pid), '/T', '/F'], { stdio: 'ignore' }) + kill.on('error', () => { /* reported as stuck below if the tree survives */ }) + kill.on('close', (status) => reportIfStuck(p, pid, status, handlers)) } catch { try { p.kill() } catch { /* nothing else to do */ } } // 5 s, not 1.5: Ground Crew's shutdown must send deapi's disconnect // before dying — a taskkilled backend leaves a dead client session on @@ -263,6 +294,23 @@ export function stopBackend(options: { immediate?: boolean } = {}): void { } } +/** + * After the backstop's kill, report `backend_stuck` if the backend's output is + * still open a little later: something in its tree outlived the kill, and the + * app cannot otherwise tell a backend still running from one that has gone. + * `status` is taskkill's exit status (0: the tree was ended). + */ +function reportIfStuck( + p: ChildProcess, pid: number, status: number | null, handlers: BackendHandlers | null, +): void { + let closed = false + p.once('close', () => { closed = true }) + setTimeout(() => { + if (closed || p.stdout?.readableEnded || p.stdout?.destroyed) return + handlers?.onMessage({ type: 'backend_stuck', pid, kill_status: status }) + }, STUCK_AFTER_KILL_MS).unref?.() +} + /** * Kill the sidecar's whole process tree and return once the request has been * made — no timers, so it still runs when the caller is about to end this diff --git a/de_shell/js/renderer/index.ts b/de_shell/js/renderer/index.ts index a68409e..35344e0 100644 --- a/de_shell/js/renderer/index.ts +++ b/de_shell/js/renderer/index.ts @@ -27,7 +27,7 @@ export { asShellMessage } from './protocol' export type { MsgBase, ShellMessage, ReadyMessage, StatusMessage, ErrorMessage, ProgressMessage, - BackendExitedMessage, EnvSetupMessage, EnvSetupEvent, + BackendExitedMessage, BackendStuckMessage, EnvSetupMessage, EnvSetupEvent, FigureMessage, StateUpdateMessage, StateUpdateBinaryMessage, WindowClosedMessage, WindowTitleMessage, WindowComputingMessage, LogMessage, LogBackfillMessage, LogLevelMessage, diff --git a/de_shell/js/renderer/protocol.ts b/de_shell/js/renderer/protocol.ts index c235dd5..7f43da9 100644 --- a/de_shell/js/renderer/protocol.ts +++ b/de_shell/js/renderer/protocol.ts @@ -13,8 +13,8 @@ * being a compile error. This types the shape the renderer relies on; it is not * a schema. * - * `backend_exited` is synthesised by @de/shell-main's backendProcess, not a real - * PLOTAPP line. + * `backend_exited` and `backend_stuck` are synthesised by @de/shell-main's + * backendProcess, not real PLOTAPP lines. */ /** Any field not explicitly modelled is still readable (as `unknown`). */ @@ -50,6 +50,18 @@ export interface BackendExitedMessage extends MsgBase { /** Set when main synthesises this for a packaged env-setup failure, as * distinct from a plain runtime death. */ reason?: string + /** The backend process exited but its output is still open: a process it + * started, such as the interpreter behind a venv launcher, still holds it. */ + stdio_open?: boolean +} + +/** The backstop killed the backend's tree, yet its output is still open: + * something outlived the kill. `kill_status` is the kill command's exit + * status (0: it reported the tree ended). Synthesised by backendProcess. */ +export interface BackendStuckMessage extends MsgBase { + type: 'backend_stuck' + pid: number + kill_status: number | null } /** A classified `uv` line (main's EnvProgressEvent, restated). */ @@ -165,6 +177,7 @@ export type ShellMessage = | ErrorMessage | ProgressMessage | BackendExitedMessage + | BackendStuckMessage | EnvSetupMessage | FigureMessage | StateUpdateMessage diff --git a/tests/test_ipc_read_messages.py b/tests/test_ipc_read_messages.py new file mode 100644 index 0000000..0719141 --- /dev/null +++ b/tests/test_ipc_read_messages.py @@ -0,0 +1,42 @@ +""" +test_ipc_read_messages.py — the stdin pump stops reading at ``quit``. + +A read still in flight when the interpreter shuts down holds stdin's buffer +lock; on Python 3.13 that thread is parked forever and closing stdin at exit +aborts the process. So nothing after ``quit`` may be read. +""" +from __future__ import annotations + +import asyncio +import io +import sys +import types + +from de_shell import ipc + + +def read_all(lines: bytes, monkeypatch) -> tuple[list[dict], io.BytesIO]: + stream = io.BytesIO(lines) + monkeypatch.setattr(sys, "stdin", types.SimpleNamespace(buffer=stream)) + + async def collect(): + return [message async for message in ipc.read_messages(asyncio.get_running_loop())] + + return asyncio.run(collect()), stream + + +class TestReadMessages: + def test_reading_stops_after_quit(self, monkeypatch): + before = b'{"type": "action", "action": "a"}\n{"type": "quit"}\n' + messages, stream = read_all(before + b'{"type": "tick"}\n', monkeypatch) + assert [m["type"] for m in messages] == ["action", "quit"] + assert stream.tell() == len(before) + + def test_quit_inside_a_payload_does_not_stop_reading(self, monkeypatch): + lines = b'{"type": "action", "payload": {"label": "quit"}}\n{"type": "tick"}\n' + messages, _ = read_all(lines, monkeypatch) + assert [m["type"] for m in messages] == ["action", "tick"] + + def test_end_of_input_still_ends_the_stream(self, monkeypatch): + messages, _ = read_all(b'{"type": "tick"}\n', monkeypatch) + assert [m["type"] for m in messages] == ["tick"] From d31c54d756d8415648b15a72d0003b9479e7b5a5 Mon Sep 17 00:00:00 2001 From: Carter Francis Date: Sat, 7 Nov 2026 08:39:05 -0600 Subject: [PATCH 2/2] Add changelog fragments for #39 --- upcoming_changes/39.api_change.rst | 1 + upcoming_changes/39.bugfix.rst | 1 + 2 files changed, 2 insertions(+) create mode 100644 upcoming_changes/39.api_change.rst create mode 100644 upcoming_changes/39.bugfix.rst diff --git a/upcoming_changes/39.api_change.rst b/upcoming_changes/39.api_change.rst new file mode 100644 index 0000000..6bee5cd --- /dev/null +++ b/upcoming_changes/39.api_change.rst @@ -0,0 +1 @@ +``backend_exited`` arrives when the backend process exits, with ``stdio_open: true`` if its output is still held open, instead of only once every pipe closes; a new ``backend_stuck`` message reports a backend tree that survived the shutdown backstop. diff --git a/upcoming_changes/39.bugfix.rst b/upcoming_changes/39.bugfix.rst new file mode 100644 index 0000000..1a2da95 --- /dev/null +++ b/upcoming_changes/39.bugfix.rst @@ -0,0 +1 @@ +The backend no longer aborts at exit on Python 3.13 when the app keeps writing to it after asking it to quit.