Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions de_shell/ipc.py
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,17 @@ def emit_window_computing(window_id: int | None, computing: bool) -> None:
emit({"type": "window_computing", "window_id": window_id, "computing": bool(computing)})


def _is_quit(line: str) -> bool:
"""True for a ``{"type": "quit"}`` line; cheap for every other line."""
if '"quit"' not in line:
return False
try:
message = json.loads(line)
except json.JSONDecodeError:
return False
return isinstance(message, dict) and message.get("type") == "quit"


async def read_messages(loop: asyncio.AbstractEventLoop | None = None):
"""
Async generator that yields parsed JSON dicts from stdin.
Expand Down Expand Up @@ -216,6 +227,13 @@ def _pump() -> None:
if binary:
raw = raw.decode("utf-8", errors="replace")
loop.call_soon_threadsafe(q.put_nowait, raw)
if _is_quit(raw):
# Stop reading: the frontend may keep writing after `quit`,
# and a readline still in flight when the interpreter shuts
# down holds stdin's buffer lock. Python 3.13 then parks this
# thread forever, lock held, and closing stdin at exit aborts
# the process (SIGABRT, "could not acquire lock").
break
except Exception as e:
log.debug("stdin pump stopped: %s", e)
finally:
Expand Down
34 changes: 34 additions & 0 deletions de_shell/js/main/backendProcess.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -68,3 +68,37 @@ test('a missing uv names uv and says how to fix it', async () => {
const text = streams.join('')
assert.match(text, /PATH/i)
})

test('a clean exit is reported once, after the output closes', async () => {
const { messages } = collectBackend([process.execPath, '-e', 'process.exit(3)'])
try {
await waitFor(() => messages.some((m) => m.type === 'backend_exited'))
await new Promise((r) => setTimeout(r, 1500))
} finally {
stopBackend()
}
const exits = messages.filter((m) => m.type === 'backend_exited')
assert.deepEqual(exits, [{ type: 'backend_exited', code: 3 }])
})

// On Windows a child's pipes report closed when it exits, even while a
// process it started still writes to them, so there is no case to construct.
const outputOutlivesExit = process.platform !== 'win32'

test('an exit whose output a child keeps open is reported without waiting for it',
{ skip: !outputOutlivesExit && 'pipes close at exit on Windows' }, async () => {
// The packaged layout: the spawned launcher exits while the interpreter it
// started still holds the same stdout and stderr.
const holdsOutput = "require('child_process').spawn(process.execPath, " +
"['-e', 'setTimeout(() => {}, 6000)'], { stdio: 'inherit' }); process.exit(0)"
const started = Date.now()
const { messages } = collectBackend([process.execPath, '-e', holdsOutput])
try {
await waitFor(() => messages.some((m) => m.type === 'backend_exited'))
} finally {
stopBackend()
}
assert.ok(Date.now() - started < 5000, 'the exit waited for the output to close')
const exited = messages.find((m) => m.type === 'backend_exited')
assert.deepEqual(exited, { type: 'backend_exited', code: 0, stdio_open: true })
})
66 changes: 57 additions & 9 deletions de_shell/js/main/backendProcess.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,14 @@ export interface BackendHandlers {

let proc: ChildProcess | null = null
let tickTimer: ReturnType<typeof setInterval> | null = null
/** The running backend's handlers, so stopBackend can report a stuck tree. */
let procHandlers: BackendHandlers | null = null
/** How long after the backstop's kill the backend's output may stay open
* before it is reported as stuck. */
const STUCK_AFTER_KILL_MS = 3000
/** How long after the backend process exits its output may take to close
* before the exit is reported without it. */
const CLOSE_AFTER_EXIT_MS = 1000

/**
* The sidecar's last few hundred output lines, for a problem report.
Expand Down Expand Up @@ -77,6 +85,7 @@ export function startBackend(
stdio: ['pipe', 'pipe', 'pipe'],
})
proc = child
procHandlers = handlers

// BACKEND TICK (0.5 Hz): Windows throttles timer delivery to the hidden
// Python child so aggressively that its timer waits (time.sleep,
Expand Down Expand Up @@ -114,6 +123,18 @@ export function startBackend(
handlers.onStream(d.toString(), 'stderr')
})

// Surface the death to the renderer so the UI doesn't silently freeze —
// every sendAction() after this no-ops (proc is null), so without this the
// user gets no indication the analysis backend stopped. Routed through the
// same onMessage path as a synthetic message (not a PLOTAPP: line). Once per
// backend: a failed spawn, 'exit' and 'close' can each arrive.
let exitReported = false
const reportExit = (details: Record<string, unknown>) => {
if (exitReported) return
exitReported = true
handlers.onMessage({ type: 'backend_exited', ...details })
}

// SPAWN-ERROR TRAP: a command that cannot be started (uv missing from PATH is
// the recurring case) emits 'error' on the ChildProcess — and an 'error' event
// with no listener CRASHES the Electron main process. 'close' never fires for
Expand All @@ -128,21 +149,28 @@ export function startBackend(
(enoent ? ` — ${cmd} was not found. Is it installed and on PATH?` : '') +
'\n',
'stderr')
handlers.onMessage({ type: 'backend_exited', code: null, error: String(err.message ?? err) })
reportExit({ code: null, error: String(err.message ?? err) })
})

child.on('close', (code) => {
// 'close' also waits for stdout and stderr to close, and in the packaged
// layout the spawned command is the venv's launcher while the real
// interpreter is its child holding the same pipes — so the launcher can be
// gone long before 'close' arrives (23.7 s in one case, the interpreter's
// exit pending on a DLL load). So 'exit' reports too, once 'close' has had
// a moment to follow, with `stdio_open`: the launcher exited, its output is
// still open.
child.on('exit', (code) => {
// Only forget THIS child. After stopBackend() + startBackend(), the old
// process's late close event must not null the new backend's handle —
// process's late exit event must not null the new backend's handle —
// every sendAction() after that would silently no-op.
if (proc === child) proc = null
setTimeout(() => reportExit({ code, stdio_open: true }), CLOSE_AFTER_EXIT_MS).unref()
})
child.on('close', (code) => {
if (proc === child) proc = null
rememberOutput(`[exited with code ${code}]`)
handlers.onStream(`[${shellConfig().appName} exited with code ${code}]\n`, 'stderr')
// Surface the death to the renderer so the UI doesn't silently freeze —
// every sendAction() after this no-ops (proc is null), so without this the
// user gets no indication the analysis backend stopped. Routed through the
// same onMessage path as a synthetic message (not a PLOTAPP: line).
handlers.onMessage({ type: 'backend_exited', code })
reportExit({ code })
})
}

Expand Down Expand Up @@ -211,6 +239,7 @@ export function stopBackend(options: { immediate?: boolean } = {}): void {
}
stopping = true
proc = null // every sendAction() after this no-ops; prevents re-entrant kills
const handlers = procHandlers
// KNOWN BUG (open): quitting while a find-vectors batch is still streaming can
// WEDGE shutdown on Windows. Clearing the tick timer here stops the stdin tick
// that keeps the hidden backend scheduled (the very starvation the tick was
Expand Down Expand Up @@ -241,7 +270,9 @@ export function stopBackend(options: { immediate?: boolean } = {}): void {
setTimeout(() => {
if (p.exitCode !== null || p.signalCode !== null) return // already gone
try {
spawn('taskkill', ['/pid', String(pid), '/T', '/F'], { stdio: 'ignore' })
const kill = spawn('taskkill', ['/pid', String(pid), '/T', '/F'], { stdio: 'ignore' })
kill.on('error', () => { /* reported as stuck below if the tree survives */ })
kill.on('close', (status) => reportIfStuck(p, pid, status, handlers))
} catch { try { p.kill() } catch { /* nothing else to do */ } }
// 5 s, not 1.5: Ground Crew's shutdown must send deapi's disconnect
// before dying — a taskkilled backend leaves a dead client session on
Expand All @@ -263,6 +294,23 @@ export function stopBackend(options: { immediate?: boolean } = {}): void {
}
}

/**
* After the backstop's kill, report `backend_stuck` if the backend's output is
* still open a little later: something in its tree outlived the kill, and the
* app cannot otherwise tell a backend still running from one that has gone.
* `status` is taskkill's exit status (0: the tree was ended).
*/
function reportIfStuck(
p: ChildProcess, pid: number, status: number | null, handlers: BackendHandlers | null,
): void {
let closed = false
p.once('close', () => { closed = true })
setTimeout(() => {
if (closed || p.stdout?.readableEnded || p.stdout?.destroyed) return
handlers?.onMessage({ type: 'backend_stuck', pid, kill_status: status })
}, STUCK_AFTER_KILL_MS).unref?.()
}

/**
* Kill the sidecar's whole process tree and return once the request has been
* made — no timers, so it still runs when the caller is about to end this
Expand Down
2 changes: 1 addition & 1 deletion de_shell/js/renderer/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ export { asShellMessage } from './protocol'
export type {
MsgBase, ShellMessage,
ReadyMessage, StatusMessage, ErrorMessage, ProgressMessage,
BackendExitedMessage, EnvSetupMessage, EnvSetupEvent,
BackendExitedMessage, BackendStuckMessage, EnvSetupMessage, EnvSetupEvent,
FigureMessage, StateUpdateMessage, StateUpdateBinaryMessage,
WindowClosedMessage, WindowTitleMessage, WindowComputingMessage,
LogMessage, LogBackfillMessage, LogLevelMessage,
Expand Down
17 changes: 15 additions & 2 deletions de_shell/js/renderer/protocol.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@
* being a compile error. This types the shape the renderer relies on; it is not
* a schema.
*
* `backend_exited` is synthesised by @de/shell-main's backendProcess, not a real
* PLOTAPP line.
* `backend_exited` and `backend_stuck` are synthesised by @de/shell-main's
* backendProcess, not real PLOTAPP lines.
*/

/** Any field not explicitly modelled is still readable (as `unknown`). */
Expand Down Expand Up @@ -50,6 +50,18 @@ export interface BackendExitedMessage extends MsgBase {
/** Set when main synthesises this for a packaged env-setup failure, as
* distinct from a plain runtime death. */
reason?: string
/** The backend process exited but its output is still open: a process it
* started, such as the interpreter behind a venv launcher, still holds it. */
stdio_open?: boolean
}

/** The backstop killed the backend's tree, yet its output is still open:
* something outlived the kill. `kill_status` is the kill command's exit
* status (0: it reported the tree ended). Synthesised by backendProcess. */
export interface BackendStuckMessage extends MsgBase {
type: 'backend_stuck'
pid: number
kill_status: number | null
}

/** A classified `uv` line (main's EnvProgressEvent, restated). */
Expand Down Expand Up @@ -165,6 +177,7 @@ export type ShellMessage =
| ErrorMessage
| ProgressMessage
| BackendExitedMessage
| BackendStuckMessage
| EnvSetupMessage
| FigureMessage
| StateUpdateMessage
Expand Down
42 changes: 42 additions & 0 deletions tests/test_ipc_read_messages.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
"""
test_ipc_read_messages.py — the stdin pump stops reading at ``quit``.

A read still in flight when the interpreter shuts down holds stdin's buffer
lock; on Python 3.13 that thread is parked forever and closing stdin at exit
aborts the process. So nothing after ``quit`` may be read.
"""
from __future__ import annotations

import asyncio
import io
import sys
import types

from de_shell import ipc


def read_all(lines: bytes, monkeypatch) -> tuple[list[dict], io.BytesIO]:
stream = io.BytesIO(lines)
monkeypatch.setattr(sys, "stdin", types.SimpleNamespace(buffer=stream))

async def collect():
return [message async for message in ipc.read_messages(asyncio.get_running_loop())]

return asyncio.run(collect()), stream


class TestReadMessages:
def test_reading_stops_after_quit(self, monkeypatch):
before = b'{"type": "action", "action": "a"}\n{"type": "quit"}\n'
messages, stream = read_all(before + b'{"type": "tick"}\n', monkeypatch)
assert [m["type"] for m in messages] == ["action", "quit"]
assert stream.tell() == len(before)

def test_quit_inside_a_payload_does_not_stop_reading(self, monkeypatch):
lines = b'{"type": "action", "payload": {"label": "quit"}}\n{"type": "tick"}\n'
messages, _ = read_all(lines, monkeypatch)
assert [m["type"] for m in messages] == ["action", "tick"]

def test_end_of_input_still_ends_the_stream(self, monkeypatch):
messages, _ = read_all(b'{"type": "tick"}\n', monkeypatch)
assert [m["type"] for m in messages] == ["tick"]
1 change: 1 addition & 0 deletions upcoming_changes/39.api_change.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
``backend_exited`` arrives when the backend process exits, with ``stdio_open: true`` if its output is still held open, instead of only once every pipe closes; a new ``backend_stuck`` message reports a backend tree that survived the shutdown backstop.
1 change: 1 addition & 0 deletions upcoming_changes/39.bugfix.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
The backend no longer aborts at exit on Python 3.13 when the app keeps writing to it after asking it to quit.
Loading