diff --git a/de_shell/js/main/backendProcess.ts b/de_shell/js/main/backendProcess.ts index 4ef38b0..f56d46e 100644 --- a/de_shell/js/main/backendProcess.ts +++ b/de_shell/js/main/backendProcess.ts @@ -8,6 +8,7 @@ import { spawn, spawnSync, ChildProcess } from 'child_process' import process from 'process' // Extension spelled out so node:test can load this module without a bundler // (native type-stripping resolves relative imports literally). +import { withoutPythonRedirects } from './childEnv.ts' import { shellConfig } from './config.ts' import { createStdoutDemux } from './stdoutDemux.ts' @@ -79,7 +80,7 @@ export function startBackend( // ~200 ms/frame base64+JSON+atob transport on a 4k movie. Set to "0" to force // the base64 fallback. env: { - ...process.env, PYTHONUNBUFFERED: '1', + ...withoutPythonRedirects(process.env), PYTHONUNBUFFERED: '1', APL_BINARY_TRANSPORT: process.env.APL_BINARY_TRANSPORT ?? '1', }, stdio: ['pipe', 'pipe', 'pipe'], diff --git a/de_shell/js/main/childEnv.test.ts b/de_shell/js/main/childEnv.test.ts new file mode 100644 index 0000000..18b39dc --- /dev/null +++ b/de_shell/js/main/childEnv.test.ts @@ -0,0 +1,18 @@ +/** + * childEnv.test.ts — the variables that redirect Python never reach a child. + */ +import { test } from 'node:test' +import assert from 'node:assert/strict' +import { withoutPythonRedirects } from './childEnv.ts' + +test('PYTHONPATH and PYTHONHOME are removed, in any case', () => { + const env = withoutPythonRedirects({ + PYTHONPATH: 'shadow', PythonHome: 'elsewhere', pythonpath: 'shadow', Path: 'C:\bin', + }) + assert.deepEqual(env, { Path: 'C:\bin' }) +}) + +test('other Python variables a user sets on purpose are kept', () => { + const kept = { PYTHONFAULTHANDLER: '1', PYTHONUTF8: '1', PYTHONUNBUFFERED: '1', PYTHONPATHS: 'x' } + assert.deepEqual(withoutPythonRedirects(kept), kept) +}) diff --git a/de_shell/js/main/childEnv.ts b/de_shell/js/main/childEnv.ts new file mode 100644 index 0000000..6b1290f --- /dev/null +++ b/de_shell/js/main/childEnv.ts @@ -0,0 +1,25 @@ +/** + * childEnv.ts — the environment the shell passes to the processes it starts. + */ + +/** + * Variables that point Python somewhere other than the app's own environment. + * Scientific workstations often set them for another Python: `PYTHONPATH` puts + * a module of the same name ahead of the app's pinned dependency, and a foreign + * `PYTHONHOME` stops the interpreter before any app code runs, with an error + * that names neither. Windows variable names are case-insensitive. + */ +const PYTHON_REDIRECTS = /^PYTHON(PATH|HOME)$/i + +/** + * `env` without the variables that redirect Python. + * + * Removed from the environment rather than ignored with `python -E`: `-E` + * would also drop the `PYTHONUNBUFFERED` the stdout protocol needs and any + * variable a user sets on purpose (`PYTHONFAULTHANDLER`, `PYTHONUTF8`), and it + * covers only that one interpreter, while every worker process the backend + * starts inherits this environment. + */ +export function withoutPythonRedirects(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv { + return Object.fromEntries(Object.entries(env).filter(([name]) => !PYTHON_REDIRECTS.test(name))) +} diff --git a/de_shell/js/main/pythonEnv.test.ts b/de_shell/js/main/pythonEnv.test.ts index 67d038b..e17b570 100644 --- a/de_shell/js/main/pythonEnv.test.ts +++ b/de_shell/js/main/pythonEnv.test.ts @@ -30,7 +30,7 @@ import { findUv, resolvePythonEnv, hasNvidiaGpu, readWheelhouse, findWheelhouse, wheelhouseCandidates, lockStamp, buildSyncArgs, offlinePlan, effectiveExtras, gpuEnvOverride, platformMatches, fileUrl, checkWheelhouseExtras, venvPython, clearBrokenEnv, - withoutPythonDownloadLines, uvNetworkEnv, + withoutPythonDownloadLines, uvNetworkEnv, envStamp, stagedWheelsDigest, } from './pythonEnv.ts' configureShell({ @@ -430,3 +430,48 @@ test('the building step names the configured app, not SpyDE', () => { const event = parseUvLine('Building test-app @ file:///C:/app') assert.equal(event?.step, 'Building the Test App package') }) + +// ── the staged wheels are part of the env stamp ───────────────────────────── + +test('envStamp is the lock stamp when no wheels are staged', () => { + assert.equal(envStamp('abc', ['gpu'], null), lockStamp('abc', ['gpu'])) +}) + +test('envStamp appends the wheels digest', () => { + assert.equal(envStamp('abc', [], 'd1'), 'abc+wheels=d1') +}) + +test('stagedWheelsDigest changes when a wheel is rebuilt under the same name', () => { + const projectDir = tempDir('wheels-') + mkdirSync(join(projectDir, 'wheels')) + const wheel = join(projectDir, 'wheels', 'testapp-1.0.0-py3-none-any.whl') + writeFileSync(wheel, 'first build') + const first = stagedWheelsDigest(projectDir) + writeFileSync(wheel, 'rebuilt with a fix') + assert.notEqual(stagedWheelsDigest(projectDir), first) + assert.match(first ?? '', /^[0-9a-f]{64}$/) +}) + +test('stagedWheelsDigest is null with no wheels folder or no wheels in it', () => { + const projectDir = tempDir('no-wheels-') + assert.equal(stagedWheelsDigest(projectDir), null) + mkdirSync(join(projectDir, 'wheels')) + writeFileSync(join(projectDir, 'wheels', 'notes.txt'), '') + assert.equal(stagedWheelsDigest(projectDir), null) +}) + +test('packaged resolvePythonEnv accepts an env stamped with the same staged wheels', async () => { + const { resourcesPath, userData, envDir } = packagedLayout() + const projectDir = join(resourcesPath, 'python') + mkdirSync(join(projectDir, 'wheels')) + writeFileSync(join(projectDir, 'wheels', 'testapp-1.0.0-py3-none-any.whl'), 'build') + const lockHash = createHash('sha256').update(Buffer.from('version = 1\n')).digest('hex') + writeFileSync(join(envDir, '.testapp-lock-hash'), + envStamp(lockHash, [], stagedWheelsDigest(projectDir))) + + const resolved = await resolvePythonEnv({ + isPackaged: true, resourcesPath, projectRoot: '/unused', userData, + }) + + assert.deepEqual(resolved.cmd, [venvPython(envDir), '-m', 'testapp']) +}) diff --git a/de_shell/js/main/pythonEnv.ts b/de_shell/js/main/pythonEnv.ts index 030c221..38cf6eb 100644 --- a/de_shell/js/main/pythonEnv.ts +++ b/de_shell/js/main/pythonEnv.ts @@ -45,6 +45,7 @@ import { createHash } from 'crypto' import { dirname, join } from 'path' // Extension spelled out so node:test can load this module without a bundler // (native type-stripping resolves relative imports literally). +import { withoutPythonRedirects } from './childEnv.ts' import { envPrefix, shellConfig } from './config.ts' export interface ResolvedPython { @@ -232,6 +233,35 @@ export function lockStamp(lockHash: string, extras: string[]): string { return extras.length ? `${lockHash}+extras=${[...extras].sort().join(',')}` : lockHash } +/** + * The environment stamp: {@link lockStamp} plus a digest of the staged wheels. + * + * The app's own wheel and its workspace members reach the env only through + * `installAppWheel`, which runs only when the stamp changes. `uv.lock` records + * versions, so a rebuild that leaves them alone (a fixed tag re-run, a member + * whose version did not move) used to put new wheels on disk and keep running + * the old ones. Null `wheelsDigest` (no staged wheels) keeps the bare stamp. + */ +export function envStamp(lockHash: string, extras: string[], wheelsDigest: string | null): string { + const stamp = lockStamp(lockHash, extras) + return wheelsDigest ? `${stamp}+wheels=${wheelsDigest}` : stamp +} + +/** sha256 over every staged wheel's name and bytes, or null when none are staged. */ +export function stagedWheelsDigest(projectDir: string): string | null { + const dir = join(projectDir, 'wheels') + let wheels: string[] + try { + wheels = readdirSync(dir).filter((f) => f.endsWith('.whl')).sort() + } catch { return null } + if (wheels.length === 0) return null + const hash = createHash('sha256') + for (const wheel of wheels) { + hash.update(wheel).update('').update(readFileSync(join(dir, wheel))).update('') + } + return hash.digest('hex') +} + /** `['--extra','a','--extra','b']` for `uv sync` / `uv export`. */ export function extraArgs(extras: string[]): string[] { return extras.flatMap((e) => ['--extra', e]) @@ -417,6 +447,7 @@ export async function resolvePythonEnv(opts: EnsureOptions): Promise _GPU > nvidia-smi probe. The probe only runs // when the app declared gpuExtras — otherwise there is nothing to decide. An @@ -427,13 +458,13 @@ export async function resolvePythonEnv(opts: EnsureOptions): Promise_GPU=1, re-evaluates. const cpuOnly = effectiveExtras(cfg.pythonExtras, gpuExtras, false) - const gpuUnavailableStamp = `${lockStamp(lockHash, cpuOnly)}+gpu=unavailable` + const gpuUnavailableStamp = `${envStamp(lockHash, cpuOnly, wheelsDigest)}+gpu=unavailable` // Skip the sync if the venv already matches the shipped lock AND extras. const onDisk = existsSync(stampFile) ? readSafe(stampFile) : null @@ -565,7 +596,7 @@ function uvEnv( : basePath return { - ...process.env, + ...withoutPythonRedirects(process.env), ...scmEnv, [pathKey]: mergedPath, // UV_PROJECT_ENVIRONMENT redirects `uv sync`'s venv out of the read-only @@ -785,7 +816,9 @@ function installAppWheel( ) return runUv( projectDir, envDir, - ['pip', 'install', ...(offlineEnv ? ['--offline'] : []), '--no-deps', + // --reinstall: a rebuild can keep the version, and uv skips a wheel whose + // name and version are already installed. --no-deps keeps it to this wheel. + ['pip', 'install', ...(offlineEnv ? ['--offline'] : []), '--no-deps', '--reinstall', '--python', venvPython(envDir), wheel], '', onProgress, offlineEnv, ) diff --git a/de_shell/session.py b/de_shell/session.py index 774276d..3389566 100644 --- a/de_shell/session.py +++ b/de_shell/session.py @@ -27,6 +27,7 @@ import json import logging import os +import time from typing import Any log = logging.getLogger(__name__) @@ -138,16 +139,50 @@ def controller_by_window_id(self, window_id: int | None): # ── Settings & recent files ─────────────────────────────────────────────── def _load_settings(self) -> dict: + """The saved settings, or ``{}`` when there are none or they are unreadable. + + An unreadable file is moved aside to ``settings.json.bad-