Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion de_shell/js/main/backendProcess.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { spawn, spawnSync, ChildProcess } from 'child_process'
import process from 'process'
// Extension spelled out so node:test can load this module without a bundler
// (native type-stripping resolves relative imports literally).
import { withoutPythonRedirects } from './childEnv.ts'
import { shellConfig } from './config.ts'
import { createStdoutDemux } from './stdoutDemux.ts'

Expand Down Expand Up @@ -79,7 +80,7 @@ export function startBackend(
// ~200 ms/frame base64+JSON+atob transport on a 4k movie. Set to "0" to force
// the base64 fallback.
env: {
...process.env, PYTHONUNBUFFERED: '1',
...withoutPythonRedirects(process.env), PYTHONUNBUFFERED: '1',
APL_BINARY_TRANSPORT: process.env.APL_BINARY_TRANSPORT ?? '1',
},
stdio: ['pipe', 'pipe', 'pipe'],
Expand Down
18 changes: 18 additions & 0 deletions de_shell/js/main/childEnv.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
/**
* childEnv.test.ts — the variables that redirect Python never reach a child.
*/
import { test } from 'node:test'
import assert from 'node:assert/strict'
import { withoutPythonRedirects } from './childEnv.ts'

test('PYTHONPATH and PYTHONHOME are removed, in any case', () => {
const env = withoutPythonRedirects({
PYTHONPATH: 'shadow', PythonHome: 'elsewhere', pythonpath: 'shadow', Path: 'C:\bin',
})
assert.deepEqual(env, { Path: 'C:\bin' })
})

test('other Python variables a user sets on purpose are kept', () => {
const kept = { PYTHONFAULTHANDLER: '1', PYTHONUTF8: '1', PYTHONUNBUFFERED: '1', PYTHONPATHS: 'x' }
assert.deepEqual(withoutPythonRedirects(kept), kept)
})
25 changes: 25 additions & 0 deletions de_shell/js/main/childEnv.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/**
* childEnv.ts — the environment the shell passes to the processes it starts.
*/

/**
* Variables that point Python somewhere other than the app's own environment.
* Scientific workstations often set them for another Python: `PYTHONPATH` puts
* a module of the same name ahead of the app's pinned dependency, and a foreign
* `PYTHONHOME` stops the interpreter before any app code runs, with an error
* that names neither. Windows variable names are case-insensitive.
*/
const PYTHON_REDIRECTS = /^PYTHON(PATH|HOME)$/i

/**
* `env` without the variables that redirect Python.
*
* Removed from the environment rather than ignored with `python -E`: `-E`
* would also drop the `PYTHONUNBUFFERED` the stdout protocol needs and any
* variable a user sets on purpose (`PYTHONFAULTHANDLER`, `PYTHONUTF8`), and it
* covers only that one interpreter, while every worker process the backend
* starts inherits this environment.
*/
export function withoutPythonRedirects(env: NodeJS.ProcessEnv): NodeJS.ProcessEnv {
return Object.fromEntries(Object.entries(env).filter(([name]) => !PYTHON_REDIRECTS.test(name)))
}
47 changes: 46 additions & 1 deletion de_shell/js/main/pythonEnv.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,7 @@ import {
findUv, resolvePythonEnv, hasNvidiaGpu, readWheelhouse, findWheelhouse,
wheelhouseCandidates, lockStamp, buildSyncArgs, offlinePlan, effectiveExtras,
gpuEnvOverride, platformMatches, fileUrl, checkWheelhouseExtras, venvPython, clearBrokenEnv,
withoutPythonDownloadLines, uvNetworkEnv,
withoutPythonDownloadLines, uvNetworkEnv, envStamp, stagedWheelsDigest,
} from './pythonEnv.ts'

configureShell({
Expand Down Expand Up @@ -430,3 +430,48 @@ test('the building step names the configured app, not SpyDE', () => {
const event = parseUvLine('Building test-app @ file:///C:/app')
assert.equal(event?.step, 'Building the Test App package')
})

// ── the staged wheels are part of the env stamp ─────────────────────────────

test('envStamp is the lock stamp when no wheels are staged', () => {
assert.equal(envStamp('abc', ['gpu'], null), lockStamp('abc', ['gpu']))
})

test('envStamp appends the wheels digest', () => {
assert.equal(envStamp('abc', [], 'd1'), 'abc+wheels=d1')
})

test('stagedWheelsDigest changes when a wheel is rebuilt under the same name', () => {
const projectDir = tempDir('wheels-')
mkdirSync(join(projectDir, 'wheels'))
const wheel = join(projectDir, 'wheels', 'testapp-1.0.0-py3-none-any.whl')
writeFileSync(wheel, 'first build')
const first = stagedWheelsDigest(projectDir)
writeFileSync(wheel, 'rebuilt with a fix')
assert.notEqual(stagedWheelsDigest(projectDir), first)
assert.match(first ?? '', /^[0-9a-f]{64}$/)
})

test('stagedWheelsDigest is null with no wheels folder or no wheels in it', () => {
const projectDir = tempDir('no-wheels-')
assert.equal(stagedWheelsDigest(projectDir), null)
mkdirSync(join(projectDir, 'wheels'))
writeFileSync(join(projectDir, 'wheels', 'notes.txt'), '')
assert.equal(stagedWheelsDigest(projectDir), null)
})

test('packaged resolvePythonEnv accepts an env stamped with the same staged wheels', async () => {
const { resourcesPath, userData, envDir } = packagedLayout()
const projectDir = join(resourcesPath, 'python')
mkdirSync(join(projectDir, 'wheels'))
writeFileSync(join(projectDir, 'wheels', 'testapp-1.0.0-py3-none-any.whl'), 'build')
const lockHash = createHash('sha256').update(Buffer.from('version = 1\n')).digest('hex')
writeFileSync(join(envDir, '.testapp-lock-hash'),
envStamp(lockHash, [], stagedWheelsDigest(projectDir)))

const resolved = await resolvePythonEnv({
isPackaged: true, resourcesPath, projectRoot: '/unused', userData,
})

assert.deepEqual(resolved.cmd, [venvPython(envDir), '-m', 'testapp'])
})
41 changes: 37 additions & 4 deletions de_shell/js/main/pythonEnv.ts
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ import { createHash } from 'crypto'
import { dirname, join } from 'path'
// Extension spelled out so node:test can load this module without a bundler
// (native type-stripping resolves relative imports literally).
import { withoutPythonRedirects } from './childEnv.ts'
import { envPrefix, shellConfig } from './config.ts'

export interface ResolvedPython {
Expand Down Expand Up @@ -232,6 +233,35 @@ export function lockStamp(lockHash: string, extras: string[]): string {
return extras.length ? `${lockHash}+extras=${[...extras].sort().join(',')}` : lockHash
}

/**
* The environment stamp: {@link lockStamp} plus a digest of the staged wheels.
*
* The app's own wheel and its workspace members reach the env only through
* `installAppWheel`, which runs only when the stamp changes. `uv.lock` records
* versions, so a rebuild that leaves them alone (a fixed tag re-run, a member
* whose version did not move) used to put new wheels on disk and keep running
* the old ones. Null `wheelsDigest` (no staged wheels) keeps the bare stamp.
*/
export function envStamp(lockHash: string, extras: string[], wheelsDigest: string | null): string {
const stamp = lockStamp(lockHash, extras)
return wheelsDigest ? `${stamp}+wheels=${wheelsDigest}` : stamp
}

/** sha256 over every staged wheel's name and bytes, or null when none are staged. */
export function stagedWheelsDigest(projectDir: string): string | null {
const dir = join(projectDir, 'wheels')
let wheels: string[]
try {
wheels = readdirSync(dir).filter((f) => f.endsWith('.whl')).sort()
} catch { return null }
if (wheels.length === 0) return null
const hash = createHash('sha256')
for (const wheel of wheels) {
hash.update(wheel).update('').update(readFileSync(join(dir, wheel))).update('')
}
return hash.digest('hex')
}

/** `['--extra','a','--extra','b']` for `uv sync` / `uv export`. */
export function extraArgs(extras: string[]): string[] {
return extras.flatMap((e) => ['--extra', e])
Expand Down Expand Up @@ -417,6 +447,7 @@ export async function resolvePythonEnv(opts: EnsureOptions): Promise<ResolvedPyt
const pythonExe = venvPython(envDir)
const stampFile = join(envDir, `.${cfg.appId}-lock-hash`)
const lockHash = createHash('sha256').update(readFileSync(bundledLock)).digest('hex')
const wheelsDigest = stagedWheelsDigest(bundledProject)

// GPU decision: option > <APPID>_GPU > nvidia-smi probe. The probe only runs
// when the app declared gpuExtras — otherwise there is nothing to decide. An
Expand All @@ -427,13 +458,13 @@ export async function resolvePythonEnv(opts: EnsureOptions): Promise<ResolvedPyt
const gpu = gpuExtras.length === 0 ? false
: (gpuExplicit ?? await hasNvidiaGpu())
let extras = effectiveExtras(cfg.pythonExtras, gpuExtras, gpu)
let stamp = lockStamp(lockHash, extras)
let stamp = envStamp(lockHash, extras, wheelsDigest)
// The stamp written when a probed GPU was dropped because the wheelhouse was
// built without the GPU extras. Accepting it below is what stops an
// air-gapped GPU machine from re-syncing — online, so failing — on every
// launch; a new lock, or an explicit <APPID>_GPU=1, re-evaluates.
const cpuOnly = effectiveExtras(cfg.pythonExtras, gpuExtras, false)
const gpuUnavailableStamp = `${lockStamp(lockHash, cpuOnly)}+gpu=unavailable`
const gpuUnavailableStamp = `${envStamp(lockHash, cpuOnly, wheelsDigest)}+gpu=unavailable`

// Skip the sync if the venv already matches the shipped lock AND extras.
const onDisk = existsSync(stampFile) ? readSafe(stampFile) : null
Expand Down Expand Up @@ -565,7 +596,7 @@ function uvEnv(
: basePath

return {
...process.env,
...withoutPythonRedirects(process.env),
...scmEnv,
[pathKey]: mergedPath,
// UV_PROJECT_ENVIRONMENT redirects `uv sync`'s venv out of the read-only
Expand Down Expand Up @@ -785,7 +816,9 @@ function installAppWheel(
)
return runUv(
projectDir, envDir,
['pip', 'install', ...(offlineEnv ? ['--offline'] : []), '--no-deps',
// --reinstall: a rebuild can keep the version, and uv skips a wheel whose
// name and version are already installed. --no-deps keeps it to this wheel.
['pip', 'install', ...(offlineEnv ? ['--offline'] : []), '--no-deps', '--reinstall',
'--python', venvPython(envDir), wheel],
'', onProgress, offlineEnv,
)
Expand Down
47 changes: 41 additions & 6 deletions de_shell/session.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@
import json
import logging
import os
import time
from typing import Any

log = logging.getLogger(__name__)
Expand Down Expand Up @@ -138,16 +139,50 @@ def controller_by_window_id(self, window_id: int | None):
# ── Settings & recent files ───────────────────────────────────────────────

def _load_settings(self) -> dict:
"""The saved settings, or ``{}`` when there are none or they are unreadable.

An unreadable file is moved aside to ``settings.json.bad-<time>`` rather
than left in place: the next save writes the defaults, and that must not
destroy a file that only has a stray comma from a hand edit.
"""
try:
with open(self._settings_path, encoding="utf-8") as fh:
return json.load(fh)
except Exception:
settings = json.load(fh)
except FileNotFoundError:
return {}
except (OSError, ValueError) as e:
self._set_aside_unreadable_settings(e)
return {}
if not isinstance(settings, dict):
self._set_aside_unreadable_settings(TypeError("not a JSON object"))
return {}
return settings

def _set_aside_unreadable_settings(self, error: Exception) -> None:
aside = f"{self._settings_path}.bad-{time.strftime('%Y%m%d-%H%M%S')}"
try:
os.replace(self._settings_path, aside)
except OSError as e:
log.warning("settings file %s is unreadable (%s) and could not be moved aside: %s; "
"using defaults", self._settings_path, error, e)
return
log.warning("settings file %s is unreadable (%s); using defaults and keeping it as %s",
self._settings_path, error, aside)

def _save_settings(self) -> None:
"""Write the settings atomically: to a temporary file, then rename over.

Writing in place truncates first, so a backend killed mid-write (the
shell force-ends one that does not quit within 5 s) left an empty or
half-written file and lost every setting.
"""
os.makedirs(os.path.dirname(self._settings_path), exist_ok=True)
with open(self._settings_path, "w", encoding="utf-8") as fh:
temporary = f"{self._settings_path}.tmp"
with open(temporary, "w", encoding="utf-8") as fh:
json.dump(self._settings, fh, indent=2)
fh.flush()
os.fsync(fh.fileno())
os.replace(temporary, self._settings_path)

def _add_recent(self, path: str) -> None:
if path in self._recent_files:
Expand All @@ -157,7 +192,7 @@ def _add_recent(self, path: str) -> None:
try:
self._save_settings()
except Exception as e:
log.debug("saving recent-files settings failed: %s", e)
log.warning("saving recent-files settings failed: %s", e)

def get_recent_files(self) -> list[str]:
return list(self._recent_files[:self.MAX_RECENT])
Expand All @@ -177,7 +212,7 @@ def set_update_channel(self, channel: str) -> None:
try:
self._save_settings()
except Exception as e:
log.debug("saving update_channel setting failed: %s", e)
log.warning("saving update_channel setting failed: %s", e)

# ── First-run welcome tour ────────────────────────────────────────────────

Expand All @@ -196,7 +231,7 @@ def mark_tutorial_seen(self) -> None:
try:
self._save_settings()
except Exception as e:
log.debug("saving tutorial_seen setting failed: %s", e)
log.warning("saving tutorial_seen setting failed: %s", e)

# ── Shutdown ──────────────────────────────────────────────────────────────

Expand Down
32 changes: 32 additions & 0 deletions tests/test_session_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,38 @@ def test_corrupt_settings_file_is_not_an_error(self, tmp_path):
s = SessionBase(settings_dir=str(tmp_path))
assert s._settings == {}

def test_an_unreadable_file_is_kept_aside_not_overwritten(self, tmp_path, caplog):
hand_edited = '{"update_channel": "beta", "recent_files": ["a.dat"],}'
(tmp_path / "settings.json").write_text(hand_edited, encoding="utf-8")
s = SessionBase(settings_dir=str(tmp_path))
s.mark_tutorial_seen()
kept = list(tmp_path.glob("settings.json.bad-*"))
assert len(kept) == 1
assert kept[0].read_text(encoding="utf-8") == hand_edited
assert "unreadable" in caplog.text

def test_a_json_value_that_is_not_an_object_is_kept_aside(self, tmp_path):
(tmp_path / "settings.json").write_text("[1, 2]", encoding="utf-8")
assert SessionBase(settings_dir=str(tmp_path))._settings == {}
assert len(list(tmp_path.glob("settings.json.bad-*"))) == 1

def test_a_save_leaves_no_temporary_file(self, tmp_path):
s = SessionBase(settings_dir=str(tmp_path))
s._add_recent("/a")
assert sorted(p.name for p in tmp_path.iterdir()) == ["settings.json"]

def test_a_save_that_fails_midway_keeps_the_previous_file(self, tmp_path, monkeypatch):
s = SessionBase(settings_dir=str(tmp_path))
s._add_recent("/a")

def fail(*args, **kwargs):
raise OSError("killed mid-write")

monkeypatch.setattr(json, "dump", fail)
s._add_recent("/b")
on_disk = json.loads((tmp_path / "settings.json").read_text(encoding="utf-8"))
assert on_disk["recent_files"] == ["/a"]

def test_recent_files_round_trip_and_are_most_recent_first(self, tmp_path):
s = SessionBase(settings_dir=str(tmp_path))
s._add_recent("/a")
Expand Down
1 change: 1 addition & 0 deletions upcoming_changes/38.bugfix.1.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
A damaged ``settings.json`` is kept as ``settings.json.bad-<time>`` instead of being silently replaced with defaults, and saves can no longer leave a half-written file.
1 change: 1 addition & 0 deletions upcoming_changes/38.bugfix.2.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
A ``PYTHONPATH`` or ``PYTHONHOME`` set for another Python no longer reaches the app's backend or its environment setup.
1 change: 1 addition & 0 deletions upcoming_changes/38.bugfix.rst
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
An update rebuilt without a version change now installs its new Python code; before, the app kept running the previous build's code until the lock file changed.
Loading