Use disk-backed isolated hosts for packaged upgrade checks

Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>

Author: StellaHuang95

build/releaseCompatibility/README.md

@@ -42,9 +42,9 @@ For the DevDiv cutover, supply `upgradeFromEnvironments` with the same artifact
 
 Each boolean mode gets its own workspace, user-data directory, home, extension directory, and VS Code process. Settings are written **before host launch**. Two real `venv --without-pip` fixtures are created as direct workspace children, within the legacy locator's two-directory search depth, and their actual executable/prefix verified. The controller requires discovery of both fixtures before selecting each and invoking normal `python.execInTerminal`; observed candidates are retained even on failure. The script atomically writes a nonce, `sys.executable`, and `sys.prefix`; a terminal appearing or a command resolving is not a pass.
 
-Only the independent test controller is an `extensionDevelopmentPath`. Its own public `ExtensionContext` must report test mode. Products must load from the exact installed VSIX directories, outside that sole development root, with the expected identities/versions. The public `Extension` object does not expose another extension's context mode; it remains unknown rather than fabricating a `Production` value. Installed-path and controller-only launch evidence prove these are not source/development-extension runs. A transparent, restored observer of the public Environments `runInTerminal` method proves core's on-mode delegation without replacing execution. Prototype methods are shadowed only on the instance and restored by removing the shadow; shared prototypes are never modified. An immutable API that cannot be observed is an explicit infrastructure error, not a pass.
+Only the independent test controller is an `extensionDevelopmentPath`. Its own public `ExtensionContext` must report test mode for single-phase checks, or development mode for real-storage upgrade hosts. Products must load from the exact installed VSIX directories, outside that sole development root, with the expected identities/versions. The public `Extension` object does not expose another extension's context mode; it remains unknown rather than fabricating a `Production` value. Installed-path and controller-only launch evidence prove these are not source/development-extension runs. A transparent, restored observer of the public Environments `runInTerminal` method proves core's on-mode delegation without replacing execution. Prototype methods are shadowed only on the instance and restored by removing the shadow; shared prototypes are never modified. An immutable API that cannot be observed is an explicit infrastructure error, not a pass.
 
-For an upgrade, the baseline host fully exits before replacing only the installed Environments directory. Workspace settings, user profile, and selection storage are retained. The upgraded host must observe the last selected fixture **before any selection write**, and explicit true/false must survive. An unsuccessful baseline cannot certify an upgrade.
+For an upgrade, both phases use normal isolated VS Code hosts **without `--extensionTestsPath`**: [VS Code deliberately uses in-memory storage during extension tests](https://github.com/microsoft/vscode/blob/0870c2a0c7c0564e7631bfed2675573a94ba4455/src/vs/platform/storage/electron-main/storageMainService.ts#L99-L102). The independent controller runs on normal development-extension activation and requests a graceful quit after writing its report. The baseline fully exits before replacing only the installed Environments directory. Workspace settings, user profile, and selection storage are retained. An independent controller-owned global/workspace Memento token must survive the restart; failure is an infrastructure/storage error, not a product association verdict. The upgraded host must observe the last selected fixture **before any product selection write**, and explicit true/false must survive. Neither the probe nor the runner reseeds product selection. An unsuccessful baseline cannot certify an upgrade.
 
 ## Evidence and isolation
 

build/releaseCompatibility/checks.js

@@ -387,6 +387,36 @@ function profileSettings(mode, layout) {
     };
 }
 
+async function storageContinuity(context, token, verifyExisting) {
+    if (typeof token !== 'string' || token.length === 0) {
+        throw new InfrastructureError('Persistent-host storage probe token is missing');
+    }
+    const key = 'pythonReleaseCompatibility.storageContinuity';
+    let global;
+    let workspace;
+    try {
+        if (!verifyExisting) {
+            await context.globalState.update(key, token);
+            await context.workspaceState.update(key, token);
+        }
+        global = context.globalState.get(key);
+        workspace = context.workspaceState.get(key);
+    } catch (error) {
+        throw new InfrastructureError(`Controller storage probe failed: ${error.message}`);
+    }
+    if (global !== token || workspace !== token) {
+        throw new InfrastructureError(
+            'Controller storage did not persist across hosts; product selection is not certified',
+        );
+    }
+    return {
+        kind: 'disk-backed-normal-host',
+        phase: verifyExisting ? 'verified-existing' : 'initialized',
+        globalToken: global,
+        workspaceToken: workspace,
+    };
+}
+
 function verifyPetIdentity(expected, actual, platform = process.platform) {
     check(samePath(expected.path, actual.path, platform), 'Runtime PET came from an unexpected supplier or location');
     check(expected.sha256 === actual.sha256, 'Runtime PET differs from the final VSIX binary');
@@ -496,6 +526,7 @@ module.exports = {
     locatorBackend,
     fixtureLayout,
     profileSettings,
+    storageContinuity,
     verifyPetIdentity,
     observeMethod,
     childEnvironment,

build/releaseCompatibility/checks.test.js

@@ -26,11 +26,12 @@ const {
     locatorBackend,
     fixtureLayout,
     profileSettings,
+    storageContinuity,
     verifyPetIdentity,
     observeMethod,
     childEnvironment,
 } = require('./checks');
-const { run, runProcess } = require('./run');
+const { run, runProcess, hostLaunchArgs } = require('./run');
 const { prepare } = require('./prepare');
 const { pack } = require('./pack');
 const { candidateCore, selfTest } = require('./selfTest');
@@ -314,6 +315,57 @@ describe('Packaged release compatibility guards', () => {
         assert.strictEqual(settings.workspace['extensions.autoUpdate'], undefined);
     });
 
+    it('launches upgrade hosts without extensionTestsPath and preserves the same explicit storage directories', () => {
+        const testCase = {
+            workspace: path.join(directory, 'workspace'),
+            userData: path.join(directory, 'user-data'),
+            extensions: path.join(directory, 'extensions'),
+            persistentStorage: true,
+        };
+        const args = hostLaunchArgs(testCase);
+        assert(!args.some((arg) => arg.startsWith('--extensionTestsPath')));
+        assert(args.some((arg) => arg.startsWith('--extensionDevelopmentPath=')));
+        assert.strictEqual(args[args.indexOf('--user-data-dir') + 1], testCase.userData);
+        assert.strictEqual(args[args.indexOf('--extensions-dir') + 1], testCase.extensions);
+        assert.strictEqual(args[0], testCase.workspace);
+    });
+
+    it('proves controller Mementos persist before attributing missing product selection to an upgrade', async () => {
+        const global = new Map();
+        const workspace = new Map();
+        const memento = (values) => ({
+            get: (key) => values.get(key),
+            update: async (key, value) => values.set(key, value),
+        });
+        const baseline = { globalState: memento(global), workspaceState: memento(workspace) };
+        await storageContinuity(baseline, 'same-profile-token', false);
+        const upgraded = { globalState: memento(global), workspaceState: memento(workspace) };
+        assert.strictEqual((await storageContinuity(upgraded, 'same-profile-token', true)).phase, 'verified-existing');
+        const resetHost = { globalState: memento(new Map()), workspaceState: memento(new Map()) };
+        await assert.rejects(storageContinuity(resetHost, 'same-profile-token', true), InfrastructureError);
+        assert.strictEqual(resetHost.globalState.get('pythonReleaseCompatibility.storageContinuity'), undefined);
+        await assert.rejects(storageContinuity(upgraded, 'wrong-profile-token', true), InfrastructureError);
+        const readOnlyAfter = {
+            globalState: {
+                get: () => 'same-profile-token',
+                update: () => assert.fail('Must not reseed after restart'),
+            },
+            workspaceState: {
+                get: () => 'same-profile-token',
+                update: () => assert.fail('Must not reseed after restart'),
+            },
+        };
+        await storageContinuity(readOnlyAfter, 'same-profile-token', true);
+        const failedStore = {
+            globalState: {
+                update: async () => {
+                    throw new Error('disk unavailable');
+                },
+            },
+        };
+        await assert.rejects(storageContinuity(failedStore, 'same-profile-token', false), InfrastructureError);
+    });
+
     it('accepts positively observed legacy JS discovery only for off mode, without fabricating PET identity', () => {
         const legacy = legacyDiscoveryLines(
             '[info] Searching for custom virtual environments',

build/releaseCompatibility/controller/index.js

@@ -13,13 +13,27 @@ const {
     verifyInstalledExtension,
     verifyPythonResult,
     observeMethod,
+    storageContinuity,
 } = require('../checks');
 
 const TIMEOUT = 60000;
 let controllerContext;
 
 function activate(context) {
     controllerContext = context;
+    if (process.env.PYTHON_COMPATIBILITY_PERSISTENT_HOST === '1') {
+        setTimeout(() => {
+            run()
+                .then(
+                    () => vscode.commands.executeCommand('workbench.action.quit'),
+                    (error) => {
+                        console.error('Persistent compatibility case failed', error);
+                        return vscode.commands.executeCommand('workbench.action.quit');
+                    },
+                )
+                .catch((error) => console.error('Persistent compatibility host could not quit gracefully', error));
+        }, 0);
+    }
 }
 
 async function bounded(label, operation) {
@@ -157,8 +171,9 @@ async function run() {
         check(controller, 'Independent test controller is missing');
         await bounded('controller activation', controller.activate());
         check(
-            controllerContext?.extensionMode === vscode.ExtensionMode.Test,
-            'Controller is not in the isolated test host',
+            controllerContext?.extensionMode ===
+                (testCase.persistentStorage ? vscode.ExtensionMode.Development : vscode.ExtensionMode.Test),
+            'Controller is not in the expected isolated host mode',
         );
         check(
             samePath(controllerContext.extensionPath, __dirname),
@@ -167,7 +182,15 @@ async function run() {
         report.controller = {
             location: controllerContext.extensionPath,
             extensionMode: controllerContext.extensionMode,
+            persistentStorage: Boolean(testCase.persistentStorage),
         };
+        if (testCase.persistentStorage) {
+            report.storage = await storageContinuity(
+                controllerContext,
+                testCase.storageProbeToken,
+                Boolean(testCase.persisted),
+            );
+        }
         report.vscode = { version: vscode.version, platform: process.platform, arch: process.arch };
         report.configurationBefore = configuredMode(testCase);
         const core = extension(testCase.core);

build/releaseCompatibility/run.js

@@ -307,7 +307,16 @@ async function makeProfile(directory, mode, pythonExecutable) {
         check(samePath(fixture.prefix, prefix), 'Fixture bootstrap did not create an isolated venv');
         fixtures.push(fixture);
     }
-    return { directory, workspace, userData, home, extensions, env, fixtures };
+    return {
+        directory,
+        workspace,
+        userData,
+        home,
+        extensions,
+        env,
+        fixtures,
+        storageProbeToken: crypto.randomBytes(16).toString('hex'),
+    };
 }
 
 async function runPhase(plan, profile, core, environments, phase, vscodeExecutablePath, persisted) {
@@ -320,7 +329,7 @@ async function runPhase(plan, profile, core, environments, phase, vscodeExecutab
     const caseFile = path.join(phaseDirectory, 'case.json');
     const logDirectory = path.join(profile.userData, 'logs');
     const offsets = logOffsets(logDirectory);
-    writeJson(caseFile, {
+    const testCase = {
         mode: profile.mode,
         phase,
         workspace: profile.workspace,
@@ -329,19 +338,29 @@ async function runPhase(plan, profile, core, environments, phase, vscodeExecutab
         environments: { id: environments.id, version: environments.version, location: environmentLocation },
         fixtures: profile.fixtures,
         persisted,
+        persistentStorage: Boolean(plan.upgradeFromEnvironments),
+        storageProbeToken: profile.storageProbeToken,
         vscodeVersion: plan.vscodeVersion,
         nonce: crypto.randomBytes(16).toString('hex'),
         vscodeExecutablePath,
         userData: profile.userData,
         extensions: profile.extensions,
-    });
+    };
+    writeJson(caseFile, testCase);
     try {
-        const result = await runProcess(process.execPath, [__filename, '--worker', caseFile], {
-            cwd: phaseDirectory,
-            env: profile.env,
-            timeout: HOST_TIMEOUT,
-            log: path.join(phaseDirectory, 'process.log'),
-        });
+        const persistent = testCase.persistentStorage;
+        const result = await runProcess(
+            persistent ? vscodeExecutablePath : process.execPath,
+            persistent ? hostLaunchArgs(testCase) : [__filename, '--worker', caseFile],
+            {
+                cwd: phaseDirectory,
+                env: persistent
+                    ? { ...profile.env, PYTHON_COMPATIBILITY_CASE: caseFile, PYTHON_COMPATIBILITY_PERSISTENT_HOST: '1' }
+                    : profile.env,
+                timeout: HOST_TIMEOUT,
+                log: path.join(phaseDirectory, 'process.log'),
+            },
+        );
         record.process = { pid: result.pid, exitCode: result.code, signal: result.signal, cleanup: result.cleanup };
         if (fs.existsSync(stateFile)) record.host = JSON.parse(fs.readFileSync(stateFile, 'utf8'));
         if (record.host?.status === 'failed') throw new CheckFailure(record.host.error.message);
@@ -412,28 +431,41 @@ async function runPhase(plan, profile, core, environments, phase, vscodeExecutab
     return record;
 }
 
+function hostLaunchArgs(testCase) {
+    return [
+        testCase.workspace,
+        '--extensions-dir',
+        testCase.extensions,
+        '--user-data-dir',
+        testCase.userData,
+        '--disable-workspace-trust',
+        '--skip-welcome',
+        '--skip-release-notes',
+        '--disable-updates',
+        '--enable-proposed-api=ms-python.python',
+        '--enable-proposed-api=ms-python.vscode-python-envs',
+        ...(process.platform === 'linux' ? ['--no-sandbox'] : []),
+        ...(testCase.persistentStorage
+            ? ['--new-window', `--extensionDevelopmentPath=${path.join(__dirname, 'controller')}`]
+            : []),
+    ];
+}
+
 async function worker(caseFile) {
     const testCase = JSON.parse(fs.readFileSync(caseFile, 'utf8'));
+    if (testCase.persistentStorage) {
+        throw new InfrastructureError(
+            'Persistent cases must use the directly owned normal host, not extensionTestsPath',
+        );
+    }
+    const launchArgs = hostLaunchArgs(testCase);
     const { runTests } = require('@vscode/test-electron');
     await runTests({
         vscodeExecutablePath: testCase.vscodeExecutablePath,
         extensionDevelopmentPath: path.join(__dirname, 'controller'),
         extensionTestsPath: path.join(__dirname, 'controller', 'index.js'),
         extensionTestsEnv: { PYTHON_COMPATIBILITY_CASE: caseFile },
-        launchArgs: [
-            testCase.workspace,
-            '--extensions-dir',
-            testCase.extensions,
-            '--user-data-dir',
-            testCase.userData,
-            '--disable-workspace-trust',
-            '--skip-welcome',
-            '--skip-release-notes',
-            '--disable-updates',
-            '--enable-proposed-api=ms-python.python',
-            '--enable-proposed-api=ms-python.vscode-python-envs',
-            ...(process.platform === 'linux' ? ['--no-sandbox'] : []),
-        ],
+        launchArgs,
     });
 }
 
@@ -559,4 +591,4 @@ if (require.main === module) {
     });
 }
 
-module.exports = { run, runProcess, observeLogs, errorRecord };
+module.exports = { run, runProcess, observeLogs, errorRecord, hostLaunchArgs };