feat(e2e): verify the migration Oracle with reference and broken applications

e2e/migration-harness/__tests__/migration-live.test.ts

@@ -0,0 +1,113 @@
+import 'dotenv/config'
+import assert from 'node:assert/strict'
+import { mkdtemp, rm } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { resolve } from 'node:path'
+import { fileURLToPath } from 'node:url'
+import { test } from 'vitest'
+import { createIsolation } from '../isolation.mjs'
+import { createRealDriver } from '../run/real-driver.mjs'
+import { run } from '../run/run.mjs'
+
+const keys = [
+  'CS_WORKSPACE_CRN',
+  'CS_CLIENT_ID',
+  'CS_CLIENT_KEY',
+  'CS_CLIENT_ACCESS_KEY',
+]
+const enabled =
+  process.env.HARNESS_LIVE === '1' && keys.every((key) => process.env[key])
+const cases = [
+  ['reference', 'pass', null],
+  ['noop', 'behaviour', 'migration.protect-dependencies-absent'],
+  ['downcase', 'behaviour', 'behaviour.downcase-new-write-lookup'],
+  ['stale-hmac', 'behaviour', 'migration.active-hmac-absent'],
+  ['bigint', 'behaviour', 'behaviour.balance-decimal-string'],
+  ['rewrite', 'behaviour', 'no-rewrite.legacy-bytes'],
+  ['missing-log', 'invalid-friction-log', null],
+  ['legacy-ffi', 'behaviour', 'migration.protect-ffi-version'],
+  ['denied-app', 'pass', null],
+  ['denied-lifecycle', 'pass', null],
+  ['tree', 'pass', null],
+] as const
+for (const [kind, outcome, checkId] of cases) {
+  test.skipIf(!enabled)(
+    `real Run self-test: ${kind}`,
+    async () => {
+      const output = await mkdtemp(resolve(tmpdir(), 'mh-selftest-'))
+      const isolation = createIsolation()
+      const manifest = {
+        stack: kind === 'tree' ? 'tree' : '^1.2.1',
+        guideHash: null,
+      }
+      const driver = createRealDriver({
+        id: `selftest-${Date.now()}`,
+        root: fileURLToPath(new URL('../../..', import.meta.url)),
+        output,
+        command: '',
+        fakeMigration: kind === 'tree' ? 'reference' : kind,
+        isolation,
+        credentials: Object.fromEntries(
+          keys.map((key) => [key, process.env[key] ?? '']),
+        ),
+        manifest,
+      })
+      try {
+        const report = await run({
+          id: kind,
+          manifest,
+          driver,
+          supervision: isolation,
+        })
+        const diagnostics = JSON.stringify({
+          outcome: report.outcome,
+          failures: report.failures,
+          checks: report.evidence
+            .filter(
+              (e) =>
+                e.kind === 'oracle-check' && Object(e.payload).passed === false,
+            )
+            .map((e) => e.payload),
+        })
+        assert.equal(report.outcome, outcome, diagnostics)
+        if (kind.startsWith('denied-'))
+          assert.ok(
+            report.evidence.some(
+              (e) =>
+                e.kind === 'isolation-event' &&
+                Object(e.payload).result === 'denied' &&
+                Object(e.payload).decision === 'forbidden' &&
+                Object(e.payload).resource === '/trace/canary',
+            ),
+          )
+        if (checkId)
+          assert.equal(
+            Object(report.evidence.find((e) => e.checkId === checkId)?.payload)
+              .passed,
+            false,
+            diagnostics,
+          )
+        if (kind === 'noop')
+          assert.ok(
+            report.evidence
+              .filter(
+                (e) =>
+                  e.kind === 'oracle-check' &&
+                  !e.checkId.startsWith('migration.'),
+              )
+              .every((e) => Object(e.payload).passed),
+          )
+        if (kind === 'reference')
+          assert.ok(
+            report.evidence.some(
+              (e) => e.checkId === 'no-rewrite.legacy-bytes',
+            ),
+          )
+      } finally {
+        await driver.close()
+        await rm(output, { recursive: true, force: true })
+      }
+    },
+    600_000,
+  )
+}

e2e/migration-harness/__tests__/migration.test.ts

@@ -0,0 +1,77 @@
+import assert from 'node:assert/strict'
+import { mkdir, mkdtemp, rm, writeFile } from 'node:fs/promises'
+import { tmpdir } from 'node:os'
+import { join } from 'node:path'
+import { test } from 'vitest'
+import { migrationChecks } from '../run/migration-checks.mjs'
+import { run } from '../run/run.mjs'
+
+for (const stale of [false, true]) {
+  test(`Run migration evidence ${stale ? 'rejects stale' : 'accepts removed'} boolean HMAC`, async () => {
+    const workspace = await mkdtemp(join(tmpdir(), 'mh-migration-'))
+    await mkdir(join(workspace, 'src'))
+    await writeFile(
+      join(workspace, 'src/app.mjs'),
+      'import { Encryption } from "@cipherstash/stack/v3"',
+    )
+    await writeFile(
+      join(workspace, 'package.json'),
+      JSON.stringify({ dependencies: { '@cipherstash/stack': '1.2.1' } }),
+    )
+    await writeFile(
+      join(workspace, 'package-lock.json'),
+      JSON.stringify({
+        packages: {
+          '': { dependencies: { '@cipherstash/stack': '1.2.1' } },
+          'node_modules/@cipherstash/stack': {
+            version: '1.2.1',
+            dependencies: { '@cipherstash/protect-ffi': '0.31.0' },
+          },
+          'node_modules/@cipherstash/protect-ffi': { version: '0.31.0' },
+        },
+      }),
+    )
+    try {
+      const report = await run({
+        id: 'migration',
+        manifest: { guideHash: 'guide' },
+        driver: {
+          provision: async () => {},
+          installUnmigrated: async () => {},
+          initializeLegacyData: async () => {},
+          snapshot: async () => {},
+          migrate: async () => ({ exitCode: 0 }),
+          friction: async () => [],
+          installAppHost: async () => {},
+          behaviour: async () => [
+            { id: 'no-rewrite.legacy-bytes', passed: true },
+            ...(await migrationChecks({
+              workspace,
+              installedWorkspace: workspace,
+              ffiRange: '0.31.0',
+              mutationSnapshot: {
+                dynamo: {
+                  Item: stale
+                    ? { active__source: 'cipher', active__hmac: 'old' }
+                    : { active__source: 'cipher' },
+                },
+              },
+              stackVersion: '1.2.1',
+            })),
+          ],
+        },
+      })
+      assert.equal(report.outcome, stale ? 'behaviour' : 'pass')
+      assert.equal(
+        Object(
+          report.evidence.find(
+            (e) => e.checkId === 'migration.active-hmac-absent',
+          )?.payload,
+        ).passed,
+        !stale,
+      )
+    } finally {
+      await rm(workspace, { recursive: true, force: true })
+    }
+  })
+}

e2e/migration-harness/__tests__/review-driver.test.ts

@@ -397,6 +397,51 @@ test.each([
   },
 )
 
+test.each([
+  [Object.assign(Error('missing table'), { code: '42P01' }), 'behaviour'],
+  [
+    Object.assign(Error('missing DynamoDB table'), {
+      name: 'ResourceNotFoundException',
+    }),
+    'behaviour',
+  ],
+  [
+    Object.assign(Error('connection lost'), { code: '08006' }),
+    'infrastructure',
+  ],
+] as const)(
+  'Mutation Data inspection classifies %s',
+  async (error, outcome) => {
+    const { driver, calls } = await setup()
+    const baseline = await observed.snapshot()
+    observed.snapshot
+      .mockResolvedValueOnce(baseline)
+      .mockResolvedValueOnce(baseline)
+      .mockRejectedValueOnce(error)
+    try {
+      const report = await run({
+        id: 'mutation-snapshot',
+        manifest: { guideHash: 'guide' },
+        driver: {
+          ...driver,
+          provision: async () => {},
+          migrate: async () => ({ exitCode: 0 }),
+          friction: async () => [],
+        },
+      })
+      expect(report.outcome).toBe(outcome)
+      expect(report.editorEligible).toBe(outcome === 'behaviour')
+      expect(observed.snapshot).toHaveBeenLastCalledWith('mutation')
+      expect(calls).toContain('setActive')
+      expect(calls.slice(calls.lastIndexOf('setActive'))).not.toContain(
+        'copyToPostgres',
+      )
+    } finally {
+      await driver.close()
+    }
+  },
+)
+
 test('F12 disposes stores even when the isolation supervisor fails to stop', async () => {
   const { driver } = await setup({ stopFails: true })
   await expect(driver.close()).rejects.toThrow('supervisor stop failed')

e2e/migration-harness/__tests__/stack-source.test.ts

@@ -0,0 +1,46 @@
+import { afterEach, expect, test, vi } from 'vitest'
+
+const npmView = vi.hoisted(() => new Map<string, string>())
+
+vi.mock('node:child_process', () => ({
+  execFile: (
+    _command: string,
+    args: string[],
+    callback: (error: Error | null, result?: { stdout: string }) => void,
+  ) => {
+    const stdout = npmView.get(args.slice(1, 3).join(' '))
+    if (stdout === undefined) callback(Error(`unexpected npm ${args}`))
+    else callback(null, { stdout })
+  },
+}))
+
+const { resolveStack } = await import('../run/stack-source.mjs')
+
+afterEach(() => npmView.clear())
+
+// npm 12 answers `npm view <spec> <field> --json` with an array even when the
+// spec names one exact version; older npm answers with the bare value.
+test.each([
+  ['npm 10 bare values', '"1.2.1"', '"0.33.0"'],
+  ['npm 12 single-element arrays', '["1.2.1"]', '["0.33.0"]'],
+])('resolves a registry Stack from %s', async (_npm, version, ffi) => {
+  npmView.set('@cipherstash/stack@^1.2.1 version', version)
+  npmView.set(
+    '@cipherstash/stack@1.2.1 dependencies.@cipherstash/protect-ffi',
+    ffi,
+  )
+  await expect(
+    resolveStack({ source: '^1.2.1', root: '/', output: '/' }),
+  ).resolves.toEqual({ version: '1.2.1', install: '1.2.1', ffiRange: '0.33.0' })
+})
+
+test('rejects an ambiguous FFI dependency answer', async () => {
+  npmView.set('@cipherstash/stack@1.2.1 version', '"1.2.1"')
+  npmView.set(
+    '@cipherstash/stack@1.2.1 dependencies.@cipherstash/protect-ffi',
+    '["0.33.0","0.34.0"]',
+  )
+  await expect(
+    resolveStack({ source: '1.2.1', root: '/', output: '/' }),
+  ).rejects.toThrow('Could not resolve Stack FFI dependency')
+})

e2e/migration-harness/isolation/Dockerfile

@@ -1,5 +1,5 @@
 FROM node:22-bookworm-slim
-RUN apt-get update && apt-get install -y --no-install-recommends strace socat python3 util-linux git && rm -rf /var/lib/apt/lists/*
+RUN apt-get update && apt-get install -y --no-install-recommends ca-certificates strace socat python3 util-linux git && rm -rf /var/lib/apt/lists/*
 # A bind mount's root can appear root-owned (Colima virtiofs on macOS) while the
 # actor runs as the host uid; Git would then refuse /workspace as dubious ownership.
 RUN git config --system --add safe.directory /workspace