From 03b3a6fdacb825b36a9cf4ff89dc4654231f5aa7 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Thu, 29 Oct 2026 19:15:16 +0100 Subject: [PATCH 1/2] fix(flow): prepare 10.0.3 home startup patch release --- .agents/plans/06-patch-release/README.md | 29 ++- .agents/plans/23-home-startup-patch/README.md | 40 +++ .github/workflows/release.yml | 2 +- CHANGELOG.md | 15 ++ README.md | 24 +- docs/quickstart.md | 6 +- docs/release-qualification.md | 14 +- docs/troubleshooting.md | 18 +- evals/qualification/patches/10.0.3.json | 40 +++ package.json | 4 +- scripts/patch-release.ts | 79 +++++- src/infrastructure/fs/workspace-paths.ts | 4 +- src/platform/opencode/plugin-composition.ts | 19 +- tests/distribution-and-surface.test.ts | 49 +++- tests/eval-release-sampling.test.ts | 27 +- tests/patch-release.test.ts | 235 ++++++++++++++++-- tests/plugin-startup-live.test.ts | 215 ++++++++++++++++ tests/qualification-cli.test.ts | 14 +- tests/release-940-preparation.test.ts | 23 ++ 19 files changed, 771 insertions(+), 86 deletions(-) create mode 100644 .agents/plans/23-home-startup-patch/README.md create mode 100644 evals/qualification/patches/10.0.3.json create mode 100644 tests/plugin-startup-live.test.ts diff --git a/.agents/plans/06-patch-release/README.md b/.agents/plans/06-patch-release/README.md index 5a6661eb..fe7c8f28 100644 --- a/.agents/plans/06-patch-release/README.md +++ b/.agents/plans/06-patch-release/README.md @@ -26,6 +26,27 @@ semantic judgment from a path or a patch version. A content change invalidates that approval. This is a maintainer review control, not a sandbox against someone who can rewrite the policy and its approvals together. +## Reviewed home-startup exception + +The operator authorized a release without new model qualification for the +home-startup defect on 2026-10-29. Exact 10.0.3 may use fully qualified 10.0.2's +OpenAI-only baseline. This does not change full-matrix profiles or admit later +runtime patches automatically. + +An optional `startup` approval binds before/after SHA-256 hashes for exactly +`src/infrastructure/fs/workspace-paths.ts` and +`src/platform/opencode/plugin-composition.ts`, both regular non-executable files. +The sole package-script change appends the provider-free startup test to +`smoke:live`. Other metadata, runtime files, guides, dependencies and build inputs +remain protected. Missing approvals, changed content, other baselines/versions, +extra runtime paths or weakened smoke commands fail. + +Review must confirm that only home-directory setup becomes inactive, preserving +mutable-state rejection, project registration and every workflow gate. Linux +packed startup must cover home, Git and non-Git folders without a model call. +The baseline is reverified as retained evidence; the patch has no new model +measurements or canary. Publication still checks exact candidate package bytes. + ## Evidence A record at evals/qualification/patches/VERSION.json uses schemaVersion 1 and kind @@ -34,10 +55,10 @@ manifest hashes, its version, the baseline artifact identity, tag commit, canary path and regraded bundle hash, approvedBy, rationale, and any guidance approval. The PatchReleaseSchema in scripts/patch-release.ts is the executable format. -The baseline must independently pass the full verifier at release time. Existing -seven-day campaign and 72-hour canary limits remain unchanged. A patch record -cannot stand in for a fully qualified baseline, extend freshness, or invent runs. -An expired baseline blocks; it never starts a paid campaign automatically. +The baseline must independently pass the full verifier as retained evidence at +release time. Its prior campaign and canary are historical, not fresh candidate +measurements. A patch record cannot become a fully qualified baseline or invent +runs. Verification never starts a paid campaign. The candidate has no new model eval or live model canary. Required assurance is clean source, strict diff eligibility, matching package bytes, review of the diff --git a/.agents/plans/23-home-startup-patch/README.md b/.agents/plans/23-home-startup-patch/README.md new file mode 100644 index 00000000..7e424cb1 --- /dev/null +++ b/.agents/plans/23-home-startup-patch/README.md @@ -0,0 +1,40 @@ +# Flow 10.0.3 home-startup patch + +The operator requested a new release without new model qualification on +2026-10-29, and explicitly invoked full-stop: prepare an open, reviewed PR; do +not merge or publish from that task. + +The published 10.0.2 package fails plugin setup when OpenCode loads its global +configuration at home. A workspace safety guard is correct for mutation but was +also used as a fatal startup precondition. The patch handles that precise error +as an inactive plugin location with an actionable warning. It does not create +home state or weaken root, missing-directory, validation or review guards. + +## Release assurance + +Use the existing baseline-qualified-patch route with an exact-version reviewed +startup exception. Compare directly against measured source `fdd11b7d` and the +published `v10.0.2` tag at `a920f737`. The patch record binds candidate package +bytes and both changed runtime files. The 10.0.2 matrix (80/81 under unchanged +thresholds) and native canary (6/6) remain historical. No model evaluation or +model canary is measured for 10.0.3. Full-matrix two-provider policy is unchanged. + +The provider-free native regression must reject the published artifact and pass +the patch at home, in a Git project and in a non-Git folder. CI also runs full +checks, replay, package smoke and strict baseline/patch verification. Changes to +approved code require refreshed hashes and review; no skip flag is added. + +Dependabot PRs #213 and #214 remain separate: dependencies and lockfiles stay +identical to the measured baseline for this narrowly reviewed patch. + +## Linux installation after publication + +```sh +opencode2 plugin add opencode-plugin-flow@10.0.3 +cd /path/to/your/project +opencode2 +``` + +An exact existing 10.0.2 pin does not update itself. No software was installed on +the user's Linux laptop by this work. Publication needs a merged commit and its +annotated version tag; the existing workflow owns npm and GitHub publication. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e3acc6e5..caac4f48 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -244,7 +244,7 @@ jobs: "opencode-plugin-flow-${version}.tgz" "$sealed" "$digest" fi fi - if node -e 'process.exit(Number(process.argv[1].split(".")[0]) >= 10 ? 0 : 1)' "$version"; then + if node -e 'process.exit(Number(process.argv[1].split(".")[0]) >= 10 ? 0 : 1)' "$version" && [[ ! -f "evals/qualification/patches/${version}.json" && ! -f "evals/qualification/features/${version}.json" ]]; then qualified="$(bun scripts/materialize-qualification.ts --descriptor "evals/qualification/archives/${version}.json" --out "${RUNNER_TEMP}/flow-qualified-${version}")" bun run scripts/restore-exact-release-artifact.ts \ "opencode-plugin-flow-${version}.tgz" --bundle "$qualified" "$version" diff --git a/CHANGELOG.md b/CHANGELOG.md index 8e89fb62..9bb7b705 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,21 @@ One short entry per release, written for users deciding whether to upgrade. +## [10.0.3] + +- Start OpenCode safely from home after a global installation. Flow stays + inactive there and explains how to open a project, instead of failing the + server plugin. Project state and source scans remain excluded from home. +- Add a provider-free packed startup check for home, Git projects and ordinary + folders to CI and release smoke. Session v5 schema, permissions, validation, review, + continuation and closure requirements are unchanged. +- Use the reviewed baseline-qualified patch route against 10.0.2. No new live + model qualification or model canary was run for 10.0.3; 10.0.2's 80/81 matrix + and 6/6 canary are historical evidence, not measurements of this patch. + +Upgrade with `opencode2 plugin add opencode-plugin-flow@10.0.3`, then start +`opencode2` inside your project folder. + ## [10.0.2] - Put ordinary Flow execution before recovery details, share goal-alignment and diff --git a/README.md b/README.md index 327bfdf5..db4bcad3 100644 --- a/README.md +++ b/README.md @@ -43,21 +43,27 @@ Install OpenCode v2 and the exact Flow release: ```sh npm install -g @opencode/cli@2.1.4 opencode2 --version -opencode2 plugin add opencode-plugin-flow@10.0.2 +opencode2 plugin add opencode-plugin-flow@10.0.3 ``` -The version command must report `opencode v2.1.4` or a later tested 2.x version. -Flow selects HTTP transport for provider, model, and variant settings while loaded. -This keeps model requests observable for continuation and validation attribution. -The v2 package provides both `opencode` and `opencode2`. Use `opencode2` when an -older `opencode` executable remains on your PATH. The plugin command installs -Flow globally. Exact pins stay fixed until you install another version. +`opencode2 --version` must report `opencode v2.1.4` or a later tested 2.x version. +Use `opencode2` to avoid an older `opencode` on PATH. The install is global; +start inside a project: + +```sh +cd /path/to/your/project +opencode2 +``` + +Flow keeps state in `/.flow/` and refuses the home directory itself. +Exact pins stay fixed. Flow uses HTTP transport for provider, model, and variant +settings to observe continuation and validation. For project scope, add this entry to the project's `opencode.json`: ```json { - "plugins": ["opencode-plugin-flow@10.0.2"] + "plugins": ["opencode-plugin-flow@10.0.3"] } ``` @@ -74,7 +80,7 @@ Set plugin defaults: ```json { "plugins": [{ - "package": "opencode-plugin-flow@10.0.2", + "package": "opencode-plugin-flow@10.0.3", "options": { "reviewer": { "model": "provider/model", "steps": 80 }, "typesafeApiKeyEnv": "MY_TYPESAFE_KEY" diff --git a/docs/quickstart.md b/docs/quickstart.md index 3cbb3992..e7b9df31 100644 --- a/docs/quickstart.md +++ b/docs/quickstart.md @@ -5,9 +5,9 @@ consequential change. Read the plan and the review before accepting the result. ## Install Flow -Install the exact release shown in the [README installation section](../README.md#install). -Use OpenCode v2 with the tested minimum from the README. Finish or explicitly -close active Flow sessions before upgrading, then restart OpenCode. +Install the exact release and start OpenCode inside your project as shown in the +[README installation section](../README.md#install). Finish or explicitly close +active Flow sessions before upgrading, then restart OpenCode. ## Select Flow models diff --git a/docs/release-qualification.md b/docs/release-qualification.md index 4380cb33..f29e0723 100644 --- a/docs/release-qualification.md +++ b/docs/release-qualification.md @@ -35,11 +35,12 @@ Verifier fixes may reuse runs for unchanged package bytes and case policy. Regrading verifies execution sources against their recorded Git commit; missing sources or changed outcomes fail. Canary retries keep manager/reviewer identity. -Reviewed offline exceptions cover [narrow patches](../.agents/plans/06-patch-release/README.md) -and [frozen features](../.agents/plans/09-planning-models/README.md). They measure -against the last fully qualified release, never another offline one. Baseline -citations are historical evidence at their recorded source. Only the candidate -needs a fresh canary; historical reconstruction measures nothing new. +Reviewed offline paths cover [narrow patches](../.agents/plans/06-patch-release/README.md) +and [frozen features](../.agents/plans/09-planning-models/README.md), always against +a fully qualified baseline. Exact 10.0.3's reviewed home-startup patch uses +10.0.2's retained evidence plus Linux startup tests, without new model calls or +canary. Other runtime changes retain full qualification. Prior passes never +become candidate measurements. New scenarios need a policy decision. Missing required cases fail qualification. @@ -106,8 +107,7 @@ on the canonical Linux host. The v2 profile uses HTTP transport with ambient OpenCode configuration disabled. Run a fresh canary against its exact `artifact.tgz`, seal/regrade the bundle, and commit only evidence without changing measured inputs. Recheck final main CI and exact artifact identity before tagging `v`. -For 9.4.0, use -`bun scripts/prepare-qualified-release-940.ts --out opencode-plugin-flow-9.4.0.tgz`. +Historical artifact reconstruction follows the versioned release workflow. Authorize dispatches using the [paid-run budget](../.agents/plans/05-release-simplification/README.md#authorize-paid-work). Keep that ledger across retries. Budget-stopped campaigns cannot qualify. diff --git a/docs/troubleshooting.md b/docs/troubleshooting.md index 2c288ebb..0d521e87 100644 --- a/docs/troubleshooting.md +++ b/docs/troubleshooting.md @@ -1,15 +1,17 @@ # Troubleshooting -## Flow commands are missing +## Home-directory startup error + +OpenCode started in `~`. Run `cd /path/to/project && opencode2`. +Flow protects home from state and source scans; no reinstall is needed. -Rerun the exact-version plugin command from the -[README installation section](../README.md#install), or confirm that the -relevant `opencode.json` holds that same exact entry. +## Flow commands are missing -Restart OpenCode and inspect its plugin-loading logs. Flow has no installer, -cache cleaner, or configuration-repair CLI. Follow the official -[OpenCode plugin documentation](https://opencode.ai/v2/docs/plugins/) for npm or -configuration failures. +In a project folder, confirm the exact [README plugin entry](../README.md#install) +in `opencode.json`, restart OpenCode, and inspect plugin-loading logs. Flow does +not repair host configuration or caches. See the official +[OpenCode plugin documentation](https://opencode.ai/v2/docs/plugins/) for +installation or configuration failures. ## Reviewer model looks shared diff --git a/evals/qualification/patches/10.0.3.json b/evals/qualification/patches/10.0.3.json new file mode 100644 index 00000000..665f876e --- /dev/null +++ b/evals/qualification/patches/10.0.3.json @@ -0,0 +1,40 @@ +{ + "schemaVersion": 1, + "kind": "baseline-qualified-patch", + "version": "10.0.3", + "baseline": { + "commit": "a920f7375d2bd5262e7fc5c272f5cac55c6b3b4b", + "artifact": { + "packageVersion": "10.0.2", + "sourceCommit": "fdd11b7dea1e95a87d781bdee89ed84aea8a29e7", + "sourceTreeSha256": "sha256:5ca3dc21436044f8a4d3b0c92f160d3cb90a7af422cfed8dd252846a818eccc9", + "tarballSha256": "sha256:6d8345ea7968799778fa4dc9c818ab37449b70c16a08a7eedba48f8c24c8f5a4", + "unpackedManifestSha256": "sha256:cf68469968b148aac2ccf30d6e0d22460079597d59ff6662ec39907362842153" + }, + "canary": "evals/canary/10.0.2.json", + "bundleSha256": "sha256:89b32228d6651737152a616dea44439b1d8acd23037c1f6123f91c2e920674ed" + }, + "artifact": { + "packageVersion": "10.0.3", + "tarballSha256": "sha256:74294e90f7ff9e9500fab1ee4b896273bca9848815eab6a494c55a7bf75b1399", + "unpackedManifestSha256": "sha256:c136169692af6ef0dd95b1d6e73e3942b60588c91dbae8b202785cc428c6ee49" + }, + "guidance": [], + "approvedBy": "Maintainer approval in this T3 Code thread on 2026-10-29: release the home-startup fix without a new model qualification; full-stop requires reviewed PR handoff before merge.", + "rationale": "Exact-version home-startup compatibility patch. Only the two reviewed runtime files and the declared smoke command may change. Retain 10.0.2 historical matrix/canary evidence; candidate assurance is deterministic checks, Linux packed startup, replay and independent review, not a new model measurement.", + "startup": { + "kind": "home-directory-startup-10.0.3", + "files": [ + { + "path": "src/infrastructure/fs/workspace-paths.ts", + "baselineSha256": "sha256:f49ac543d8253e868aafd885feaced8214b0e3704815d606477ab3ab45c1e1fa", + "sha256": "sha256:fd146e0a680bd9d879292b6e3c88e853527d3354e1db842e13934b835c1b80c2" + }, + { + "path": "src/platform/opencode/plugin-composition.ts", + "baselineSha256": "sha256:26ffd9677ca07b4ab5f8f7297ecabc10cdf7df1feae0a2b8471490b5ca3ee214", + "sha256": "sha256:9e164e3a31464e8cfee7b860d6b707ae18e6418762c5647c0a05acba65556986" + } + ] + } +} diff --git a/package.json b/package.json index ab838e75..de16acf0 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "opencode-plugin-flow", - "version": "10.0.2", + "version": "10.0.3", "description": "Durable planning, validation, and independent review for OpenCode v2", "type": "module", "repository": { @@ -55,7 +55,7 @@ "replay": "bun run evals/replay-run.ts", "triage": "bun run scripts/triage-report.ts", "release:monitor": "node scripts/release-monitor.mjs", - "smoke:live": "FLOW_LIVE_SMOKE=1 bun test tests/live-opencode-smoke.test.ts", + "smoke:live": "FLOW_LIVE_SMOKE=1 bun test tests/live-opencode-smoke.test.ts tests/plugin-startup-live.test.ts", "test": "bun test tests", "typecheck": "tsc --noEmit", "check": "bun run typecheck && bun run lint && bun run release:metadata && bun run build && bun run test" diff --git a/scripts/patch-release.ts b/scripts/patch-release.ts index 580928ff..aa720439 100644 --- a/scripts/patch-release.ts +++ b/scripts/patch-release.ts @@ -13,6 +13,23 @@ import { assertStrictReleaseEvidence } from "./release-metadata.js"; const Hash = z.string().regex(/^sha256:[a-f0-9]{64}$/); const Commit = z.string().regex(/^[a-f0-9]{40}$/); const Text = z.string().trim().min(1).max(4096); +const StartupFile = (path: Path) => + z + .object({ path: z.literal(path), baselineSha256: Hash, sha256: Hash }) + .strict(); +const StartupApproval = z + .object({ + kind: z.literal("home-directory-startup-10.0.3"), + files: z.tuple([ + StartupFile("src/infrastructure/fs/workspace-paths.ts"), + StartupFile("src/platform/opencode/plugin-composition.ts"), + ]), + }) + .strict(); +const STARTUP_SMOKE = + "FLOW_LIVE_SMOKE=1 bun test tests/live-opencode-smoke.test.ts tests/plugin-startup-live.test.ts"; +const BASELINE_SMOKE = + "FLOW_LIVE_SMOKE=1 bun test tests/live-opencode-smoke.test.ts"; export const PatchReleaseSchema = z .object({ schemaVersion: z.literal(1), @@ -43,6 +60,7 @@ export const PatchReleaseSchema = z .max(1), approvedBy: Text, rationale: Text, + startup: StartupApproval.optional(), }) .strict(); export type PatchRelease = z.infer; @@ -57,6 +75,7 @@ export function assertPatchScope(input: { package: Record; changedPaths: readonly string[]; guidance: PatchRelease["guidance"]; + startup?: PatchRelease["startup"]; }) { const version = /^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)$/; const base = input.baselineVersion.match(version); @@ -71,6 +90,29 @@ export function assertPatchScope(input: { throw new Error("Patch must advance within the baseline major/minor."); const { version: oldVersion, ...oldPackage } = input.baselinePackage; const { version: newVersion, ...newPackage } = input.package; + if (input.startup) { + StartupApproval.parse(input.startup); + if ( + input.baselineVersion !== "10.0.2" || + input.version !== "10.0.3" || + input.guidance.length + ) + throw new Error( + "Startup exception is only 10.0.3 against fully qualified 10.0.2, without guide changes.", + ); + const oldScripts = oldPackage.scripts as + | Record + | undefined; + const scripts = newPackage.scripts as Record | undefined; + if ( + oldScripts?.["smoke:live"] !== BASELINE_SMOKE || + scripts?.["smoke:live"] !== STARTUP_SMOKE + ) + throw new Error( + "Startup patch must add only the declared native startup smoke.", + ); + newPackage.scripts = { ...scripts, "smoke:live": BASELINE_SMOKE }; + } if ( oldVersion !== input.baselineVersion || newVersion !== input.version || @@ -85,12 +127,14 @@ export function assertPatchScope(input: { export function assertPatchPaths(input: { changedPaths: readonly string[]; guidance: readonly { readonly path: string }[]; + startup?: PatchRelease["startup"]; }) { const guides = new Set(input.guidance.map((entry) => entry.path)); if (guides.size !== input.guidance.length) throw new Error("Duplicate guidance approval."); for (const path of input.changedPaths) { if (guides.has(path)) continue; + if (input.startup?.files.some((file) => file.path === path)) continue; if (/^(?:docs\/|tests\/|evals\/|\.agents\/plans\/)/.test(path)) continue; if ( [ @@ -117,6 +161,9 @@ export function assertPatchPaths(input: { for (const guide of guides) if (!input.changedPaths.includes(guide)) throw new Error("Guidance approval does not name a changed guide."); + for (const file of input.startup?.files ?? []) + if (!input.changedPaths.includes(file.path)) + throw new Error("Startup approval does not name a changed runtime file."); } export async function releaseGit(root: string, args: string[]) { @@ -201,7 +248,31 @@ export async function assertPatchReleaseEvidence( package: JSON.parse(await readFile(resolve(root, "package.json"), "utf8")), changedPaths, guidance: record.guidance, + ...(record.startup ? { startup: record.startup } : {}), }); + for (const file of record.startup?.files ?? []) { + for (const [revision, expected] of [ + [record.baseline.artifact.sourceCommit, file.baselineSha256], + ["HEAD", file.sha256], + ] as const) { + if ( + !( + await releaseGit(root, ["ls-tree", revision, "--", file.path]) + ).startsWith("100644 blob ") + ) + throw new Error( + "Startup files must remain regular non-executable files.", + ); + const content = + revision === "HEAD" + ? await readFile(resolve(root, file.path)) + : Buffer.from( + await releaseGit(root, ["show", `${revision}:${file.path}`]), + ); + if (hash(content) !== expected) + throw new Error("Reviewed startup content changed."); + } + } if (changedPaths.includes("biome.json")) { const oldConfig = JSON.parse( await releaseGit(root, [ @@ -244,7 +315,13 @@ export async function assertPatchReleaseEvidence( }); if (baseline.bundleSha256 !== record.baseline.bundleSha256) throw new Error("Baseline qualification bundle changed."); - if (new Set(baseline.summary.providers.map((item) => item.provider)).size < 2) + const providers = new Set( + baseline.summary.providers.map((item) => item.provider), + ); + if ( + providers.size < 2 && + !(record.startup && providers.size === 1 && providers.has("openai")) + ) throw new Error("Patch exception requires a two-provider baseline."); return { bundleSha256: hash(bytes), diff --git a/src/infrastructure/fs/workspace-paths.ts b/src/infrastructure/fs/workspace-paths.ts index 1c1bb940..10be33e2 100644 --- a/src/infrastructure/fs/workspace-paths.ts +++ b/src/infrastructure/fs/workspace-paths.ts @@ -8,6 +8,8 @@ class InvalidFlowWorkspaceRootError extends Error { readonly code = "INVALID_FLOW_WORKSPACE_ROOT"; } +export class HomeDirectoryWorkspaceError extends InvalidFlowWorkspaceRootError {} + function normalizeWorkspaceRoot(rawPath: string | undefined): string | null { const value = rawPath?.trim(); if (!value) return null; @@ -46,7 +48,7 @@ export function assertMutableWorkspaceRoot(rawPath: string): string { } }); if (homes.includes(root)) { - throw new InvalidFlowWorkspaceRootError( + throw new HomeDirectoryWorkspaceError( "Flow refuses to use the home directory itself as mutable state.", ); } diff --git a/src/platform/opencode/plugin-composition.ts b/src/platform/opencode/plugin-composition.ts index a4b2b9a4..60addec8 100644 --- a/src/platform/opencode/plugin-composition.ts +++ b/src/platform/opencode/plugin-composition.ts @@ -12,7 +12,10 @@ import { } from "../../config-shared.js"; import { createFileSourceIdentityProvider } from "../../infrastructure/fs/source-identity.js"; import { createWorkspaceFlowService } from "../../infrastructure/fs/workspace-flow-service.js"; -import { resolveWorkspaceRoot } from "../../infrastructure/fs/workspace-paths.js"; +import { + HomeDirectoryWorkspaceError, + resolveWorkspaceRoot, +} from "../../infrastructure/fs/workspace-paths.js"; import { persistWorkspaceValidation, prepareWorkspaceValidation, @@ -54,6 +57,17 @@ export function createFlowPlugin(dependencies: { id: "opencode-plugin-flow", async setup(ctx) { const log = createFlowLog(ctx); + let workspace: string; + try { + workspace = resolveWorkspaceRoot({ directory: ctx.location.directory }); + } catch (error) { + if (!(error instanceof HomeDirectoryWorkspaceError)) throw error; + log( + "warn", + "Flow is inactive in the home directory to keep its state inside a project. Open a project folder and start OpenCode there: cd /path/to/project && opencode2.", + ); + return; + } const baseReviewer = resolveFlowReviewerConfiguration({ pluginOptions: ctx.options, onWarning: (warning) => log("warn", warning), @@ -65,9 +79,6 @@ export function createFlowPlugin(dependencies: { const pluginEntrySha256 = `sha256:${createHash("sha256") .update(await readFile(fileURLToPath(dependencies.entryUrl))) .digest("hex")}`; - const workspace = resolveWorkspaceRoot({ - directory: ctx.location.directory, - }); const runtimeGuard = registerFlowPluginInstance(workspace, { packageName: "opencode-plugin-flow", version, diff --git a/tests/distribution-and-surface.test.ts b/tests/distribution-and-surface.test.ts index 4a06f9c8..2922c8f9 100644 --- a/tests/distribution-and-surface.test.ts +++ b/tests/distribution-and-surface.test.ts @@ -1,4 +1,4 @@ -import { afterEach, describe, expect, test } from "bun:test"; +import { afterEach, describe, expect, spyOn, test } from "bun:test"; import { createHash } from "node:crypto"; import { mkdir, @@ -19,6 +19,7 @@ import { getFlowGuidance, } from "../src/guidance/catalog.js"; import FlowPlugin from "../src/index.js"; +import { assertMutableWorkspaceRoot } from "../src/infrastructure/fs/workspace-paths.js"; import { AutoDriveCoordinator, FLOW_AUTO_METADATA_KEY, @@ -299,13 +300,53 @@ describe("Flow distribution surface", () => { text: expect.stringContaining("Assignment: assignment-1"), }); }); - test("refuses to load the home directory before registering native tools", async () => { + test("loads inertly at home, preserves the state guard, and still activates in a project", async () => { const host = await createNativeFlowHost(homedir()); hosts.push(host); - await expect(host.load(FlowPlugin)).rejects.toThrow( + const log = spyOn(process.stderr, "write").mockImplementation(() => true); + try { + await host.load(FlowPlugin); + expect(log).toHaveBeenCalledWith( + expect.stringContaining("Flow is inactive in the home directory"), + ); + expect(log).toHaveBeenCalledWith(expect.stringContaining("opencode2")); + } finally { + log.mockRestore(); + } + expect(host.tools.size).toBe(0); + expect(host.commands.size).toBe(0); + expect(host.agents.size).toBe(0); + expect(host.events.subscribers.size).toBe(0); + expect(host.sessionHooks.callbacks).toEqual({}); + expect(host.toolHooks.callbacks).toEqual({}); + expect(() => assertMutableWorkspaceRoot(homedir())).toThrow( "Flow refuses to use the home directory itself", ); - expect(host.tools.size).toBe(0); + const { host: project } = await load(); + expect([...project.tools.keys()].sort()).toEqual(TOOL_NAMES); + expect([...project.commands.keys()].sort()).toEqual(COMMAND_NAMES); + expect((await project.response("flow_status", statusInput)).status).toBe( + "ok", + ); + }); + test("still fails setup for a filesystem root or a missing workspace", async () => { + const root = await createNativeFlowHost("/"); + hosts.push(root); + await expect(root.load(FlowPlugin)).rejects.toThrow( + "Flow could not resolve a workspace root", + ); + const workspace = await mkdtemp(join(tmpdir(), "flow-missing-workspace-")); + workspaces.push(workspace); + const missing = await createNativeFlowHost(workspace); + hosts.push(missing); + await rm(workspace, { recursive: true }); + await expect(missing.load(FlowPlugin)).rejects.toThrow( + "Flow requires an existing workspace directory", + ); + for (const host of [root, missing]) { + expect(host.tools.size).toBe(0); + expect(host.commands.size).toBe(0); + } }); }); diff --git a/tests/eval-release-sampling.test.ts b/tests/eval-release-sampling.test.ts index 808e3f28..53767d6d 100644 --- a/tests/eval-release-sampling.test.ts +++ b/tests/eval-release-sampling.test.ts @@ -193,9 +193,10 @@ describe("release eval sampling", () => { test("checks configured CI release models before authorization", async () => { for (const [configured, expectedCode] of [ - ["openai/gpt-6.1-sol", 0], + ["openai/gpt-6.1-sol", 1], + ["openai/gpt-6.1-sol,fixture/no-calls", 0], ["openai/gpt-6-sol", 1], - ["openai/gpt-6.1-sol,xai/grok-4.7", 1], + ["openai/gpt-6.1-sol,xai/grok-4.7", 0], ["openai/gpt-6.1-sol,openai/gpt-6-sol", 1], ["xai/grok-4.6", 1], ] as const) { @@ -214,7 +215,7 @@ describe("release eval sampling", () => { expect(code).toBe(expectedCode); if (expectedCode !== 0) expect(stderr).toContain( - `Release ${packageJson.version} requires exactly openai/gpt-6.1-sol with its canonical direct-route identity.`, + "Release requires exactly 2 models on distinct route providers.", ); } }); @@ -381,24 +382,24 @@ describe("release eval sampling", () => { ).toThrow("openai/gpt-6.1-sol"); }); - test("schedules the current OpenAI-only twenty-case grid", () => { + test("keeps the full twenty-case grid at two providers for the offline patch", () => { const scenarios = releaseScenarios(packageJson.version); expect(scenarios).toHaveLength(20); const plan = campaignPlanFor({ - models: ["openai/gpt-6.1-sol"], + models: ["openai/gpt-6.1-sol", "fixture/no-calls"], scenarios, sampling: { kind: "release", packageVersion: packageJson.version }, opencodeVersion: "2.1.4", }); - expect(plan.stoppingRule.count).toBe(81); - expect(plan.budget.maxAttempts).toBe(101); - expect(plan.abortPolicy.maxReplacementBlocks).toBe(20); + expect(plan.stoppingRule.count).toBe(162); + expect(plan.budget.maxAttempts).toBe(202); + expect(plan.abortPolicy.maxReplacementBlocks).toBe(40); expect( plan.cells.filter((cell) => cell.schedule === "primary"), - ).toHaveLength(81); + ).toHaveLength(162); expect( plan.cells.filter((cell) => cell.schedule === "environment-reserve"), - ).toHaveLength(20); + ).toHaveLength(40); }); test("keeps ordinary scenario catalogs report-only", () => { @@ -463,7 +464,7 @@ describe("release eval sampling", () => { } }); - test("rejects current release grids outside the canonical OpenAI pin", async () => { + test("rejects current full grids without exactly two distinct providers", async () => { for (const models of [ ["xai/a"], ["xai/a", "xai/b"], @@ -486,7 +487,7 @@ describe("release eval sampling", () => { ]); expect(exitCode).toBe(2); expect(stderr).toContain( - `Release ${packageJson.version} requires exactly openai/gpt-6.1-sol with its canonical direct-route identity.`, + "Release requires exactly 2 models on distinct route providers.", ); } }); @@ -499,6 +500,8 @@ describe("release eval sampling", () => { "evals/run.ts", "--model", "openai/gpt-6.1-sol", + "--model", + "fixture/no-calls", "--release", "--concurrency", "2", diff --git a/tests/patch-release.test.ts b/tests/patch-release.test.ts index 867edbb2..406052d3 100644 --- a/tests/patch-release.test.ts +++ b/tests/patch-release.test.ts @@ -72,7 +72,7 @@ async function git(root: string, ...args: string[]) { if (code) throw new Error(err); return out.trim(); } -async function fixture() { +async function fixture(startup = false) { const parent = await mkdtemp(join(tmpdir(), "flow-patch-")); roots.push(parent); const root = join(parent, "repo"); @@ -82,7 +82,37 @@ async function fixture() { await git(root, "config", "user.name", "Test"); await mkdir(join(root, "src")); await mkdir(join(root, "skills/flow-run"), { recursive: true }); - await writeFile(join(root, "package.json"), JSON.stringify(metadata)); + const baselineVersion = startup ? "10.0.2" : "8.3.0"; + const version = startup ? "10.0.3" : "8.3.1"; + const baselinePackage = startup + ? { + ...metadata, + version: baselineVersion, + scripts: { + ...metadata.scripts, + "smoke:live": + "FLOW_LIVE_SMOKE=1 bun test tests/live-opencode-smoke.test.ts", + }, + } + : metadata; + await writeFile(join(root, "package.json"), JSON.stringify(baselinePackage)); + const startupFiles: NonNullable["files"] = [ + { + path: "src/infrastructure/fs/workspace-paths.ts", + baselineSha256: hash("before\n"), + sha256: hash("after\n"), + }, + { + path: "src/platform/opencode/plugin-composition.ts", + baselineSha256: hash("before\n"), + sha256: hash("after\n"), + }, + ]; + if (startup) + for (const file of startupFiles) { + await mkdir(join(root, file.path, ".."), { recursive: true }); + await writeFile(join(root, file.path), "before\n"); + } await writeFile( join(root, "biome.json"), JSON.stringify({ files: { includes: ["**"] } }), @@ -92,13 +122,31 @@ async function fixture() { await git(root, "add", "."); await git(root, "commit", "-m", "full baseline"); const commit = await git(root, "rev-parse", "HEAD"); - await git(root, "tag", "v8.3.0"); - await writeFile(join(root, "package.json"), JSON.stringify(scope.package)); - await writeFile(join(root, "skills/flow-run/SKILL.md"), "clarification"); + await git(root, "tag", `v${baselineVersion}`); + await writeFile( + join(root, "package.json"), + JSON.stringify( + startup + ? { + ...baselinePackage, + version, + scripts: { + ...baselinePackage.scripts, + "smoke:live": + "FLOW_LIVE_SMOKE=1 bun test tests/live-opencode-smoke.test.ts tests/plugin-startup-live.test.ts", + }, + } + : scope.package, + ), + ); + if (startup) + for (const file of startupFiles) + await writeFile(join(root, file.path), "after\n"); + else await writeFile(join(root, "skills/flow-run/SKILL.md"), "clarification"); await git(root, "add", "."); await git(root, "commit", "-m", "patch"); const baselineArtifact = { - packageVersion: "8.3.0", + packageVersion: baselineVersion, sourceCommit: commit, sourceTreeSha256: hash("tree"), tarballSha256: hash("baseline"), @@ -106,13 +154,13 @@ async function fixture() { }; const candidate = { ...baselineArtifact, - packageVersion: "8.3.1", + packageVersion: version, tarballSha256: hash("candidate"), }; const record: PatchRelease = { schemaVersion: 1, kind: "baseline-qualified-patch", - version: "8.3.1", + version, baseline: { commit, artifact: baselineArtifact, @@ -124,15 +172,25 @@ async function fixture() { tarballSha256: candidate.tarballSha256, unpackedManifestSha256: candidate.unpackedManifestSha256, }, - guidance: [ - { - path: "skills/flow-run/SKILL.md", - sha256: hash("clarification"), - rationale: "Clarify existing handoff", - }, - ], + guidance: startup + ? [] + : [ + { + path: "skills/flow-run/SKILL.md", + sha256: hash("clarification"), + rationale: "Clarify existing handoff", + }, + ], approvedBy: "test operator", rationale: "No core guarantees changed", + ...(startup + ? { + startup: { + kind: "home-directory-startup-10.0.3" as const, + files: startupFiles, + }, + } + : {}), }; const path = join(parent, "patch.json"); await writeFile(path, JSON.stringify(record)); @@ -143,20 +201,22 @@ async function fixture() { bundleSha256: hash("bundle"), summary: { schemaVersion: 1, - packageVersion: "8.3.0", + packageVersion: baselineVersion, reportId: "baseline-report", verdict: "VERIFIED", bundleSha256: hash("bundle"), artifact: baselineArtifact, canarySha256: hash("canary"), totals: { scheduled: 76, passed: 76, scored: 76 }, - providers: ["openai", "xai"].map((provider) => ({ - provider, - scheduled: 38, - scored: 38, - passed: 38, - passRate: 1, - })), + providers: (startup ? ["openai"] : ["openai", "xai"]).map( + (provider) => ({ + provider, + scheduled: 38, + scored: 38, + passed: 38, + passRate: 1, + }), + ), } as ReleaseEvidenceSummary, }; }; @@ -184,6 +244,135 @@ test("eligible patch verifies baseline and discloses prior-only evidence", async expect(result.notes).toContain("not measurements of this candidate"); expect(result.bundleSha256).toBe(hash(await readFile(f.input.path, "utf8"))); }); +test("exact startup patch verifies the retained OpenAI baseline without claiming a new measurement", async () => { + const f = await fixture(true); + const result = await assertPatchReleaseEvidence(f.input, f.verify); + expect(f.calls()).toBe(1); + expect(result.notes).toContain("No live model evals or canary were run"); + expect(result.notes).toContain("Prior measured version: 10.0.2"); + await expect( + assertPatchReleaseEvidence(f.input, async () => { + const verified = await f.verify(); + return { + ...verified, + summary: { + ...verified.summary, + providers: verified.summary.providers.map((row) => ({ + ...row, + provider: "fixture", + })), + }, + }; + }), + ).rejects.toThrow("two-provider baseline"); +}); +test("startup approval rejects changed baseline or candidate code, extra runtime paths and script edits", async () => { + const f = await fixture(true); + const startup = f.record.startup; + if (!startup) throw new Error("Missing fixture approval"); + for (const field of ["baselineSha256", "sha256"] as const) { + const prior = startup.files[0][field]; + startup.files[0][field] = hash("substituted"); + await f.writeRecord(); + await expect(assertPatchReleaseEvidence(f.input, f.verify)).rejects.toThrow( + "startup content changed", + ); + startup.files[0][field] = prior; + } + await f.writeRecord(); + await writeFile(join(f.root, "src/index.ts"), "other runtime change"); + await git(f.root, "add", "."); + await git(f.root, "commit", "-m", "unapproved runtime"); + await expect(assertPatchReleaseEvidence(f.input, f.verify)).rejects.toThrow( + "Full qualification", + ); + expect(f.calls()).toBe(0); +}); +test("startup scope cannot extend to other versions, guides, dependencies or smoke bypasses", () => { + const startup = { + kind: "home-directory-startup-10.0.3" as const, + files: [ + { + path: "src/infrastructure/fs/workspace-paths.ts" as const, + baselineSha256: hash("a"), + sha256: hash("b"), + }, + { + path: "src/platform/opencode/plugin-composition.ts" as const, + baselineSha256: hash("a"), + sha256: hash("b"), + }, + ] as NonNullable["files"], + }; + const before = { + ...metadata, + version: "10.0.2", + scripts: { + build: "fixed", + "smoke:live": + "FLOW_LIVE_SMOKE=1 bun test tests/live-opencode-smoke.test.ts", + }, + }; + const after = { + ...before, + version: "10.0.3", + scripts: { + ...before.scripts, + "smoke:live": `${before.scripts["smoke:live"]} tests/plugin-startup-live.test.ts`, + }, + }; + const input = { + ...scope, + baselineVersion: "10.0.2", + version: "10.0.3", + baselinePackage: before, + package: after, + changedPaths: startup.files.map((file) => file.path), + startup, + }; + expect(() => assertPatchScope(input)).not.toThrow(); + for (const version of ["10.0.4", "10.1.0", "11.0.0"]) + expect(() => + assertPatchScope({ ...input, version, package: { ...after, version } }), + ).toThrow(); + expect(() => + assertPatchScope({ ...input, baselineVersion: "10.0.1" }), + ).toThrow(); + expect(() => assertPatchScope({ ...input, startup: undefined })).toThrow(); + expect(() => + assertPatchScope({ + ...input, + package: { ...after, dependencies: { test: "2.0" } }, + }), + ).toThrow("metadata"); + expect(() => + assertPatchScope({ + ...input, + package: { ...after, scripts: { ...after.scripts, build: "changed" } }, + }), + ).toThrow("metadata"); + expect(() => + assertPatchScope({ + ...input, + package: { + ...after, + scripts: { ...after.scripts, "smoke:live": "echo passed" }, + }, + }), + ).toThrow("declared native startup smoke"); + expect(() => + assertPatchScope({ + ...input, + guidance: [ + { + path: "skills/flow-run/SKILL.md", + sha256: hash("guide"), + rationale: "extra", + }, + ], + }), + ).toThrow("without guide changes"); +}); test("refuses a one-provider release as a patch baseline", async () => { const f = await fixture(); await expect( diff --git a/tests/plugin-startup-live.test.ts b/tests/plugin-startup-live.test.ts new file mode 100644 index 00000000..0f2c72ca --- /dev/null +++ b/tests/plugin-startup-live.test.ts @@ -0,0 +1,215 @@ +import { expect, test } from "bun:test"; +import { type ChildProcess, spawn, spawnSync } from "node:child_process"; +import { once } from "node:events"; +import { + access, + mkdir, + mkdtemp, + realpath, + rm, + writeFile, +} from "node:fs/promises"; +import { createServer } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { OpenCodeClient } from "@opencode/client"; +import { currentBunToolchain } from "../evals/bun-toolchain.js"; +import { packPlugin, preparePackageCache } from "../evals/harness.js"; +import { + installedOpenCodePluginEntry, + OPENCODE_HOST_VERSION, + OpenCodeStartup, + openCodeInvocation, +} from "../evals/opencode-host.js"; +import packageJson from "../package.json" with { type: "json" }; + +async function availablePort(): Promise { + const server = createServer(); + server.listen(0, "127.0.0.1"); + await once(server, "listening"); + const address = server.address(); + if (!address || typeof address === "string") + throw new Error("No loopback port"); + await new Promise((resolve, reject) => + server.close((error) => (error ? reject(error) : resolve())), + ); + return address.port; +} + +// This child starts in its own process group. Stop only that owned group and +// await the child, without inspecting the machine's process table. +async function stopOwned(child: ChildProcess): Promise { + if ( + child.pid === undefined || + child.exitCode !== null || + child.signalCode !== null + ) + return; + const exited = once(child, "exit"); + const signal = (value: NodeJS.Signals) => { + try { + if (process.platform !== "win32" && child.pid !== undefined) + process.kill(-child.pid, value); + else child.kill(value); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "ESRCH") throw error; + } + }; + signal("SIGTERM"); + if ( + await Promise.race([ + exited.then(() => false), + Bun.sleep(3_000).then(() => true), + ]) + ) + signal("SIGKILL"); + await exited; +} + +test.skipIf(process.env.FLOW_LIVE_SMOKE !== "1")( + "global packed plugin starts at home and activates in project locations without a model call", + async () => { + const root = await realpath( + await mkdtemp(join(tmpdir(), "flow-plugin-startup-")), + ); + const home = join(root, "home"); + const project = join(home, "project"); + const plainProject = join(home, "plain project"); + const startup = new OpenCodeStartup(); + let child: ChildProcess | undefined; + try { + const toolchain = currentBunToolchain(packageJson.packageManager); + const tarball = + process.env.FLOW_LIVE_SMOKE_TARBALL ?? + (await packPlugin(join(import.meta.dir, ".."), root, toolchain)); + const cache = await preparePackageCache(tarball, root, toolchain); + const plugin = join(cache, "node_modules", "opencode-plugin-flow"); + const entry = await installedOpenCodePluginEntry( + plugin, + join(root, "entry"), + ); + for (const path of [ + project, + plainProject, + join(home, ".config", "opencode"), + ...["data", "cache", "state", "tmp"].map((name) => join(root, name)), + ]) + await mkdir(path, { recursive: true }); + await writeFile( + join(home, ".config", "opencode", "opencode.json"), + JSON.stringify({ plugins: [entry] }), + ); + const port = await availablePort(); + const version = + process.env.FLOW_OPENCODE_SMOKE_VERSION ?? OPENCODE_HOST_VERSION; + const invoke = (args: string[]) => + process.env.FLOW_OPENCODE_SMOKE_EXECUTABLE + ? { command: process.env.FLOW_OPENCODE_SMOKE_EXECUTABLE, args } + : openCodeInvocation(args, version); + const environment = { + PATH: toolchain.environment.PATH, + HOME: home, + XDG_CONFIG_HOME: join(home, ".config"), + XDG_DATA_HOME: join(root, "data"), + XDG_CACHE_HOME: join(root, "cache"), + XDG_STATE_HOME: join(root, "state"), + TMPDIR: join(root, "tmp"), + }; + expect( + spawnSync("git", ["init", "-q", project], { env: environment }).status, + ).toBe(0); + expect( + spawnSync( + "git", + [ + "-C", + project, + "-c", + "core.hooksPath=/dev/null", + "-c", + "user.name=Startup fixture", + "-c", + "user.email=startup@example.invalid", + "-c", + "commit.gpgSign=false", + "commit", + "--allow-empty", + "-qm", + "fixture", + ], + { env: environment }, + ).status, + ).toBe(0); + const invocation = invoke([ + "serve", + "--hostname", + "127.0.0.1", + "--port", + String(port), + ]); + child = spawn(invocation.command, invocation.args, { + cwd: home, + detached: process.platform !== "win32", + stdio: ["ignore", "pipe", "pipe"], + env: environment, + }); + child.stdout?.on("data", (chunk) => startup.consume(String(chunk))); + child.stderr?.on("data", (chunk) => + startup.consume(String(chunk), "stderr"), + ); + let spawnError: Error | undefined; + child.once("error", (error) => { + spawnError = error; + }); + let client: OpenCodeClient | null = null; + const deadline = Date.now() + 120_000; + while (Date.now() < deadline) { + if (spawnError) throw spawnError; + if (child.exitCode !== null) + throw new Error("Isolated OpenCode exited before readiness."); + client = startup.client(`http://127.0.0.1:${port}`, home); + if (client) break; + await Bun.sleep(50); + } + if (!client) throw new Error("Isolated OpenCode did not become ready."); + const request = () => ({ signal: AbortSignal.timeout(30_000) }); + expect((await client.server.info(request())).version).toBe(version); + for (const directory of [home, project, plainProject, home]) { + const location = { directory }; + const scoped = startup.client(`http://127.0.0.1:${port}`, directory); + if (!scoped) throw new Error("Missing isolated server authentication."); + const configuration = await scoped.config.get({ location }, request()); + expect(JSON.stringify(configuration).includes(entry)).toBe(true); + const inventoryDeadline = Date.now() + 30_000; + let commands = await scoped.command.list({ location }, request()); + while ( + directory !== home && + !commands.data.some((item) => item.name === "flow-status") && + Date.now() < inventoryDeadline + ) { + await Bun.sleep(50); + commands = await scoped.command.list({ location }, request()); + } + expect( + commands.data.some((command) => command.name === "flow-status"), + ).toBe(directory !== home); + const plugins = await scoped.plugin.list({ location }, request()); + expect( + plugins.data.filter((item) => item.state.status === "failed"), + ).toEqual([]); + } + expect(startup.log).toContain("Flow is inactive in the home directory"); + await expect(access(join(home, ".flow"))).rejects.toMatchObject({ + code: "ENOENT", + }); + } finally { + try { + if (child) await stopOwned(child); + } finally { + startup.clear(); + await rm(root, { recursive: true, force: true }); + } + } + }, + 180_000, +); diff --git a/tests/qualification-cli.test.ts b/tests/qualification-cli.test.ts index 073cdcd7..665d67f4 100644 --- a/tests/qualification-cli.test.ts +++ b/tests/qualification-cli.test.ts @@ -55,29 +55,29 @@ import { assertQualificationBundle } from "../scripts/release-metadata.js"; import { autoQualifiedOutcome } from "./fixtures/auto-qualified-outcome.js"; -test("current release defaults retain 20 cases on OpenAI and three autonomous attempts", () => { - expect(packageJson.version).toBe("10.0.2"); +test("full qualification retains two providers; the startup patch uses separate offline evidence", () => { + expect(packageJson.version).toBe("10.0.3"); const scenarios = releaseScenarios(packageJson.version); const catalog = releaseCatalog(packageJson.version); expect(scenarios).toHaveLength(20); - expect([...new Set(catalog.map((row) => row.minProviders))]).toEqual([1]); + expect([...new Set(catalog.map((row) => row.minProviders))]).toEqual([2]); const plan = campaignPlanFor({ - models: ["openai/gpt-6.1-sol"], + models: ["openai/gpt-6.1-sol", "fixture/no-calls"], scenarios, sampling: { kind: "release" }, opencodeVersion: releaseHostPolicy(packageJson.version).opencodeVersion, }); expect(plan.cells.filter((cell) => cell.schedule === "primary")).toHaveLength( - 81, + 162, ); expect( plan.cells.filter((cell) => cell.schedule === "environment-reserve"), - ).toHaveLength(20); + ).toHaveLength(40); expect( [ ...new Set(plan.cells.map((cell) => cell.managerModel?.routeProvider)), ].sort(), - ).toEqual(["openai"]); + ).toEqual(["fixture", "openai"]); expect( attemptsForScenario("auto-two-features-evidence", { kind: "release" }), ).toBe(3); diff --git a/tests/release-940-preparation.test.ts b/tests/release-940-preparation.test.ts index 5ec2cdcc..490445bf 100644 --- a/tests/release-940-preparation.test.ts +++ b/tests/release-940-preparation.test.ts @@ -483,3 +483,26 @@ for (const [job, publish, refuse, status, contents] of [ } }); } + +test("v2 offline patch keeps candidate bytes for strict patch verification instead of requiring a model archive", async () => { + const f = await fixture("10.0.3"); + const patches = join(f.root, "evals/qualification/patches"); + await mkdir(patches, { recursive: true }); + await writeFile(join(patches, "10.0.3.json"), "{}"); + const result = execute(await step("release", "Prepare package"), f, "10.0.3"); + expect(result.status, result.stderr).toBe(0); + expect( + await readFile(join(f.root, "opencode-plugin-flow-10.0.3.tgz"), "utf8"), + ).toBe("generic-10.0.3"); + await expect( + readFile(join(f.root, ".restore-attempt")), + ).rejects.toMatchObject({ code: "ENOENT" }); + // The preparation step makes no verdict; the existing next step still parses + // and validates the selected patch before any publication record is persisted. + expect(await step("release", "Verify selected release evidence")).toContain( + "--patch", + ); + expect(await step("release", "Verify selected release evidence")).toContain( + "scripts/release.ts init", + ); +}); From 0f4b857bdd3fbe7c15937bc9abbe1b2d3525db79 Mon Sep 17 00:00:00 2001 From: Douwe de Vries Date: Thu, 29 Oct 2026 19:22:17 +0100 Subject: [PATCH 2/2] fix(release): verify the retained startup baseline at its release tag --- .agents/plans/06-patch-release/README.md | 7 +- docs/release-qualification.md | 2 +- scripts/patch-release.ts | 84 ++++++++++++++++++++++-- tests/patch-release.test.ts | 43 ++++++++++++ 4 files changed, 127 insertions(+), 9 deletions(-) diff --git a/.agents/plans/06-patch-release/README.md b/.agents/plans/06-patch-release/README.md index fe7c8f28..f1ceb120 100644 --- a/.agents/plans/06-patch-release/README.md +++ b/.agents/plans/06-patch-release/README.md @@ -44,8 +44,11 @@ extra runtime paths or weakened smoke commands fail. Review must confirm that only home-directory setup becomes inactive, preserving mutable-state rejection, project registration and every workflow gate. Linux packed startup must cover home, Git and non-Git folders without a model call. -The baseline is reverified as retained evidence; the patch has no new model -measurements or canary. Publication still checks exact candidate package bytes. +The baseline is reverified by its original released-tag authority in a temporary +worktree with the unchanged dependencies. This avoids resealing historical +evidence merely because the candidate runtime/version changed. No credentials +are inherited and no model calls occur. The patch has no new model measurements +or canary. Publication still checks exact candidate package bytes. ## Evidence diff --git a/docs/release-qualification.md b/docs/release-qualification.md index f29e0723..1208cb9b 100644 --- a/docs/release-qualification.md +++ b/docs/release-qualification.md @@ -38,7 +38,7 @@ sources or changed outcomes fail. Canary retries keep manager/reviewer identity. Reviewed offline paths cover [narrow patches](../.agents/plans/06-patch-release/README.md) and [frozen features](../.agents/plans/09-planning-models/README.md), always against a fully qualified baseline. Exact 10.0.3's reviewed home-startup patch uses -10.0.2's retained evidence plus Linux startup tests, without new model calls or +10.0.2's released-tag evidence plus Linux startup tests, without new model calls or canary. Other runtime changes retain full qualification. Prior passes never become candidate measurements. diff --git a/scripts/patch-release.ts b/scripts/patch-release.ts index aa720439..99eed7ad 100644 --- a/scripts/patch-release.ts +++ b/scripts/patch-release.ts @@ -1,6 +1,7 @@ import { createHash } from "node:crypto"; -import { readFile } from "node:fs/promises"; -import { resolve } from "node:path"; +import { mkdir, mkdtemp, readFile, rm, symlink } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; import { z } from "zod"; import { canonicalJson } from "../evals/canonical-json.js"; import { samePackedArtifact } from "../evals/provenance.js"; @@ -181,6 +182,72 @@ export async function releaseGit(root: string, args: string[]) { return out; } +/** Retained evidence is checked by its released authority, not the new runtime. */ +async function verifyStartupBaseline(root: string, record: PatchRelease) { + const temporary = await mkdtemp(join(tmpdir(), "flow-startup-baseline-")); + const source = join(temporary, "source"); + let added = false; + try { + await releaseGit(root, [ + "-c", + "core.hooksPath=/dev/null", + "worktree", + "add", + "--quiet", + "--detach", + source, + record.baseline.commit, + ]); + added = true; + // Scope validation already requires unchanged dependencies and lockfiles. + await symlink( + resolve(root, "node_modules"), + join(source, "node_modules"), + process.platform === "win32" ? "junction" : "dir", + ); + const home = join(temporary, "home"); + await mkdir(home); + const child = Bun.spawn( + [ + process.execPath, + "-e", + ` +import { assertStrictReleaseEvidence } from "./scripts/release-metadata.ts"; +const artifact = JSON.parse(process.argv[1]); +const result = await assertStrictReleaseEvidence({ + version: "10.0.2", expectedArtifact: artifact, + canaryPath: "evals/canary/10.0.2.json", + bundlesDirectory: "evals/qualification/bundles", freshness: "retained", +}); +console.log(JSON.stringify(result)); +`, + JSON.stringify(record.baseline.artifact), + ], + { + cwd: source, + env: { PATH: process.env.PATH ?? "", HOME: home }, + timeout: 120_000, + stdout: "pipe", + stderr: "pipe", + }, + ); + const [out, error, code] = await Promise.all([ + new Response(child.stdout).text(), + new Response(child.stderr).text(), + child.exited, + ]); + if (code !== 0) + throw new Error(`Released baseline verification failed: ${error.trim()}`); + return JSON.parse(out) as Awaited< + ReturnType + >; + } finally { + if (added) + await releaseGit(root, ["worktree", "remove", "--force", source]); + await rm(temporary, { recursive: true, force: true }); + } +} + export async function assertPatchReleaseEvidence( input: { path: string; @@ -188,7 +255,7 @@ export async function assertPatchReleaseEvidence( bundlesDirectory: string; repositoryRoot?: string; }, - verifyBaseline = assertStrictReleaseEvidence, + verifyBaseline?: typeof assertStrictReleaseEvidence, ) { const root = resolve(input.repositoryRoot ?? process.cwd()); const bytes = await readFile(input.path); @@ -303,7 +370,7 @@ export async function assertPatchReleaseEvidence( if (hash(await readFile(resolve(root, entry.path))) !== entry.sha256) throw new Error("Reviewed guidance content changed."); } - const baseline = await verifyBaseline({ + const baselineInput = { version: record.baseline.artifact.packageVersion, expectedArtifact: record.baseline.artifact, canaryPath: record.baseline.canary, @@ -311,8 +378,13 @@ export async function assertPatchReleaseEvidence( // Retained: this seal is prior evidence the patch cites, never a // measurement of the candidate, so its canary window has already served // its purpose for the release that recorded it. - freshness: "retained", - }); + freshness: "retained" as const, + }; + const baseline = verifyBaseline + ? await verifyBaseline(baselineInput) + : record.startup + ? await verifyStartupBaseline(root, record) + : await assertStrictReleaseEvidence(baselineInput); if (baseline.bundleSha256 !== record.baseline.bundleSha256) throw new Error("Baseline qualification bundle changed."); const providers = new Set( diff --git a/tests/patch-release.test.ts b/tests/patch-release.test.ts index 406052d3..20cb5c96 100644 --- a/tests/patch-release.test.ts +++ b/tests/patch-release.test.ts @@ -119,6 +119,28 @@ async function fixture(startup = false) { ); await writeFile(join(root, "src/index.ts"), "export const fixed = true;\n"); await writeFile(join(root, "skills/flow-run/SKILL.md"), "original"); + if (startup) { + await mkdir(join(root, "scripts")); + await mkdir(join(root, "node_modules")); + await writeFile( + join(root, "scripts/release-metadata.ts"), + ` +import { readFile } from "node:fs/promises"; +export async function assertStrictReleaseEvidence(input) { + const { version } = JSON.parse(await readFile("package.json", "utf8")); + if (version !== "10.0.2" || input.version !== version || input.freshness !== "retained") + throw new Error("Must verify the released baseline, never the candidate"); + if (process.env.FLOW_TEST_PARENT_ONLY) throw new Error("Parent environment leaked"); + return { bundleSha256: ${JSON.stringify(hash("bundle"))}, summary: { + schemaVersion: 1, packageVersion: version, reportId: "retained", verdict: "VERIFIED", + bundleSha256: ${JSON.stringify(hash("bundle"))}, artifact: input.expectedArtifact, + canarySha256: ${JSON.stringify(hash("canary"))}, totals: { scheduled: 81, scored: 81, passed: 80 }, + providers: [{ provider: "openai", scheduled: 81, scored: 81, passed: 80, passRate: 80/81 }] + }}; +} +`, + ); + } await git(root, "add", "."); await git(root, "commit", "-m", "full baseline"); const commit = await git(root, "rev-parse", "HEAD"); @@ -266,6 +288,27 @@ test("exact startup patch verifies the retained OpenAI baseline without claiming }), ).rejects.toThrow("two-provider baseline"); }); +test("startup baseline uses the released tag's verifier with isolated environment and cleans its worktree", async () => { + const f = await fixture(true); + await writeFile( + join(f.root, "scripts/release-metadata.ts"), + "throw new Error('candidate verifier must not run');\n", + ); + await git(f.root, "add", "."); + await git(f.root, "commit", "-m", "candidate authority differs"); + const prior = process.env.FLOW_TEST_PARENT_ONLY; + process.env.FLOW_TEST_PARENT_ONLY = "fixture-value"; + try { + const result = await assertPatchReleaseEvidence(f.input); + expect(result.notes).toContain("80/81 passed"); + } finally { + if (prior === undefined) delete process.env.FLOW_TEST_PARENT_ONLY; + else process.env.FLOW_TEST_PARENT_ONLY = prior; + } + expect(await git(f.root, "worktree", "list", "--porcelain")).not.toContain( + "flow-startup-baseline-", + ); +}); test("startup approval rejects changed baseline or candidate code, extra runtime paths and script edits", async () => { const f = await fixture(true); const startup = f.record.startup;