diff --git a/.dockerignore b/.dockerignore index 017a93543..5b1b5dfcb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,3 +11,9 @@ packages/ai/src/providers/data packages/coding-agent/binaries *.log *.tsbuildinfo + +.env +.env.* +**/.env +**/.env.* +**/__pycache__ diff --git a/.github/workflows/runtime-image.yml b/.github/workflows/runtime-image.yml index 02823b44e..e9cfc14ef 100644 --- a/.github/workflows/runtime-image.yml +++ b/.github/workflows/runtime-image.yml @@ -1,26 +1,6 @@ name: Runtime Image on: - push: - branches: [main] - paths: - - '.github/workflows/runtime-image.yml' - - '.dockerignore' - - 'Dockerfile.runtime' - - 'package.json' - - 'package-lock.json' - - 'packages/**' - - 'scripts/**' - pull_request: - branches: [main] - paths: - - '.github/workflows/runtime-image.yml' - - '.dockerignore' - - 'Dockerfile.runtime' - - 'package.json' - - 'package-lock.json' - - 'packages/**' - - 'scripts/**' workflow_dispatch: permissions: @@ -28,40 +8,57 @@ permissions: concurrency: group: runtime-image-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: false jobs: - build: - # Publication is paused until the runtime image and deployment consumers are validated. - if: ${{ false }} - runs-on: ubuntu-latest - permissions: - contents: read - packages: write + validate: + runs-on: ubuntu-24.04 + timeout-minutes: 25 steps: - - name: Checkout + - name: Checkout candidate uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - + with: + persist-credentials: false + - name: Resolve candidate identity + id: source + run: echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - name: Set up Docker Buildx uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - - - name: Log in to GitHub Container Registry - if: github.event_name != 'pull_request' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and publish immutable runtime image + - name: Prepare private evidence directory + run: mkdir -p .artifacts/runtime + - name: Build Linux amd64 candidate locally uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: context: . file: Dockerfile.runtime platforms: linux/amd64 - push: ${{ github.event_name != 'pull_request' }} - tags: ghcr.io/diffexai/diffex-runtime:sha-${{ github.sha }} + load: true + push: false + tags: diffex-runtime:candidate + build-args: SOURCE_COMMIT=${{ steps.source.outputs.commit }} cache-from: type=gha,scope=diffex-runtime cache-to: type=gha,mode=max,scope=diffex-runtime - provenance: mode=max - sbom: true + # Docker's local exporter cannot retain attestations. + # GHCR publication requires a separate reviewed OCI attestation gate. + provenance: false + sbom: false + - name: Validate hosted process without external networking + run: | + docker run --rm --network none \ + --mount "type=bind,src=$GITHUB_WORKSPACE/scripts/fixtures,dst=/acceptance,readonly" \ + --entrypoint node diffex-runtime:candidate /acceptance/runtime-smoke.mjs \ + > .artifacts/runtime/smoke.json + - name: Inspect every candidate image layer + run: | + docker save --output .artifacts/runtime/runtime.tar diffex-runtime:candidate + python3 scripts/check-runtime-image.py .artifacts/runtime/runtime.tar > .artifacts/runtime/payload.json + docker inspect diffex-runtime:candidate > .artifacts/runtime/image.json + sha256sum .artifacts/runtime/runtime.tar > .artifacts/runtime/SHA256SUMS + - name: Retain tested image and private evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: runtime-candidate-${{ steps.source.outputs.commit }} + path: .artifacts/runtime/ + if-no-files-found: error + retention-days: 14 diff --git a/AGENTS.md b/AGENTS.md index 19dd77621..1bd35aeb1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -171,3 +171,9 @@ If the user's instructions conflict with any rule in this document, ask for expl - 2026-09-13: Node rejects package import targets containing node_modules -> Preserve existing package import targets and validate staged tarballs through installed CLI startup before publication. - 2026-09-13: npm accepted publication before registry metadata propagated -> Preserve the uploaded artifact identity and resume verification without republishing or rebuilding. + +- 2026-09-13: Guessed release helper paths and reused npm caches obscured validation -> Discover paths before reading and use a fresh cache when proving that an update downloads artifact bytes. + +- 2026-09-13: Assumed interrupted npm downloads preserve the prefix -> Test real npm processes, retain user data, and recover ENOTEMPTY by matching the affected installation or backup against actual isolated directories instead of assuming every failure is a backup rename. + +- 2026-09-13: Runtime staging copied a redundant project NOTICE after attribution was consolidated -> Generate only the consolidated attribution file and reject separate NOTICE payloads. diff --git a/Dockerfile.runtime b/Dockerfile.runtime index e06993a6e..95819e284 100644 --- a/Dockerfile.runtime +++ b/Dockerfile.runtime @@ -1,28 +1,42 @@ -FROM node:24-bookworm-slim AS build +FROM node:24.16.0-bookworm-slim@sha256:2c87ef9bd3c6a3bd4b472b4bec2ce9d16354b0c574f736c476489d09f560a203 AS build WORKDIR /opt/diffex -COPY package.json package-lock.json ./ -COPY tsconfig.base.json ./ +ARG SOURCE_COMMIT +ENV DIFFEX_SOURCE_COMMIT=$SOURCE_COMMIT + +COPY package.json package-lock.json tsconfig.base.json ./ +COPY LICENSE ./ +COPY legal ./legal +COPY docs/release/distribution-policy.json ./docs/release/distribution-policy.json COPY packages ./packages COPY scripts ./scripts RUN npm ci --ignore-scripts -RUN npm run build +RUN npm run hydrate:model-data && npm run build:offline RUN npm prune --omit=dev --ignore-scripts +RUN node scripts/stage-runtime-distribution.mjs /opt/runtime -FROM node:24-bookworm-slim AS runtime +FROM node:24.16.0-bookworm-slim@sha256:2c87ef9bd3c6a3bd4b472b4bec2ce9d16354b0c574f736c476489d09f560a203 AS runtime RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates \ + && sed -i "s|http://deb.debian.org|https://deb.debian.org|g" /etc/apt/sources.list.d/debian.sources \ + && apt-get update \ && apt-get install -y --no-install-recommends ca-certificates fd-find git openssh-client ripgrep \ && ln -s /usr/bin/fdfind /usr/local/bin/fd \ && rm -rf /var/lib/apt/lists/* -COPY --from=build /opt/diffex /opt/diffex +COPY --from=build /opt/runtime /opt/diffex RUN ln -s /opt/diffex/packages/coding-agent-server/dist/cli.js /usr/local/bin/diffex-runtime-server WORKDIR /workspace +ARG SOURCE_COMMIT LABEL org.opencontainers.image.source="https://github.com/diffexai/diffex" -LABEL org.opencontainers.image.description="Pi-based hosted Diffex coding-agent runtime" +LABEL org.opencontainers.image.revision=$SOURCE_COMMIT +LABEL org.opencontainers.image.licenses="Apache-2.0 AND MIT" +LABEL org.opencontainers.image.description="Hosted Diffex coding-agent runtime" + +CMD ["diffex-runtime-server"] diff --git a/docs/cloud-runtime.md b/docs/cloud-runtime.md index 5c03552e3..2626023bc 100644 --- a/docs/cloud-runtime.md +++ b/docs/cloud-runtime.md @@ -24,10 +24,11 @@ Only Daytona's opaque placeholder enters the sandbox environment, and the real p ## Runtime Image -`Dockerfile.runtime` produces the AMD64 image used to create the Daytona sandbox snapshot. -The runtime-image workflow is disabled during the identity migration, including manual dispatches. -Re-enable the workflow for validation and immutable `ghcr.io/diffexai/diffex-runtime:sha-` publication after reviewing the image and its consumers. -Keep production on its existing tested image and Daytona snapshot until the replacement image and snapshot pass release validation. +`Dockerfile.runtime` produces a Linux amd64 candidate with a digest-pinned Node base image, compiled workspace outputs, user documentation, and legal notices. +The Runtime Image workflow now permits manual validation and retains the tested image privately. +Registry publication and automatic main-branch publication remain disabled. +The owner confirmed on 2026-09-13 that no staging deployment or Daytona consumer is configured, so no existing production digest or snapshot can be assumed. +See [runtime validation and promotion prerequisites](release/runtime-validation.md) for local commands and the pending hosted acceptance. Before a cloud release, register GHCR with Daytona, create a Daytona snapshot from that exact immutable image tag or digest, and set `DAYTONA_SANDBOX_SNAPSHOT` to the resulting snapshot name. Do not use a floating image tag for production. diff --git a/docs/release/artifact-acceptance.md b/docs/release/artifact-acceptance.md index d4b68059b..9fdfee1e1 100644 --- a/docs/release/artifact-acceptance.md +++ b/docs/release/artifact-acceptance.md @@ -28,6 +28,21 @@ Acceptance executes file discovery and content search, records their versions, a Native runners install those prerequisites through apt or Homebrew before package acceptance. This validates the npm installation with controlled system helpers; it does not establish the separate automatic-helper-download lifecycle. +## Installed lifecycle acceptance + +The acceptance command also runs `scripts/npm-lifecycle.mjs` against the exact reviewed CLI tarball. +It installs globally into an isolated prefix, checks PATH, version, help, and models, then invokes the installed self-updater with real npm subprocesses. +Only discovery and the Diffex-scoped registry are controlled fixtures; dependency downloads use npmjs.org. +An isolated version-metadata predecessor exercises upgrade to the reviewed candidate without publishing a fake release. +Fresh caches require actual target downloads. +Malformed and foreign metadata, redirects, outages, no release, and checksum mismatches are rejected without changing the installed prefix, including force. +An interrupted npm download demonstrates the documented pinned repair and, when npm reports `ENOTEMPTY`, relocation of the affected CLI installation or its exact stale backup before retrying. +Interruption itself can mutate npm's prefix and is not claimed to preserve the executable. +Uninstall verifies executable removal and retained global and project data. +The same harness verifies Bun global installation and PATH, rejects a foreign release identity without mutation, upgrades with the real Bun subprocess, checks the installed runtime hashes, and uninstalls while retaining user and project data. +An isolated Bun global scope registry ensures that the upgrade receives the reviewed candidate bytes. +`npm-lifecycle.json` records commands, identities, results, limitations, and cleanup. + ## Supported distribution scope The approved channel remains npm, installed with npm or Bun and executed on Node 22.19.0 or newer. diff --git a/docs/release/distribution-policy.json b/docs/release/distribution-policy.json index cf7ffd738..e36671f26 100644 --- a/docs/release/distribution-policy.json +++ b/docs/release/distribution-policy.json @@ -35,7 +35,7 @@ "HTML export templates and their reviewed marked/highlight.js bundles", "runtime theme and model JSON", "SQLite migration SQL", - "documentation including inline examples", + "user-facing documentation explicitly listed in scripts/distribution-docs.json", "Diffex-owned TUI native prebuilds using system APIs" ], "publicHosting": "npmjs", @@ -55,5 +55,6 @@ "protocol", "telemetry", "tui" - ] + ], + "developmentDocumentation": "exclude from all distribution artifacts, including bootstrap and hosted runtime; keep private development records in the source repository" } diff --git a/docs/release/runtime-validation.md b/docs/release/runtime-validation.md new file mode 100644 index 000000000..997f6784d --- /dev/null +++ b/docs/release/runtime-validation.md @@ -0,0 +1,74 @@ +# Hosted runtime candidate validation + +The owner confirmed on 2026-09-13 that no staging deployment or Daytona consumer is configured. +Local runtime validation is the available acceptance boundary. +It does not promote a hosted deployment or establish a prior-runtime rollback baseline. +The release remains npm-only; this image is not an interactive CLI installation channel. + +## Build and inspect locally + +Run from the private source checkout with Docker running: + +```sh +DIFFEX_RUNTIME_COMMIT="$(git rev-parse HEAD)" +docker build --platform linux/amd64 \ + --build-arg "SOURCE_COMMIT=$DIFFEX_RUNTIME_COMMIT" \ + --file Dockerfile.runtime --tag diffex-runtime:candidate . +docker run --rm --platform linux/amd64 --network none \ + --mount "type=bind,src=$PWD/scripts/fixtures,dst=/acceptance,readonly" \ + --entrypoint node diffex-runtime:candidate /acceptance/runtime-smoke.mjs +docker save --output /tmp/diffex-runtime-candidate.tar diffex-runtime:candidate +python3 scripts/check-runtime-image.py /tmp/diffex-runtime-candidate.tar +docker inspect diffex-runtime:candidate +shasum -a 256 /tmp/diffex-runtime-candidate.tar +``` + +Use a clean committed source for retained acceptance and record its full commit. +Dirty-checkout results must also identify the uncommitted diff and cannot be attributed solely to the base commit label. +The Dockerfile pins the Node 24.16.0 base manifest digest and uses Linux amd64. +It hydrates validated model data, compiles workspace outputs, prunes development dependencies, and stages the required hosted runtime dependency closure. +The image carries compiled files, native runtime assets, the explicit user-documentation allowlist, and legal notices. +Project build inputs and private development documentation stay in the build stage. +Third-party redistribution materials remain with dependencies. + +The fixture is mounted read-only for testing and never copied into a distribution layer. +It launches the actual server executable, uses a mock model provider and usage endpoint, verifies anonymous and incorrect-credential denial, rejects another session ID, exchanges a prompt and reply, and restarts using persisted session data. +External networking is disabled by Docker. +The test removes its directories and the container exits cleanly. +The layer checker examines every saved image layer so deleting an internal document in a later layer cannot hide its inclusion. + +The manually dispatched Runtime Image workflow runs the same build, smoke, and layer checks on Linux amd64. +It retains the tested Docker archive, local image ID, archive checksum, and evidence in private Actions storage for 14 days. +It has read-only repository permissions and no registry publishing job. +There is no automatic main-branch image publication. + +## Local evidence + +On 2026-09-13, the development candidate passed the Linux amd64 build and smoke on Docker running on Darwin arm64. +The initial local image ID was `sha256:cfd4495ec712460a1400a1f6dda7f4352b7e48a659bed79505cf0c2f3714aad9`. +Its source label identifies base commit `06830a99220c3dad14292830171c433c916d49b1`; the branch's uncommitted runtime and packaging changes were also build inputs. +Layer inspection accepted 1,426 project files across 12 runtime workspaces and all 30 approved documentation assets. +This development image is not a registry release digest or a promoted artifact. +Final committed-candidate workflow evidence belongs in the implementation PR. + +The npm lifecycle harness also passed on Darwin arm64 with Node 22.19.0, npm 10.9.3, and Bun 1.3.11. +It used the local candidate tarball, real npm installation processes, controlled release discovery, and a loopback registry for the canonical package. +Malformed release inputs and checksum mismatch preserved the installed prefix. +An interrupted download changed npm's prefix and required moving the exact stale backup before a pinned repair succeeded. +Bun global installation, PATH, real self-update, installed-byte verification, foreign-identity rejection, and removal also passed. +Removal retained user and project data, and all test-owned directories were removed. +Its version-metadata predecessor is a simulation; public-to-public upgrade acceptance remains separate. + +## Registry and deployment prerequisites + +Keep the future image private at `ghcr.io/diffexai/diffex-runtime` and configure scoped publisher and pull identities before publication. +Review the base-image and dependency license inventory, generate OCI SBOM and provenance attestations, and retain the image's registry manifest digest with its exact source and model-data inputs. +Docker's local exporter does not retain attestations, so the local archive does not claim SBOM/provenance acceptance. +Use a registry or OCI exporter for attested artifacts as described in [Docker's attestation documentation](https://docs.docker.com/build/metadata/attestations/). + +The first hosted acceptance must pull the exact candidate from the intended staging environment, create a Daytona snapshot from its digest, and configure the cloud consumer explicitly. +Record the cloud commit, runtime commit, registry digest, snapshot, deployment owner, architecture, pull identity, and approved credential source without credential values. +Exercise real cloud allocation, attachment, streaming, reconnect, stop, cross-actor denial, and cleanup. +Only then retain the tested digest and snapshot as the first recovery baseline. +Every replacement must keep its previous tested configuration available, demonstrate restoration, and verify cleanup before retirement. +Registry publication, attestations, real provider inference, deployment promotion, and prior-runtime rollback are not completed by local testing. diff --git a/packages/coding-agent/docs/installation.md b/packages/coding-agent/docs/installation.md new file mode 100644 index 000000000..cb9b44a90 --- /dev/null +++ b/packages/coding-agent/docs/installation.md @@ -0,0 +1,118 @@ +# Install, update, and remove Diffex + +Diffex is distributed through npmjs.org as `@diffexai/diffex`. +Use Node.js 22.19.0 or newer and npm on macOS or Linux, on arm64 or x64. +Windows, Android/Termux, standalone binaries, and additional installation channels are not accepted release targets. +The hosted runtime image is for the cloud service. + +## Install and verify + +```sh +node --version +npm --version +npm install -g --ignore-scripts --registry=https://registry.npmjs.org @diffexai/diffex +diffex --version +diffex --help +diffex --list-models +``` + +Only the CLI package is needed; its internal Diffex modules are included. +`--ignore-scripts` disables dependency installation scripts. +Npm verifies downloaded package bytes against registry integrity metadata. +For a reproducible installation, replace the package argument with `@diffexai/diffex@`, using an exact version you have checked on npmjs.org. + +The executable directory is `$(npm prefix -g)/bin` on supported platforms. +If `diffex` is not found, add that directory to your shell's PATH: + +```sh +export PATH="$(npm prefix -g)/bin:$PATH" +command -v diffex +diffex --version +``` + +Persist the PATH setting in your shell configuration if necessary. +Use a Node version manager or a user-writable npm prefix when the global directory is not writable. +Changing Node installations can change the npm prefix; check `command -v diffex` after switching. + +Start `diffex` in your project directory, use `/login` to configure your provider, and send a short prompt. +See [Quickstart](quickstart.md) and [Providers](providers.md) for authentication and execution. + +## Upgrade + +```sh +diffex update --self +diffex --version +``` + +Self-update validates the response from `https://api.diffex.ai/api/latest-version` and installs exactly `@diffexai/diffex@` using the detected package manager. +It rejects malformed metadata, foreign package names, redirects, and unavailable discovery before starting installation. +`--force` permits reinstalling a verified release; it does not bypass identity checks. +The discovery service can report no generally announced release even when a bootstrap package exists on npm. + +To install a reviewed npm version directly, including when discovery has no release: + +```sh +npm install -g --ignore-scripts --registry=https://registry.npmjs.org '@diffexai/diffex@' +diffex --version +``` + +Replace `` before running the command. +Use your original npm prefix if it differs from the active default. +Legacy clients that still trust Pi cannot migrate just by contacting the Diffex endpoint. +Install the canonical package directly with the command above, then verify `command -v diffex` and `diffex --version`. + +## Recover an interrupted installation + +Npm owns replacement of an npm installation. +If an update was interrupted or `diffex` no longer starts, repeat the pinned npm install command with the same verified version and prefix, then check `diffex --version` and `diffex --help`. +Do not remove your user-data directory to repair an executable installation. +Npm can begin filesystem replacement before a download finishes. +If retrying reports `ENOTEMPTY` while renaming to a `.diffex-...` backup, move that reported backup directory aside. +If npm instead reports `ENOTEMPTY` while removing a dependency inside `$(npm root -g)/@diffexai/diffex`, move that complete CLI installation directory aside. +This preserves the partially installed package and lets npm install into an empty destination. +Use a separate backup directory and retry: + +```sh +DIFFEX_RECOVERY_FROM='/exact/affected/CLI/installation/or/backup' +DIFFEX_RECOVERY_DIR="$(mktemp -d)" +mv "$DIFFEX_RECOVERY_FROM" "$DIFFEX_RECOVERY_DIR/" +npm install -g --ignore-scripts --registry=https://registry.npmjs.org '@diffexai/diffex@' +diffex --version +diffex --help +``` + +Replace both placeholders with the affected Diffex installation or backup path and the verified version. +Move only the `@diffexai/diffex` package directory or its reported `.diffex-...` backup, preserving it until the repaired installation works. +Do not move the global prefix, its shared `node_modules`, or another package. +This procedure does not move or delete your settings or sessions. + +## Remove + +```sh +npm uninstall -g --ignore-scripts @diffexai/diffex +command -v diffex +``` + +Use the npm prefix that installed the executable. +The final command should find no executable unless another installation remains on PATH. +If your shell caches command locations, open a new terminal or refresh its command cache. + +Removal retains settings, credentials, sessions, installed extensions, and harness data under `~/.diffex/agent/`, or the directory selected by `DIFFEX_CODING_AGENT_DIR`. +Project `.diffex` directories and any separately configured session directory are also retained. +Keep this data for reinstallation; delete it separately only if you intend to erase it. + +## Bun package installations + +The same npm package can also be installed by Bun while running on Node: + +```sh +bun install -g --ignore-scripts @diffexai/diffex +diffex --version +diffex update --self +bun uninstall -g @diffexai/diffex +``` + +Use `bun pm bin -g` to locate Bun's global executable directory and add it to PATH. +Pin direct upgrades to the reviewed `@diffexai/diffex@` in the install command. +Use the manager that owns the installation for repair and removal. +User data follows the same retention policy. diff --git a/packages/coding-agent/docs/quickstart.md b/packages/coding-agent/docs/quickstart.md index fc4d6e09c..6c9849602 100644 --- a/packages/coding-agent/docs/quickstart.md +++ b/packages/coding-agent/docs/quickstart.md @@ -4,33 +4,15 @@ This page gets you from install to a useful first Diffex session. ## Install -Diffex is distributed as an npm package: +Use Node.js 22.19.0 or newer on macOS or Linux, on arm64 or x64. +Install the canonical package from npmjs.org: ```bash -npm install -g --ignore-scripts @diffexai/diffex +npm install -g --ignore-scripts --registry=https://registry.npmjs.org @diffexai/diffex +diffex --version ``` -`--ignore-scripts` disables dependency lifecycle scripts during install. Diffex does not require install scripts for normal npm installs. - -### Uninstall - -Use the package manager that installed Diffex: - -```bash -# npm -npm uninstall -g @diffexai/diffex - -# pnpm -pnpm remove -g @diffexai/diffex - -# Yarn -yarn global remove @diffexai/diffex - -# Bun -bun uninstall -g @diffexai/diffex -``` - -Uninstalling Diffex leaves settings, credentials, sessions, and installed Diffex packages in `~/.diffex/agent/`. +See [Installation](installation.md) for PATH setup, pinned versions, updates, recovery, removal, and retained user data. Then start Diffex in the project directory you want it to work on: @@ -175,4 +157,5 @@ Use `--mode json` for JSON event output or `--mode rpc` for process integration. - [Keybindings](keybindings.md) - shortcuts and customization. - [Diffex Packages](packages.md) - install shared extensions, skills, prompts, and themes. -Platform notes: [Windows](windows.md), [Termux](termux.md), [tmux](tmux.md), [Terminal setup](terminal-setup.md), [Shell aliases](shell-aliases.md). +Terminal notes: [tmux](tmux.md), [Terminal setup](terminal-setup.md), [Shell aliases](shell-aliases.md). +Windows and Android/Termux are not accepted release targets. diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index 02c527050..6035cd036 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -30,11 +30,39 @@ }, "files": [ "dist", - "docs", + "docs/compaction.md", + "docs/containerization.md", + "docs/custom-provider.md", + "docs/environment-variables.md", + "docs/evolution.md", + "docs/extensions.md", + "docs/installation.md", + "docs/json.md", + "docs/keybindings.md", + "docs/llama-cpp.md", + "docs/models.md", + "docs/packages.md", + "docs/prompt-templates.md", + "docs/providers.md", + "docs/quickstart.md", + "docs/rpc.md", + "docs/sdk.md", + "docs/security.md", + "docs/session-format.md", + "docs/sessions.md", + "docs/settings.md", + "docs/shell-aliases.md", + "docs/skills.md", + "docs/terminal-setup.md", + "docs/themes.md", + "docs/tmux.md", + "docs/tui.md", + "docs/usage.md", + "docs/images/interactive-mode.png", + "docs/images/tree-view.png", "CHANGELOG.md", "npm-shrinkwrap.json", "LICENSE", - "NOTICE", "THIRD-PARTY-NOTICES.txt", "distribution-components.json", "distribution-files.json", @@ -45,7 +73,7 @@ "build": "tsgo -p tsconfig.build.json && shx chmod +x dist/cli.js dist/rpc-entry.js && npm run copy-assets", "build:binary": "node -e \"throw new Error('Standalone distribution is deferred pending runtime redistribution review')\"", "copy-assets": "shx mkdir -p dist/modes/interactive/theme && shx cp src/modes/interactive/theme/*.json dist/modes/interactive/theme/ && shx mkdir -p dist/core/export-html/vendor && shx cp src/core/export-html/template.html src/core/export-html/template.css src/core/export-html/template.js dist/core/export-html/ && shx cp src/core/export-html/vendor/*.js dist/core/export-html/vendor/ && shx rm -rf dist/vendor && shx mkdir -p dist/vendor && shx cp -r ../harness-trace/dist dist/vendor/harness-trace && shx cp -r ../evolution/dist dist/vendor/evolution", - "copy-binary-assets": "shx cp package.json dist/ && shx cp README.md dist/ && shx cp CHANGELOG.md dist/ && shx mkdir -p dist/theme && shx cp src/modes/interactive/theme/*.json dist/theme/ && shx mkdir -p dist/export-html/vendor && shx cp src/core/export-html/template.html dist/export-html/ && shx cp src/core/export-html/vendor/*.js dist/export-html/vendor/ && shx cp -r docs dist/ && shx cp ../../node_modules/@silvia-odwyer/photon-node/photon_rs_bg.wasm dist/ && shx cp LICENSE NOTICE THIRD-PARTY-NOTICES.txt distribution-components.json dist/", + "copy-binary-assets": "shx cp package.json dist/ && shx cp README.md dist/ && shx cp CHANGELOG.md dist/ && shx mkdir -p dist/theme && shx cp src/modes/interactive/theme/*.json dist/theme/ && shx mkdir -p dist/export-html/vendor && shx cp src/core/export-html/template.html dist/export-html/ && shx cp src/core/export-html/vendor/*.js dist/export-html/vendor/ && node ../../scripts/copy-distribution-docs.mjs . dist && shx cp ../../node_modules/@silvia-odwyer/photon-node/photon_rs_bg.wasm dist/ && shx cp LICENSE THIRD-PARTY-NOTICES.txt distribution-components.json dist/", "test": "vitest --run", "shrinkwrap": "node ../../scripts/generate-coding-agent-shrinkwrap.mjs", "prepublishOnly": "npm run clean && npm run build && npm run shrinkwrap", diff --git a/scripts/artifact-acceptance.mjs b/scripts/artifact-acceptance.mjs index c2b4cfd0a..b48f92533 100644 --- a/scripts/artifact-acceptance.mjs +++ b/scripts/artifact-acceptance.mjs @@ -320,6 +320,17 @@ export function acceptArtifacts(artifactDirectory, outputDirectory, { managers = helpers, }); } + const cli = records.find((record) => record.name === "@diffexai/diffex"); + execFileSync( + process.execPath, + [ + join(root, "scripts/npm-lifecycle.mjs"), + join(tarballDirectory, cli.filename), + outputDirectory, + ...(managers.includes("bun") ? [] : ["--skip-bun"]), + ], + { cwd: root, stdio: "inherit", timeout: 600000 }, + ); verifyTarballs(tarballDirectory, records, packages); writeFileSync( join(outputDirectory, "SHA256SUMS"), diff --git a/scripts/check-distribution-payload.py b/scripts/check-distribution-payload.py index f4f299df5..b2af21b8c 100644 --- a/scripts/check-distribution-payload.py +++ b/scripts/check-distribution-payload.py @@ -15,6 +15,9 @@ from pathlib import Path, PurePosixPath +DOCUMENTATION = set(json.loads(Path(__file__).with_name('distribution-docs.json').read_text())) +LEGAL_DOCUMENTS = {'LICENSE', 'THIRD-PARTY-NOTICES.txt'} + MAX_BYTES = 1024 * 1024 * 1024 MAX_FILES = 100000 MAX_DEPTH = 8 @@ -40,6 +43,17 @@ def check_path(name, directory=False): fail(name, 'prohibited source or private path') if any(part.lower() == 'agents.md' for part in path.parts): fail(name, 'prohibited repository instructions') + if not directory and path.name.upper() == 'NOTICE': + fail(name, 'separate NOTICE file is excluded; attribution belongs in THIRD-PARTY-NOTICES.txt') + if not directory: + if 'docs' in path.parts: + local = '/'.join(path.parts[path.parts.index('docs'):]) + if local not in DOCUMENTATION: + fail(name, 'unapproved distribution documentation') + elif path.suffix.lower() in {'.md', '.mdx', '.rst', '.txt', '.pdf'} and path.name not in LEGAL_DOCUMENTS: + # Only the reviewed public README and release history belong at the distribution root. + if not (path.name in {'README.md', 'CHANGELOG.md'} and len(path.parts) <= 3 and 'dist' not in path.parts): + fail(name, 'unapproved distribution documentation') declaration = name.endswith(('.d.ts', '.d.mts', '.d.cts')) if path.suffix in SOURCE_SUFFIXES and not declaration: fail(name, 'prohibited source file') @@ -150,6 +164,9 @@ def inspect_archive(file, license_file, expected_name=None, expected_version=Non if hashlib.sha256(entries[root + item['path']]).hexdigest() != item['sha256']: fail(str(file), 'runtime file hash mismatch') if metadata.get('name') == '@diffexai/diffex': + for document in DOCUMENTATION | {'README.md'}: + if not entries.get(root + document, b'').strip(): + fail(str(file), 'missing required user documentation: ' + document) for field in ['dependencies', 'optionalDependencies', 'peerDependencies']: if any(name.startswith('@diffexai/') for name in metadata.get(field, {})): fail(str(file), 'CLI must not fetch separate Diffex packages') diff --git a/scripts/check-runtime-image.py b/scripts/check-runtime-image.py new file mode 100644 index 000000000..100be0180 --- /dev/null +++ b/scripts/check-runtime-image.py @@ -0,0 +1,78 @@ +"""Inspect project payloads in every saved Docker image layer, including deleted files.""" +import argparse +import importlib.util +import json +import tarfile +import sys +from pathlib import Path, PurePosixPath + +sys.dont_write_bytecode = True + +spec = importlib.util.spec_from_file_location('payload', Path(__file__).with_name('check-distribution-payload.py')) +payload = importlib.util.module_from_spec(spec) +spec.loader.exec_module(payload) + + +def inspect_image(filename): + documents = set() + workspaces = set() + notices = set() + files = 0 + identity = None + with tarfile.open(filename) as image: + manifest = json.load(image.extractfile('manifest.json')) + if len(manifest) != 1: + raise ValueError('Expected exactly one runtime image') + config = json.load(image.extractfile(manifest[0]['Config'])) + if config['architecture'] != 'amd64' or config['os'] != 'linux': + raise ValueError('Expected Linux amd64 runtime') + for layer in manifest[0]['Layers']: + with tarfile.open(fileobj=image.extractfile(layer), mode='r|') as archive: + for entry in archive: + name = entry.name.removeprefix('./') + if not name.startswith('opt/diffex/') or not entry.isfile(): + continue + if PurePosixPath(name).name.startswith('.wh.'): + continue + local = name.removeprefix('opt/diffex/') + if local.startswith('node_modules/'): + continue # Third-party redistribution notices remain with dependencies. + if local == 'runtime-distribution.json': + identity = json.load(archive.extractfile(entry)) + continue + if not local.startswith('packages/'): + if local not in {'README.md', 'LICENSE', 'THIRD-PARTY-NOTICES.txt'}: + raise ValueError(f'Unexpected project root file: {local}') + continue + parts = PurePosixPath(local).parts + offset = 3 if parts[1] == 'session-backends' else 2 + workspace = '/'.join(parts[:offset]) + relative = '/'.join(parts[offset:]) + if relative.startswith('node_modules/'): + continue + workspaces.add(workspace) + if relative not in {'package.json', 'LICENSE', 'THIRD-PARTY-NOTICES.txt', 'distribution-components.json', 'distribution-files.json'} and not relative.startswith(('dist/', 'native/', 'docs/')): + raise ValueError(f'Unexpected runtime workspace file: {local}') + content = archive.extractfile(entry).read() + payload.scan_bytes(content, 'package/' + relative, {'bytes': 0, 'files': 0}) + if relative.startswith('docs/'): + documents.add(relative) + if relative in {'LICENSE', 'THIRD-PARTY-NOTICES.txt'} and content.strip(): + notices.add((workspace, relative)) + files += 1 + if not identity or not workspaces or documents != payload.DOCUMENTATION: + raise ValueError('Missing runtime identity, workspaces, or approved user documentation') + for workspace in workspaces: + for notice in ['LICENSE', 'THIRD-PARTY-NOTICES.txt']: + if (workspace, notice) not in notices: + raise ValueError(f'Missing runtime notice: {workspace}/{notice}') + labels = config['config'].get('Labels', {}) + if labels.get('org.opencontainers.image.revision') != identity['sourceCommit']: + raise ValueError('Runtime source identity mismatch') + return {'status': 'passed', 'projectFiles': files, 'workspaces': len(workspaces), 'documentation': sorted(documents), 'sourceCommit': identity['sourceCommit'], 'architecture': config['architecture']} + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('image') + print(json.dumps(inspect_image(parser.parse_args().image))) diff --git a/scripts/cli-distribution.mjs b/scripts/cli-distribution.mjs index e40b44b20..9b03d47cb 100644 --- a/scripts/cli-distribution.mjs +++ b/scripts/cli-distribution.mjs @@ -1,7 +1,12 @@ import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; import { join, relative } from "node:path"; import { findPackageDirectories } from "./package-workspaces.mjs"; +export const distributionDocs = JSON.parse( + readFileSync(fileURLToPath(new URL("./distribution-docs.json", import.meta.url)), "utf8"), +); + export function cliDistribution(root = ".") { const workspaces = new Map( findPackageDirectories(join(root, "packages")) @@ -58,9 +63,18 @@ export function cliDistribution(root = ".") { manifest.diffexConfig = { ...manifest.diffexConfig, production: true }; delete manifest.repository; delete manifest.bugs; - manifest.files = manifest.files.filter( - (file) => !["notice", "contributing.md"].includes(file.split("/")[0].toLowerCase()), - ); + manifest.files = [ + "dist", + ...distributionDocs, + "README.md", + "CHANGELOG.md", + "npm-shrinkwrap.json", + "LICENSE", + "THIRD-PARTY-NOTICES.txt", + "distribution-components.json", + "distribution-files.json", + "release", + ]; manifest.dependencies = Object.fromEntries(Object.entries(dependencies).sort(([a], [b]) => a.localeCompare(b))); manifest.optionalDependencies = Object.fromEntries( Object.entries(optionalDependencies).sort(([a], [b]) => a.localeCompare(b)), diff --git a/scripts/copy-distribution-docs.mjs b/scripts/copy-distribution-docs.mjs new file mode 100644 index 000000000..9235f2d4c --- /dev/null +++ b/scripts/copy-distribution-docs.mjs @@ -0,0 +1,10 @@ +import { cpSync, mkdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { distributionDocs } from "./cli-distribution.mjs"; + +const [source, target] = process.argv.slice(2); +if (!source || !target) throw new Error("Usage: copy-distribution-docs.mjs SOURCE TARGET"); +for (const file of distributionDocs) { + mkdirSync(dirname(join(target, file)), { recursive: true }); + cpSync(join(source, file), join(target, file)); +} diff --git a/scripts/distribution-docs.json b/scripts/distribution-docs.json new file mode 100644 index 000000000..1b14ca6fa --- /dev/null +++ b/scripts/distribution-docs.json @@ -0,0 +1,32 @@ +[ + "docs/compaction.md", + "docs/containerization.md", + "docs/custom-provider.md", + "docs/environment-variables.md", + "docs/evolution.md", + "docs/extensions.md", + "docs/installation.md", + "docs/json.md", + "docs/keybindings.md", + "docs/llama-cpp.md", + "docs/models.md", + "docs/packages.md", + "docs/prompt-templates.md", + "docs/providers.md", + "docs/quickstart.md", + "docs/rpc.md", + "docs/sdk.md", + "docs/security.md", + "docs/session-format.md", + "docs/sessions.md", + "docs/settings.md", + "docs/shell-aliases.md", + "docs/skills.md", + "docs/terminal-setup.md", + "docs/themes.md", + "docs/tmux.md", + "docs/tui.md", + "docs/usage.md", + "docs/images/interactive-mode.png", + "docs/images/tree-view.png" +] diff --git a/scripts/distribution-payload.test.mjs b/scripts/distribution-payload.test.mjs index 779bf6de3..bd2d784f6 100644 --- a/scripts/distribution-payload.test.mjs +++ b/scripts/distribution-payload.test.mjs @@ -6,6 +6,7 @@ import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { test } from "node:test"; +import { distributionDocs } from "./cli-distribution.mjs"; import { createDistributionManifest } from "./release-distribution-manifest.mjs"; import { getPublicWorkspacePackages } from "./release-packages.mjs"; @@ -16,9 +17,10 @@ const version = getPublicWorkspacePackages()[0].version; function fixture(extra, omit) { const directory = mkdtempSync(join(tmpdir(), "diffex-payload-test-")); const files = { + ...Object.fromEntries(distributionDocs.map((path) => [path, "Reviewed user guide"])), + "README.md": "Diffex installation and usage", "package.json": JSON.stringify({ name: "@diffexai/diffex", version, license: "Apache-2.0" }), LICENSE: readFileSync(license), - NOTICE: "Diffex modifications copyright 2026 Luntraa, Inc. Pi-derived portions retain MIT terms in LICENSE.\n", "THIRD-PARTY-NOTICES.txt": "Test fixture with no external bundled dependencies.\n", "distribution-components.json": JSON.stringify([{ name: "Diffex", license: "Apache-2.0 AND MIT" }]), "dist/index.js": "export const value = 1;\n", @@ -72,7 +74,7 @@ test("accepts a tarball with readable JavaScript, declarations, and complete not "native/darwin/prebuilds/darwin-arm64/helper.node": Buffer.from("native fixture"), "dist/vendor/evolution/index.js": "export const value = 1;", "dist/vendor/harness-trace/index.d.ts": "export declare const value: number;", - "docs/context-files.md": "Diffex loads AGENTS.md from the user's project.\n", + "docs/usage.md": "Diffex loads AGENTS.md from the user's project.\n", }); assert.equal(result.status, 0, result.stderr); }); @@ -90,12 +92,17 @@ for (const notice of [ }); } -test("accepts consolidated attribution without a separate NOTICE file", () => { - const result = inspect({}, "NOTICE"); - assert.equal(result.status, 0, result.stderr); +test("rejects a separate NOTICE file because attribution is consolidated", () => { + const result = inspect({ NOTICE: "Duplicate project attribution" }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /separate NOTICE file/); }); for (const [name, content, reason] of [ + ["docs/development.md", "Private build instructions", /unapproved distribution documentation/], + ["docs/new-internal-audit.md", "Private evidence", /unapproved distribution documentation/], + ["dist/internal-handoff.txt", "Private handoff", /unapproved distribution documentation/], + ["dist/node_modules/@diffexai/diffex-ai/docs/audit.md", "Private audit", /unapproved distribution documentation/], ["dist/implementation.ts", "export const value = 1", /prohibited source file/], ["dist/renamed.js", "export interface Hidden {", /TypeScript implementation content/], ["dist/native.c", "int main() {}", /prohibited source file/], @@ -107,7 +114,11 @@ for (const [name, content, reason] of [ ["examples/example.ts", "export const value = 1", /prohibited source or private path/], ["AGENTS.md", "Private repository instructions", /prohibited repository instructions/], ["docs/agents.MD", "Private repository instructions", /prohibited repository instructions/], - ["dist/node_modules/@diffexai/diffex-ai/AGENTS.md", "Private repository instructions", /prohibited repository instructions/], + [ + "dist/node_modules/@diffexai/diffex-ai/AGENTS.md", + "Private repository instructions", + /prohibited repository instructions/, + ], ]) { test(`rejects prohibited content in ${name}`, () => { const result = inspect({ [name]: content }); @@ -117,7 +128,7 @@ for (const [name, content, reason] of [ } test("inspects a nested archive even when renamed to a binary resource", () => { - const nested = fixture({ "dist/private.ts": "export const hidden = 1" }); + const nested = fixture({ "dist/private.ts": "export const hidden = 1" }, "README.md"); try { const result = inspect({ "dist/resource.bin": readFileSync(nested.archive) }); assert.notEqual(result.status, 0); @@ -128,7 +139,7 @@ test("inspects a nested archive even when renamed to a binary resource", () => { }); test("rejects repository instructions inside a nested archive", () => { - const nested = fixture({ "docs/AGENTS.md": "Private repository instructions" }); + const nested = fixture({ "docs/AGENTS.md": "Private repository instructions" }, "README.md"); try { const result = inspect({ "dist/resource.bin": readFileSync(nested.archive) }); assert.notEqual(result.status, 0); @@ -178,3 +189,11 @@ test("rejects duplicate members before interpreting package metadata", () => { rmSync(directory, { recursive: true, force: true }); } }); + +for (const guide of ["README.md", "docs/installation.md"]) { + test(`rejects a CLI artifact missing required user documentation ${guide}`, () => { + const result = inspect({}, guide); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /missing required user documentation/); + }); +} diff --git a/scripts/fixtures/installed-update-guard.mjs b/scripts/fixtures/installed-update-guard.mjs new file mode 100644 index 000000000..fdf404df8 --- /dev/null +++ b/scripts/fixtures/installed-update-guard.mjs @@ -0,0 +1,28 @@ +// Private acceptance fixture; the runner prepends a static import of the installed HTTP dispatcher. +import { appendFileSync, readFileSync } from "node:fs"; +import childProcess from "node:child_process"; +import { syncBuiltinESMExports } from "node:module"; +import { Socket } from "node:net"; + +const record = (event) => appendFileSync(process.env.DIFFEX_LIFECYCLE_EVENTS, `${JSON.stringify(event)}\n`); +globalThis.fetch = async (input, init) => { + const url = String(input); + record({ type: "discovery", url, redirect: init?.redirect }); + if (url !== "https://api.diffex.ai/api/latest-version" || init?.redirect !== "manual") + throw new Error("Unexpected lifecycle discovery request"); + const fixture = JSON.parse(readFileSync(process.env.DIFFEX_LIFECYCLE_RESPONSE, "utf8")); + if (fixture.failure) throw new Error("Controlled discovery outage"); + return new Response(fixture.body, { + status: fixture.status, + headers: { "content-type": "application/json", ...fixture.headers }, + }); +}; +Socket.prototype.connect = function () { + throw new Error("Installed CLI attempted unexpected network access"); +}; +const spawn = childProcess.spawn; +childProcess.spawn = (command, args, options) => { + record({ type: "process", command, args }); + return spawn(command, args, options); +}; +syncBuiltinESMExports(); diff --git a/scripts/fixtures/runtime-fixture.mjs b/scripts/fixtures/runtime-fixture.mjs new file mode 100644 index 000000000..635c78ad6 --- /dev/null +++ b/scripts/fixtures/runtime-fixture.mjs @@ -0,0 +1,68 @@ +// Mounted into acceptance containers only; never included in an image layer. +import { appendFileSync } from "node:fs"; +import { Socket } from "node:net"; +import { ModelRuntime } from "/opt/diffex/packages/coding-agent/dist/index.js"; +import { createAssistantMessageEventStream } from "/opt/diffex/packages/ai/dist/index.js"; +import "/opt/diffex/packages/coding-agent/dist/core/http-dispatcher.js"; + +const connect = Socket.prototype.connect; +Socket.prototype.connect = function (...args) { + const options = Array.isArray(args[0]) ? args[0][0] : args[0]; + const host = typeof options === "object" ? options.host : args[1]; + if (host !== "127.0.0.1" && host !== "::1") throw new Error("Runtime smoke blocked external socket"); + return connect.apply(this, args); +}; +globalThis.fetch = async (input) => { + if (String(input) !== "http://127.0.0.1:1/usage") throw new Error("Runtime smoke blocked external fetch"); + appendFileSync(process.env.DIFFEX_SMOKE_EVENTS, "usage\n"); + return Response.json({ accepted: true }); +}; +const create = ModelRuntime.create.bind(ModelRuntime); +ModelRuntime.create = async (...args) => { + const runtime = await create(...args); + runtime.registerProvider("runtime-fixture", { + baseUrl: "https://fixture.invalid", + apiKey: "local-fixture", + api: "runtime-fixture", + models: [ + { + id: "fixture", + name: "Runtime fixture", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 8192, + maxTokens: 128, + }, + ], + streamSimple(model) { + appendFileSync(process.env.DIFFEX_SMOKE_EVENTS, "inference\n"); + const stream = createAssistantMessageEventStream(); + const message = { + role: "assistant", + content: [{ type: "text", text: "runtime-fixture-ok" }], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 1, + output: 1, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 2, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + }; + queueMicrotask(() => { + stream.push({ type: "start", partial: message }); + stream.push({ type: "text_delta", contentIndex: 0, delta: "runtime-fixture-ok", partial: message }); + stream.push({ type: "done", reason: "stop", message }); + stream.end(message); + }); + return stream; + }, + }); + return runtime; +}; diff --git a/scripts/fixtures/runtime-smoke.mjs b/scripts/fixtures/runtime-smoke.mjs new file mode 100644 index 000000000..e9414a763 --- /dev/null +++ b/scripts/fixtures/runtime-smoke.mjs @@ -0,0 +1,141 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DiffexClient } from "/opt/diffex/packages/client/dist/index.js"; +import { createWebSocketTransportFactory } from "/opt/diffex/packages/client/dist/websocket.js"; +import { WebSocket } from "/opt/diffex/node_modules/ws/wrapper.mjs"; + +const root = mkdtempSync(join(tmpdir(), "diffex-runtime-smoke-")); +for (const name of ["workspace", "agent", "state", "sessions"]) mkdirSync(join(root, name)); +const events = join(root, "events"); +writeFileSync(events, ""); +const secret = "runtime-local-smoke-fixture-32-bytes"; +const env = { + PATH: process.env.PATH, + HOME: root, + DIFFEX_OFFLINE: "1", + HOST: "127.0.0.1", + PORT: "8787", + DIFFEX_SESSION_ID: "runtime-smoke-session", + DIFFEX_WORKSPACE: join(root, "workspace"), + DIFFEX_AGENT_DIR: join(root, "agent"), + DIFFEX_CODING_AGENT_DIR: join(root, "agent"), + DIFFEX_RUNTIME_SHARED_SECRET: secret, + DIFFEX_RUNTIME_STATE_DIR: join(root, "state"), + DIFFEX_SESSION_DIR: join(root, "sessions"), + DIFFEX_OWNER_USER_ID: "runtime-smoke-user", + DIFFEX_ORG_ID: "runtime-smoke-org", + DIFFEX_MODEL_PROVIDER: "runtime-fixture", + DIFFEX_MODEL_ID: "fixture", + DIFFEX_MODEL_API_KEY: "local-unused-fixture", + DIFFEX_USAGE_ENDPOINT: "http://127.0.0.1:1/usage", + DIFFEX_SMOKE_EVENTS: events, +}; +const transport = (credential) => + createWebSocketTransportFactory({ + url: "ws://127.0.0.1:8787/runtime", + webSocket: class extends WebSocket { + constructor(url, protocols) { + super(url, protocols, { headers: credential ? { authorization: `Bearer ${credential}` } : {} }); + } + }, + }); +let child; +let completion; +const watchdog = setTimeout(() => child?.kill("SIGKILL"), 60000); +async function start() { + child = spawn( + process.execPath, + ["--import", "/acceptance/runtime-fixture.mjs", "/opt/diffex/packages/coding-agent-server/dist/cli.js"], + { cwd: env.DIFFEX_WORKSPACE, env, stdio: ["ignore", "pipe", "pipe"] }, + ); + completion = once(child, "exit"); + let output = ""; + child.stderr.on("data", (chunk) => { + output += chunk; + }); + await new Promise((done, reject) => { + child.once("error", reject); + child.once("exit", () => reject(new Error(`Runtime exited before readiness: ${output}`))); + child.stdout.on("data", (chunk) => { + output += chunk; + if (output.includes("Diffex runtime server listening")) done(); + }); + }); +} +async function stop() { + child.kill("SIGTERM"); + const [code, signal] = await completion; + assert.equal(signal, null); + assert.equal(code, 0); +} +try { + await start(); + await assert.rejects(DiffexClient.connect({ transportFactory: transport() })); + await assert.rejects(DiffexClient.connect({ transportFactory: transport("wrong-fixture") })); + const client = await DiffexClient.connect({ transportFactory: transport(secret) }); + try { + assert.deepEqual( + (await client.listSessions()).map((session) => session.id), + [env.DIFFEX_SESSION_ID], + ); + await assert.rejects(client.attachSession("another-session")); + const session = await client.attachSession(env.DIFFEX_SESSION_ID); + try { + const result = await session.prompt("Reply with the fixture confirmation."); + assert( + result.transcript.some( + (item) => + item.role === "assistant" && + item.content.some((part) => part.type === "text" && part.text.includes("runtime-fixture-ok")), + ), + ); + } finally { + await session.dispose(); + } + } finally { + await client.dispose(); + } + await stop(); + await start(); + const restored = await DiffexClient.connect({ transportFactory: transport(secret) }); + try { + const session = await restored.attachSession(env.DIFFEX_SESSION_ID); + assert(session.snapshot.transcript.some((item) => item.role === "assistant")); + await session.dispose(); + } finally { + await restored.dispose(); + } + await stop(); + assert(readFileSync(events, "utf8").includes("inference")); + assert(readFileSync(events, "utf8").includes("usage")); + console.log( + JSON.stringify({ + status: "passed", + platform: process.platform, + architecture: process.arch, + node: process.version, + startup: true, + anonymousRejected: true, + wrongCredentialRejected: true, + foreignSessionRejected: true, + promptReply: "mock-provider", + usage: "mock-endpoint", + restartPersistence: true, + shutdown: true, + externalNetwork: "disabled by container", + sourceCommit: JSON.parse(readFileSync("/opt/diffex/runtime-distribution.json", "utf8")).sourceCommit, + }), + ); +} finally { + clearTimeout(watchdog); + if (child && child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + await completion; + } + rmSync(root, { recursive: true, force: true }); + assert(!existsSync(root)); +} diff --git a/scripts/npm-lifecycle.mjs b/scripts/npm-lifecycle.mjs new file mode 100644 index 000000000..589f005dd --- /dev/null +++ b/scripts/npm-lifecycle.mjs @@ -0,0 +1,332 @@ +import assert from "node:assert/strict"; +import { execFile, spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + readdirSync, + readlinkSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { isolatedEnvironment } from "./artifact-acceptance.mjs"; +import { findNpmRecoveryDirectory } from "./npm-recovery.mjs"; + +const run = promisify(execFile); +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const [archiveArgument, evidenceArgument] = process.argv.slice(2); +assert(archiveArgument && evidenceArgument, "Usage: npm-lifecycle.mjs REVIEWED_TARBALL EVIDENCE_DIRECTORY"); +const archive = resolve(archiveArgument); +const output = resolve(evidenceArgument); +const bytes = readFileSync(archive); +const metadata = JSON.parse((await run("tar", ["-xOf", archive, "package/package.json"])).stdout); +assert.equal(metadata.name, "@diffexai/diffex"); +await run( + "python3", + [ + join(root, "scripts/check-distribution-payload.py"), + archive, + "--license", + join(root, "LICENSE"), + "--name", + metadata.name, + "--version", + metadata.version, + ], + { maxBuffer: 20 * 1024 * 1024 }, +); +const inventory = JSON.parse( + (await run("tar", ["-xOf", archive, "package/distribution-files.json"], { maxBuffer: 10 * 1024 * 1024 })).stdout, +); +function verifyInstalled(directory) { + assert.deepEqual(JSON.parse(readFileSync(join(directory, "distribution-files.json"), "utf8")), inventory); + for (const file of inventory.files) + assert.equal( + createHash("sha256") + .update(readFileSync(join(directory, file.path))) + .digest("hex"), + file.sha256, + `Installed runtime bytes changed: ${file.path}`, + ); +} +const temporary = mkdtempSync(join(tmpdir(), "diffex npm lifecycle ")); +const env = isolatedEnvironment(temporary); +delete env.DIFFEX_OFFLINE; +env.npm_config_fetch_retries = "0"; +env.npm_config_audit = "false"; +env.npm_config_fund = "false"; +const prefix = env.npm_config_prefix; +env.PATH = `${join(prefix, "bin")}:${env.PATH}`; +const packageRoot = join(prefix, "lib/node_modules/@diffexai/diffex"); +const executable = join(prefix, "bin/diffex"); +const options = { cwd: join(temporary, "project"), env, timeout: 180000, maxBuffer: 10 * 1024 * 1024 }; +const events = join(temporary, "events.jsonl"); +const response = join(temporary, "response.json"); +env.DIFFEX_LIFECYCLE_EVENTS = events; +env.DIFFEX_LIFECYCLE_RESPONSE = response; +const evidence = { + sourceCommit: "", + platform: process.platform, + arch: process.arch, + node: process.version, + version: metadata.version, + sha256: createHash("sha256").update(bytes).digest("hex"), + commands: [], + checks: [], + status: "failed", + cleanup: false, + limitation: + "Real npm installs reviewed bytes; discovery and scoped registry use controlled fixtures. The predecessor changes only isolated installed version metadata. No public-to-public upgrade or real provider inference is claimed.", +}; +let mode = "valid"; +let interrupt; +let tarballRequests = 0; +const server = createServer((request, reply) => { + const path = decodeURIComponent(request.url); + if (path === "/@diffexai/diffex") { + const pkg = { + ...metadata, + dist: { + tarball: `http://127.0.0.1:${server.address().port}/diffex.tgz`, + integrity: `sha512-${createHash("sha512") + .update(mode === "corrupt" ? "wrong bytes" : bytes) + .digest("base64")}`, + }, + }; + reply.setHeader("content-type", "application/json"); + reply.end( + JSON.stringify({ + name: metadata.name, + "dist-tags": { latest: metadata.version }, + versions: { [metadata.version]: pkg }, + }), + ); + } else if (path === "/diffex.tgz") { + tarballRequests++; + if (mode === "interrupt") { + interrupt?.(); + return; + } + reply.end(bytes); + } else { + reply.writeHead(404).end(); + } +}); +await new Promise((done) => server.listen(0, "127.0.0.1", done)); + +async function command(command, args, extra = {}) { + try { + console.log(`Lifecycle: ${command} ${args.join(" ")}`); + const result = await run(command, args, { ...options, ...extra }); + evidence.commands.push({ command, args, exitCode: 0 }); + return result.stdout; + } catch (error) { + evidence.commands.push({ command, args, exitCode: error.code, signal: error.signal }); + throw error; + } +} +function snapshot(directory) { + const result = {}; + function visit(current, local = "") { + for (const entry of readdirSync(current, { withFileTypes: true })) { + const file = join(current, entry.name); + const key = `${local}${entry.name}`; + if (entry.isDirectory()) visit(file, `${key}/`); + else + result[key] = entry.isSymbolicLink() + ? readlinkSync(file) + : createHash("sha256").update(readFileSync(file)).digest("hex"); + } + } + visit(directory); + return createHash("sha256") + .update(JSON.stringify(Object.entries(result).sort(([a], [b]) => a.localeCompare(b)))) + .digest("hex"); +} +function fixture(value) { + writeFileSync(response, JSON.stringify(value)); + writeFileSync(events, ""); +} +const valid = { status: 200, body: JSON.stringify({ packageName: metadata.name, version: metadata.version }) }; +const guard = join(temporary, "guard.mjs"); +const update = (force = false) => + command(process.execPath, ["--import", guard, executable, "update", "--self", ...(force ? ["--force"] : [])]); +try { + writeFileSync(env.npm_config_userconfig, `@diffexai:registry=http://127.0.0.1:${server.address().port}/\n`); + evidence.npm = (await command("npm", ["--version"])).trim(); + await command("npm", ["install", "-g", "--ignore-scripts", "--registry=https://registry.npmjs.org", archive]); + verifyInstalled(packageRoot); + evidence.sourceCommit = JSON.parse(readFileSync(join(packageRoot, "distribution-files.json"), "utf8")).sourceCommit; + assert.equal((await command("diffex", ["--version"])).trim(), metadata.version); + assert((await command("diffex", ["--help"])).includes("diffex")); + await command("diffex", ["--list-models"], { env: { ...env, DIFFEX_OFFLINE: "1" } }); + writeFileSync( + guard, + `import ${JSON.stringify(pathToFileURL(join(packageRoot, "dist/core/http-dispatcher.js")).href)};\n${readFileSync(join(root, "scripts/fixtures/installed-update-guard.mjs"), "utf8")}`, + ); + const retained = join(env.DIFFEX_CODING_AGENT_DIR, "lifecycle-retained.json"); + writeFileSync(retained, '{"retained":"session and settings fixture"}\n'); + const projectData = join(options.cwd, ".diffex"); + mkdirSync(projectData); + writeFileSync(join(projectData, "retained.txt"), "project data"); + const before = snapshot(prefix); + for (const invalid of [ + { status: 200, body: "{" }, + { status: 200, body: JSON.stringify({ packageName: "@foreign/cli", version: metadata.version }) }, + { status: 200, body: JSON.stringify({ packageName: metadata.name, version: "1.2.3-beta.1" }) }, + { status: 302, body: "{}", headers: { location: "https://foreign.invalid" } }, + { status: 503, body: "{}" }, + { status: 404, body: '{"error":"no_published_release"}' }, + ]) { + fixture(invalid); + await assert.rejects(update(true)); + assert.deepEqual(snapshot(prefix), before); + const recorded = readFileSync(events, "utf8"); + assert(!recorded.includes('"type":"process"')); + evidence.checks.push({ invalid, installationUnchanged: true }); + } + fixture(valid); + await update(); + assert.deepEqual(snapshot(prefix), before); + assert(!readFileSync(events, "utf8").includes('"type":"process"')); + // Simulate an older installed client without publishing or changing reviewed candidate bytes. + writeFileSync(join(packageRoot, "package.json"), JSON.stringify({ ...metadata, version: "0.0.0" })); + env.npm_config_cache = join(temporary, "cache-upgrade"); + await update(); + assert.equal((await command("diffex", ["--version"])).trim(), metadata.version); + assert(readFileSync(events, "utf8").includes(`${metadata.name}@${metadata.version}`)); + assert(tarballRequests > 0, "Real npm must download the target tarball"); + evidence.checks.push({ upgrade: "0.0.0 metadata fixture to reviewed candidate", realNpm: true }); + for (const fault of ["corrupt", "interrupt"]) { + mode = fault; + env.npm_config_cache = join(temporary, `cache-${fault}`); + fixture(valid); + const stable = snapshot(prefix); + if (fault === "corrupt") await assert.rejects(update(true), /integrity|checksum|EINTEGRITY/i); + else { + const child = spawn(process.execPath, ["--import", guard, executable, "update", "--self", "--force"], { + ...options, + detached: true, + stdio: "ignore", + }); + let interrupted = false; + interrupt = () => { + interrupted = true; + process.kill(-child.pid, "SIGTERM"); + }; + const deadline = setTimeout(() => process.kill(-child.pid, "SIGKILL"), 30000); + try { + await new Promise((done, reject) => { + child.on("close", done); + child.on("error", reject); + }); + } finally { + clearTimeout(deadline); + } + assert(interrupted, "Interrupted update must reach the real npm archive download"); + evidence.commands.push({ + command: "diffex update --self --force", + signal: "SIGTERM", + boundary: "npm archive download", + }); + } + const installationUnchanged = snapshot(prefix) === stable; + if (fault === "corrupt") assert(installationUnchanged, "Integrity failure mutated the prefix"); + mode = "valid"; + env.npm_config_cache = join(temporary, `cache-repair-${fault}`); + const repairArgs = [ + "install", + "-g", + "--ignore-scripts", + "--registry=https://registry.npmjs.org", + `${metadata.name}@${metadata.version}`, + ]; + let movedBackup = false; + try { + await command("npm", repairArgs); + } catch (error) { + if (fault !== "interrupt" || !error.stderr?.includes("ENOTEMPTY")) throw error; + const backup = findNpmRecoveryDirectory(dirname(packageRoot), error.stderr); + renameSync(backup, join(temporary, "retained-npm-recovery-backup")); + movedBackup = true; + await command("npm", repairArgs); + } + assert.equal((await command("diffex", ["--version"])).trim(), metadata.version); + verifyInstalled(packageRoot); + evidence.checks.push({ fault, installationUnchanged, movedBackup, pinnedRepair: "passed" }); + } + fixture(valid); + await update(true); + await command("npm", ["uninstall", "-g", "--ignore-scripts", metadata.name]); + assert(!existsSync(executable)); + assert(!existsSync(packageRoot)); + assert.equal(readFileSync(retained, "utf8"), '{"retained":"session and settings fixture"}\n'); + assert.equal(readFileSync(join(projectData, "retained.txt"), "utf8"), "project data"); + evidence.checks.push({ uninstall: "passed", userDataRetained: true, projectDataRetained: true }); + if (!process.argv.includes("--skip-bun")) { + env.BUN_INSTALL = join(temporary, "bun"); + env.BUN_INSTALL_BIN = join(env.BUN_INSTALL, "bin"); + env.BUN_INSTALL_GLOBAL_DIR = join(env.BUN_INSTALL, "install/global"); + env.BUN_INSTALL_CACHE_DIR = join(temporary, "bun-cache"); + env.PATH = `${env.BUN_INSTALL_BIN}:${env.PATH}`; + writeFileSync( + join(env.XDG_CONFIG_HOME, ".bunfig.toml"), + `[install.scopes]\ndiffexai = "http://127.0.0.1:${server.address().port}/"\n`, + ); + evidence.bun = (await command("bun", ["--version"])).trim(); + await command("bun", ["install", "-g", "--ignore-scripts", archive]); + assert.equal(realpathSync((await command("bun", ["pm", "bin", "-g"])).trim()), realpathSync(env.BUN_INSTALL_BIN)); + const bunPackage = join(env.BUN_INSTALL_GLOBAL_DIR, "node_modules/@diffexai/diffex"); + const bunExecutable = join(env.BUN_INSTALL_BIN, "diffex"); + verifyInstalled(bunPackage); + assert.equal((await command("diffex", ["--version"])).trim(), metadata.version); + assert((await command("diffex", ["--help"])).includes("diffex")); + writeFileSync( + guard, + `import ${JSON.stringify(pathToFileURL(join(bunPackage, "dist/core/http-dispatcher.js")).href)};\n${readFileSync(join(root, "scripts/fixtures/installed-update-guard.mjs"), "utf8")}`, + ); + fixture({ status: 200, body: JSON.stringify({ packageName: "@foreign/cli", version: metadata.version }) }); + const stable = snapshot(env.BUN_INSTALL_GLOBAL_DIR); + await assert.rejects( + command(process.execPath, ["--import", guard, bunExecutable, "update", "--self", "--force"]), + ); + assert.equal(snapshot(env.BUN_INSTALL_GLOBAL_DIR), stable); + fixture(valid); + writeFileSync(join(bunPackage, "package.json"), JSON.stringify({ ...metadata, version: "0.0.0" })); + await command(process.execPath, ["--import", guard, bunExecutable, "update", "--self"]); + assert.equal((await command("diffex", ["--version"])).trim(), metadata.version); + verifyInstalled(bunPackage); + assert(readFileSync(events, "utf8").includes('"command":"bun"')); + await command("bun", ["uninstall", "-g", metadata.name]); + assert(!existsSync(bunExecutable)); + assert(!existsSync(bunPackage)); + assert.equal(readFileSync(retained, "utf8"), '{"retained":"session and settings fixture"}\n'); + assert.equal(readFileSync(join(projectData, "retained.txt"), "utf8"), "project data"); + evidence.checks.push({ + manager: "bun", + install: "passed", + upgrade: "metadata fixture to reviewed candidate", + foreignIdentityRejected: true, + uninstall: "passed", + userDataRetained: true, + }); + } + evidence.status = "passed"; +} finally { + server.closeAllConnections(); + await new Promise((done) => server.close(done)); + rmSync(temporary, { recursive: true, force: true }); + evidence.cleanup = !existsSync(temporary); + mkdirSync(output, { recursive: true }); + writeFileSync(join(output, "npm-lifecycle.json"), `${JSON.stringify(evidence, null, 2)}\n`); +} +console.log(`npm lifecycle ${evidence.status}: ${metadata.version}`); diff --git a/scripts/npm-recovery.mjs b/scripts/npm-recovery.mjs new file mode 100644 index 000000000..35c4b27b0 --- /dev/null +++ b/scripts/npm-recovery.mjs @@ -0,0 +1,19 @@ +import assert from "node:assert/strict"; +import { readdirSync, realpathSync } from "node:fs"; +import { basename, join } from "node:path"; + +export function findNpmRecoveryDirectory(scope, errorOutput) { + assert(errorOutput.includes("ENOTEMPTY"), "Recovery requires npm ENOTEMPTY evidence"); + const candidates = readdirSync(scope, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && (entry.name === "diffex" || /^\.diffex-[a-zA-Z0-9]+$/.test(entry.name))) + .map((entry) => join(scope, entry.name)) + .filter((path) => errorOutput.includes(path) || errorOutput.includes(realpathSync(path))); + const backups = candidates.filter((path) => basename(path).startsWith(".diffex-")); + const affected = backups.length ? backups : candidates; + assert.equal( + affected.length, + 1, + `Expected one reported Diffex installation or backup inside the isolated prefix:\n${errorOutput}`, + ); + return affected[0]; +} diff --git a/scripts/npm-recovery.test.mjs b/scripts/npm-recovery.test.mjs new file mode 100644 index 000000000..240b6a2ac --- /dev/null +++ b/scripts/npm-recovery.test.mjs @@ -0,0 +1,42 @@ +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { findNpmRecoveryDirectory } from "./npm-recovery.mjs"; + +test("npm recovery identifies only the affected CLI installation or backup across rename and cleanup errors", () => { + const root = mkdtempSync(join(tmpdir(), "diffex recovery ")); + try { + const scope = join(root, "@diffexai"); + const backup = join(scope, ".diffex-Abcd1234"); + mkdirSync(backup, { recursive: true }); + const installed = join(scope, "diffex"); + mkdirSync(installed); + for (const output of [ + `npm error code ENOTEMPTY\nnpm error dest ${backup}\n`, + `npm error ENOTEMPTY: directory not empty, rename '${installed}' -> '${backup}'`, + `npm error ENOTEMPTY: directory not empty, rmdir '${backup}/dist'`, + ]) + assert.equal(findNpmRecoveryDirectory(scope, output), backup); + assert.equal( + findNpmRecoveryDirectory( + scope, + `npm error ENOTEMPTY: directory not empty, rmdir '${installed}/node_modules/@mistralai/mistralai/esm/models/components'`, + ), + installed, + ); + assert.throws(() => findNpmRecoveryDirectory(scope, "ENOTEMPTY at an unrelated path")); + assert.throws(() => findNpmRecoveryDirectory(scope, `ENOENT ${backup}`)); + const outside = join(root, ".diffex-Outside"); + mkdirSync(outside); + symlinkSync(outside, join(scope, ".diffex-Link")); + assert.throws(() => findNpmRecoveryDirectory(scope, `ENOTEMPTY ${outside}`)); + assert.throws(() => findNpmRecoveryDirectory(scope, `ENOTEMPTY ${join(scope, ".diffex-Link")}`)); + const other = join(scope, ".diffex-Other"); + mkdirSync(other); + assert.throws(() => findNpmRecoveryDirectory(scope, `ENOTEMPTY ${backup} ${other}`)); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); diff --git a/scripts/prepare-distribution.mjs b/scripts/prepare-distribution.mjs index a571da182..468c92e45 100644 --- a/scripts/prepare-distribution.mjs +++ b/scripts/prepare-distribution.mjs @@ -96,7 +96,7 @@ export function preparePackageDistribution(directory) { }); } cpSync(join(root, "LICENSE"), join(absolute, "LICENSE")); - writeFileSync(join(absolute, "NOTICE"), notice); + rmSync(join(absolute, "NOTICE"), { force: true }); writeFileSync(join(absolute, "THIRD-PARTY-NOTICES.txt"), notices); for (const component of components) { if (component.notice) diff --git a/scripts/prepare-distribution.test.mjs b/scripts/prepare-distribution.test.mjs index bd2b394cc..12818c9e4 100644 --- a/scripts/prepare-distribution.test.mjs +++ b/scripts/prepare-distribution.test.mjs @@ -1,11 +1,21 @@ import assert from "node:assert/strict"; import { execFileSync, spawnSync } from "node:child_process"; -import { cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; import { PROJECT_CHANGELOGS } from "./prepare-distribution.mjs"; -import { cliDistribution } from "./cli-distribution.mjs"; +import { cliDistribution, distributionDocs } from "./cli-distribution.mjs"; test("packed CLI exposes the product homepage and preserves referenced documentation", () => { const root = mkdtempSync(join(tmpdir(), "diffex-cli-presentation-")); @@ -20,9 +30,11 @@ test("packed CLI exposes the product homepage and preserves referenced documenta "release-distribution-manifest.mjs", "package-workspaces.mjs", "check-distribution-payload.py", + "distribution-docs.json", "cli-distribution.mjs", "stage-cli-distribution.mjs", - ]) copy(`scripts/${script}`); + ]) + copy(`scripts/${script}`); for (const file of [ "LICENSE", "docs/release/distribution-policy.json", @@ -31,7 +43,8 @@ test("packed CLI exposes the product homepage and preserves referenced documenta "packages/coding-agent/npm-shrinkwrap.json", "packages/coding-agent/install-lock/package.json", "packages/coding-agent/install-lock/package-lock.json", - ]) copy(file); + ]) + copy(file); const directories = [ "packages/coding-agent", "packages/evolution", @@ -40,8 +53,7 @@ test("packed CLI exposes the product homepage and preserves referenced documenta ]; for (const directory of directories) { copy(`${directory}/package.json`); - for (const folder of ["src", "dist", "docs"]) - mkdirSync(join(root, directory, folder), { recursive: true }); + for (const folder of ["src", "dist", "docs"]) mkdirSync(join(root, directory, folder), { recursive: true }); writeFileSync(join(root, directory, "src/index.ts"), "export const value = 1;\n"); writeFileSync(join(root, directory, "dist/index.js"), "export const value = 1;\n"); for (const file of ["README.md", "CONTRIBUTING.md", "docs/skills.md", "CHANGELOG.md"]) @@ -56,7 +68,11 @@ test("packed CLI exposes the product homepage and preserves referenced documenta "invocation-traces.md", "images/doom-extension.png", "images/exy.png", + "new-internal-audit.md", + "windows.md", + "termux.md", ]; + writeFileSync(join(pkg, "docs/new-internal-audit.md"), "Private audit must never ship"); const metadata = JSON.parse(readFileSync(join(pkg, "package.json"), "utf8")); metadata.files.push("CONTRIBUTING.md"); writeFileSync(join(pkg, "package.json"), JSON.stringify(metadata)); @@ -80,30 +96,45 @@ test("packed CLI exposes the product homepage and preserves referenced documenta }), ); const archive = join(root, "stage", packed.filename); - const published = JSON.parse(execFileSync("tar", ["-xOf", archive, "package/package.json"], { encoding: "utf8" })); + const published = JSON.parse( + execFileSync("tar", ["-xOf", archive, "package/package.json"], { encoding: "utf8" }), + ); assert.equal(published.homepage, "https://diffex.ai/"); assert.equal(published.repository, undefined); assert.equal(published.bugs, undefined); assert.equal(published.diffexConfig.production, true); - assert.equal(execFileSync("tar", ["-xOf", archive, "package/README.md"]).length, 0); + assert.match( + execFileSync("tar", ["-xOf", archive, "package/README.md"], { encoding: "utf8" }), + /docs\/installation\.md/, + ); for (const file of readdirSync(join(pkg, "docs"), { recursive: true })) { const source = join(pkg, "docs", file); if (!statSync(source).isFile()) continue; const path = `docs/${file.replaceAll("\\", "/")}`; - if (excludedDocs.includes(file.replaceAll("\\", "/"))) { + if (!distributionDocs.includes(path)) { assert(!packed.files.some((entry) => entry.path === path), `${path} must not ship`); } else { - assert.deepEqual(execFileSync("tar", ["-xOf", archive, `package/${path}`]), readFileSync(source), `${path} must remain intact`); + assert.deepEqual( + execFileSync("tar", ["-xOf", archive, `package/${path}`]), + readFileSync(source), + `${path} must remain intact`, + ); } } for (const file of excludedDocs) assert(existsSync(join(pkg, "docs", file)), `${file} must remain in source`); for (const { path } of packed.files) assert(!path.split("/").some((part) => ["notice", "contributing.md"].includes(part.toLowerCase())), path); - const attribution = execFileSync("tar", ["-xOf", archive, "package/THIRD-PARTY-NOTICES.txt"], { encoding: "utf8" }); - assert(attribution.startsWith(readFileSync(join(pkg, "NOTICE"), "utf8"))); + const attribution = execFileSync("tar", ["-xOf", archive, "package/THIRD-PARTY-NOTICES.txt"], { + encoding: "utf8", + }); + assert.match(attribution, /^Diffex\nCopyright 2026 Luntraa, Inc\./); + assert(!existsSync(join(pkg, "NOTICE"))); assert.match(attribution, /highlight\.js@11\.9\.0/); assert.match(attribution, /marked@18\.0\.5/); - const result = spawnSync("python3", ["scripts/check-distribution-payload.py", archive], { cwd: root, encoding: "utf8" }); + const result = spawnSync("python3", ["scripts/check-distribution-payload.py", archive], { + cwd: root, + encoding: "utf8", + }); assert.equal(result.status, 0, result.stderr); } finally { rmSync(root, { recursive: true, force: true }); @@ -120,6 +151,7 @@ test("actual npm tarball carries notices and excludes debug sources after prepar "release-distribution-manifest.mjs", "package-workspaces.mjs", "check-distribution-payload.py", + "distribution-docs.json", ]) cpSync(`scripts/${script}`, join(root, "scripts", script)); cpSync("LICENSE", join(root, "LICENSE")); @@ -148,6 +180,7 @@ test("actual npm tarball carries notices and excludes debug sources after prepar ], }), ); + writeFileSync(join(pkg, "NOTICE"), "Stale generated attribution"); writeFileSync(join(pkg, "dist/index.js"), "export const value = 1;\n//# sourceMappingURL=index.js.map\n"); writeFileSync(join(pkg, "dist/index.js.map"), '{"sourcesContent":["private TypeScript"]}'); writeFileSync( @@ -159,6 +192,7 @@ test("actual npm tarball carries notices and excludes debug sources after prepar cwd: tmpdir(), env: { ...process.env, DIFFEX_SOURCE_COMMIT: "a".repeat(40) }, }); + assert.equal(existsSync(join(pkg, "NOTICE")), false); assert.equal(existsSync(join(pkg, "dist/index.js.map")), false); assert.match( readFileSync( diff --git a/scripts/runtime-image.test.mjs b/scripts/runtime-image.test.mjs new file mode 100644 index 000000000..0d20fef93 --- /dev/null +++ b/scripts/runtime-image.test.mjs @@ -0,0 +1,72 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import test from "node:test"; +import { distributionDocs } from "./cli-distribution.mjs"; + +test("image inspection rejects separate NOTICE files and development documents in earlier layers", () => { + const directory = mkdtempSync(join(tmpdir(), "diffex runtime layers ")); + try { + const write = (file, content) => { + const path = join(directory, file); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, content); + }; + const layer = "layer/opt/diffex/"; + const workspace = `${layer}packages/coding-agent/`; + write(`${layer}runtime-distribution.json`, JSON.stringify({ sourceCommit: "a".repeat(40) })); + write(`${workspace}LICENSE`, readFileSync("LICENSE")); + write(`${workspace}THIRD-PARTY-NOTICES.txt`, "Reviewed notice"); + for (const doc of distributionDocs) write(workspace + doc, "User-facing guide"); + write( + "config.json", + JSON.stringify({ + architecture: "amd64", + os: "linux", + config: { Labels: { "org.opencontainers.image.revision": "a".repeat(40) } }, + }), + ); + write("manifest.json", JSON.stringify([{ Config: "config.json", Layers: ["layer.tar", "cleanup.tar"] }])); + write("cleanup/opt/diffex/packages/coding-agent/docs/.wh.development.md", ""); + execFileSync("tar", ["-cf", join(directory, "cleanup.tar"), "-C", join(directory, "cleanup"), "opt"], { + env: { ...process.env, COPYFILE_DISABLE: "1" }, + }); + const inspect = () => { + execFileSync("tar", ["-cf", join(directory, "layer.tar"), "-C", join(directory, "layer"), "opt"], { + env: { ...process.env, COPYFILE_DISABLE: "1" }, + }); + execFileSync( + "tar", + [ + "-cf", + join(directory, "image.tar"), + "-C", + directory, + "manifest.json", + "config.json", + "layer.tar", + "cleanup.tar", + ], + { env: { ...process.env, COPYFILE_DISABLE: "1" } }, + ); + return spawnSync("python3", [resolve("scripts/check-runtime-image.py"), join(directory, "image.tar")], { + encoding: "utf8", + }); + }; + const valid = inspect(); + assert.equal(valid.status, 0, valid.stderr); + write(`${workspace}NOTICE`, "Duplicate project attribution"); + const notice = inspect(); + assert.notEqual(notice.status, 0); + assert.match(notice.stderr, /NOTICE/); + rmSync(join(directory, workspace, "NOTICE")); + write(`${workspace}docs/development.md`, "Private development handoff"); + const invalid = inspect(); + assert.notEqual(invalid.status, 0); + assert.match(invalid.stderr, /unapproved distribution documentation/); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/scripts/stage-cli-distribution.mjs b/scripts/stage-cli-distribution.mjs index 630e0289d..e743fd032 100644 --- a/scripts/stage-cli-distribution.mjs +++ b/scripts/stage-cli-distribution.mjs @@ -1,32 +1,22 @@ import { cpSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { createHash } from "node:crypto"; -import { join, relative } from "node:path"; +import { dirname, join } from "node:path"; import { cliDistribution } from "./cli-distribution.mjs"; import { preparePackageDistribution } from "./prepare-distribution.mjs"; export function stageCliDistribution(directory, stage) { const { manifest, bundled } = cliDistribution(); mkdirSync(stage, { recursive: true }); - const notices = [ - "LICENSE", - "THIRD-PARTY-NOTICES.txt", - "distribution-components.json", - "distribution-files.json", - ]; - const excludedDocs = new Set([ - "docs/development.md", - "docs/docs.json", - "docs/index.md", - "docs/invocation-traces.md", - "docs/images/doom-extension.png", - "docs/images/exy.png", - ]); - for (const file of manifest.files) - cpSync(join(directory, file), join(stage, file), { - recursive: true, - filter: (source) => !excludedDocs.has(relative(directory, source).replaceAll("\\", "/")), - }); - writeFileSync(join(stage, "README.md"), ""); + const notices = ["LICENSE", "THIRD-PARTY-NOTICES.txt", "distribution-components.json", "distribution-files.json"]; + for (const file of manifest.files) { + if (file === "README.md") continue; + mkdirSync(dirname(join(stage, file)), { recursive: true }); + cpSync(join(directory, file), join(stage, file), { recursive: true }); + } + writeFileSync( + join(stage, "README.md"), + "# Diffex\n\nCoding agent for your terminal.\nSee [installation, updates, and removal](docs/installation.md), [quickstart](docs/quickstart.md), and [usage](docs/usage.md).\n\nDiffex is licensed under Apache-2.0; Pi-derived portions retain their MIT terms.\nSee LICENSE and THIRD-PARTY-NOTICES.txt.\n", + ); const inventory = JSON.parse(readFileSync(join(directory, "distribution-files.json"), "utf8")); const components = JSON.parse(readFileSync(join(directory, "distribution-components.json"), "utf8")).filter( (component) => !bundled.some((pkg) => pkg.manifest.name === component.name), diff --git a/scripts/stage-runtime-distribution.mjs b/scripts/stage-runtime-distribution.mjs new file mode 100644 index 000000000..28120545b --- /dev/null +++ b/scripts/stage-runtime-distribution.mjs @@ -0,0 +1,73 @@ +import { cpSync, existsSync, lstatSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { distributionDocs } from "./cli-distribution.mjs"; +import { findPackageDirectories } from "./package-workspaces.mjs"; +import { preparePackageDistribution } from "./prepare-distribution.mjs"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const output = resolve(process.argv[2] ?? ""); +if (!process.argv[2] || existsSync(output)) throw new Error("Specify a new runtime output directory"); +const local = relative(root, output); +if (!local.startsWith("..") && !isAbsolute(local)) throw new Error("Runtime output must be outside the source tree"); +process.chdir(root); +const workspaces = new Map( + findPackageDirectories().map((directory) => { + const manifest = JSON.parse(readFileSync(join(directory, "package.json"), "utf8")); + return [manifest.name, { directory, manifest }]; + }), +); +const selected = new Map(); +function visit(name) { + if (selected.has(name)) return; + const pkg = workspaces.get(name); + if (!pkg) throw new Error(`Missing runtime workspace: ${name}`); + selected.set(name, pkg); + for (const dependency of Object.keys({ ...pkg.manifest.dependencies, ...pkg.manifest.optionalDependencies })) + if (dependency.startsWith("@diffexai/")) visit(dependency); +} +visit("@diffexai/diffex-coding-agent-server"); +visit("@diffexai/diffex-evolution"); +visit("@diffexai/diffex-harness-trace"); +mkdirSync(output); +const notices = ["LICENSE", "THIRD-PARTY-NOTICES.txt", "distribution-components.json", "distribution-files.json"]; +for (const pkg of selected.values()) { + const prepared = preparePackageDistribution(pkg.directory); + const target = join(output, pkg.directory); + mkdirSync(target, { recursive: true }); + const manifest = structuredClone(pkg.manifest); + delete manifest.scripts; + delete manifest.devDependencies; + writeFileSync(join(target, "package.json"), `${JSON.stringify(manifest, null, 2)}\n`); + const files = [...prepared.files.map((file) => file.path), ...notices]; + if (pkg.manifest.name === "@diffexai/diffex") files.push(...distributionDocs); + for (const file of files) { + mkdirSync(dirname(join(target, file)), { recursive: true }); + cpSync(join(pkg.directory, file), join(target, file)); + } + if (existsSync(join(pkg.directory, "node_modules"))) + cpSync(join(pkg.directory, "node_modules"), join(target, "node_modules"), { recursive: true }); +} +cpSync(join(root, "node_modules"), join(output, "node_modules"), { + recursive: true, + verbatimSymlinks: true, + filter(source) { + if (!lstatSync(source).isSymbolicLink()) return true; + const resolved = realpathSync(source); + const workspace = [...workspaces.values()].find( + (pkg) => resolved === resolve(pkg.directory) || resolved.startsWith(`${resolve(pkg.directory)}/`), + ); + return !workspace || selected.has(workspace.manifest.name); + }, +}); +cpSync(join(root, "LICENSE"), join(output, "LICENSE")); +cpSync(join(output, "packages/coding-agent/THIRD-PARTY-NOTICES.txt"), join(output, "THIRD-PARTY-NOTICES.txt")); +writeFileSync( + join(output, "README.md"), + "# Diffex hosted runtime\n\nStart diffex-runtime-server with the session, workspace, internal authentication, model, and usage configuration supplied by your control plane.\nThis image is not an interactive CLI installation channel.\nSee packages/coding-agent/docs for user-facing runtime behavior and LICENSE and THIRD-PARTY-NOTICES.txt for legal terms.\n", +); +writeFileSync( + join(output, "runtime-distribution.json"), + `${JSON.stringify({ sourceCommit: process.env.DIFFEX_SOURCE_COMMIT, packages: [...selected.keys()], documentation: distributionDocs }, null, 2)}\n`, +); +console.log(`Prepared ${selected.size} private hosted runtime workspaces`);