From 803f466eb76665177d6266501808e3f42a4acf9f Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sun, 13 Sep 2026 16:31:09 -0400 Subject: [PATCH 1/4] feat(coding-agent): validate install lifecycle - Protect installed updates and distribution contents with real package-manager acceptance and explicit public documentation selection. - Validate the compiled hosted runtime locally while registry publication and deployment promotion await configured environments. --- .dockerignore | 6 + .github/workflows/runtime-image.yml | 85 +++-- AGENTS.md | 4 + Dockerfile.runtime | 28 +- docs/cloud-runtime.md | 9 +- docs/release/artifact-acceptance.md | 15 + docs/release/distribution-policy.json | 5 +- docs/release/runtime-validation.md | 74 +++++ packages/coding-agent/docs/installation.md | 114 +++++++ packages/coding-agent/docs/quickstart.md | 31 +- packages/coding-agent/package.json | 33 +- scripts/artifact-acceptance.mjs | 11 + scripts/check-distribution-payload.py | 15 + scripts/check-runtime-image.py | 78 +++++ scripts/cli-distribution.mjs | 20 +- scripts/copy-distribution-docs.mjs | 10 + scripts/distribution-docs.json | 32 ++ scripts/distribution-payload.test.mjs | 27 +- scripts/fixtures/installed-update-guard.mjs | 28 ++ scripts/fixtures/runtime-fixture.mjs | 68 ++++ scripts/fixtures/runtime-smoke.mjs | 141 +++++++++ scripts/npm-lifecycle.mjs | 333 ++++++++++++++++++++ scripts/prepare-distribution.test.mjs | 55 +++- scripts/runtime-image.test.mjs | 67 ++++ scripts/stage-cli-distribution.mjs | 32 +- scripts/stage-runtime-distribution.mjs | 79 +++++ 26 files changed, 1277 insertions(+), 123 deletions(-) create mode 100644 docs/release/runtime-validation.md create mode 100644 packages/coding-agent/docs/installation.md create mode 100644 scripts/check-runtime-image.py create mode 100644 scripts/copy-distribution-docs.mjs create mode 100644 scripts/distribution-docs.json create mode 100644 scripts/fixtures/installed-update-guard.mjs create mode 100644 scripts/fixtures/runtime-fixture.mjs create mode 100644 scripts/fixtures/runtime-smoke.mjs create mode 100644 scripts/npm-lifecycle.mjs create mode 100644 scripts/runtime-image.test.mjs create mode 100644 scripts/stage-runtime-distribution.mjs diff --git a/.dockerignore b/.dockerignore index 017a93543..5b1b5dfcb 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,3 +11,9 @@ packages/ai/src/providers/data packages/coding-agent/binaries *.log *.tsbuildinfo + +.env +.env.* +**/.env +**/.env.* +**/__pycache__ diff --git a/.github/workflows/runtime-image.yml b/.github/workflows/runtime-image.yml index 02823b44e..e9cfc14ef 100644 --- a/.github/workflows/runtime-image.yml +++ b/.github/workflows/runtime-image.yml @@ -1,26 +1,6 @@ name: Runtime Image on: - push: - branches: [main] - paths: - - '.github/workflows/runtime-image.yml' - - '.dockerignore' - - 'Dockerfile.runtime' - - 'package.json' - - 'package-lock.json' - - 'packages/**' - - 'scripts/**' - pull_request: - branches: [main] - paths: - - '.github/workflows/runtime-image.yml' - - '.dockerignore' - - 'Dockerfile.runtime' - - 'package.json' - - 'package-lock.json' - - 'packages/**' - - 'scripts/**' workflow_dispatch: permissions: @@ -28,40 +8,57 @@ permissions: concurrency: group: runtime-image-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: false jobs: - build: - # Publication is paused until the runtime image and deployment consumers are validated. - if: ${{ false }} - runs-on: ubuntu-latest - permissions: - contents: read - packages: write + validate: + runs-on: ubuntu-24.04 + timeout-minutes: 25 steps: - - name: Checkout + - name: Checkout candidate uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - + with: + persist-credentials: false + - name: Resolve candidate identity + id: source + run: echo "commit=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT" - name: Set up Docker Buildx uses: docker/setup-buildx-action@37fe631027851001ddb9b187196cc803df7f5f0e # v4.3.0 - - - name: Log in to GitHub Container Registry - if: github.event_name != 'pull_request' - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Build and publish immutable runtime image + - name: Prepare private evidence directory + run: mkdir -p .artifacts/runtime + - name: Build Linux amd64 candidate locally uses: docker/build-push-action@53b7df96c91f9c12dcc8a07bcb9ccacbed38856a # v7.3.0 with: context: . file: Dockerfile.runtime platforms: linux/amd64 - push: ${{ github.event_name != 'pull_request' }} - tags: ghcr.io/diffexai/diffex-runtime:sha-${{ github.sha }} + load: true + push: false + tags: diffex-runtime:candidate + build-args: SOURCE_COMMIT=${{ steps.source.outputs.commit }} cache-from: type=gha,scope=diffex-runtime cache-to: type=gha,mode=max,scope=diffex-runtime - provenance: mode=max - sbom: true + # Docker's local exporter cannot retain attestations. + # GHCR publication requires a separate reviewed OCI attestation gate. + provenance: false + sbom: false + - name: Validate hosted process without external networking + run: | + docker run --rm --network none \ + --mount "type=bind,src=$GITHUB_WORKSPACE/scripts/fixtures,dst=/acceptance,readonly" \ + --entrypoint node diffex-runtime:candidate /acceptance/runtime-smoke.mjs \ + > .artifacts/runtime/smoke.json + - name: Inspect every candidate image layer + run: | + docker save --output .artifacts/runtime/runtime.tar diffex-runtime:candidate + python3 scripts/check-runtime-image.py .artifacts/runtime/runtime.tar > .artifacts/runtime/payload.json + docker inspect diffex-runtime:candidate > .artifacts/runtime/image.json + sha256sum .artifacts/runtime/runtime.tar > .artifacts/runtime/SHA256SUMS + - name: Retain tested image and private evidence + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: runtime-candidate-${{ steps.source.outputs.commit }} + path: .artifacts/runtime/ + if-no-files-found: error + retention-days: 14 diff --git a/AGENTS.md b/AGENTS.md index 19dd77621..a2b2acabd 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -171,3 +171,7 @@ If the user's instructions conflict with any rule in this document, ask for expl - 2026-09-13: Node rejects package import targets containing node_modules -> Preserve existing package import targets and validate staged tarballs through installed CLI startup before publication. - 2026-09-13: npm accepted publication before registry metadata propagated -> Preserve the uploaded artifact identity and resume verification without republishing or rebuilding. + +- 2026-09-13: Guessed release helper paths and reused npm caches obscured validation -> Discover paths before reading and use a fresh cache when proving that an update downloads artifact bytes. + +- 2026-09-13: Assumed interrupted npm downloads preserve the prefix -> Test real npm processes, retain user data, and recover ENOTEMPTY by moving only the exact stale package backup before a pinned reinstall. diff --git a/Dockerfile.runtime b/Dockerfile.runtime index e06993a6e..95819e284 100644 --- a/Dockerfile.runtime +++ b/Dockerfile.runtime @@ -1,28 +1,42 @@ -FROM node:24-bookworm-slim AS build +FROM node:24.16.0-bookworm-slim@sha256:2c87ef9bd3c6a3bd4b472b4bec2ce9d16354b0c574f736c476489d09f560a203 AS build WORKDIR /opt/diffex -COPY package.json package-lock.json ./ -COPY tsconfig.base.json ./ +ARG SOURCE_COMMIT +ENV DIFFEX_SOURCE_COMMIT=$SOURCE_COMMIT + +COPY package.json package-lock.json tsconfig.base.json ./ +COPY LICENSE ./ +COPY legal ./legal +COPY docs/release/distribution-policy.json ./docs/release/distribution-policy.json COPY packages ./packages COPY scripts ./scripts RUN npm ci --ignore-scripts -RUN npm run build +RUN npm run hydrate:model-data && npm run build:offline RUN npm prune --omit=dev --ignore-scripts +RUN node scripts/stage-runtime-distribution.mjs /opt/runtime -FROM node:24-bookworm-slim AS runtime +FROM node:24.16.0-bookworm-slim@sha256:2c87ef9bd3c6a3bd4b472b4bec2ce9d16354b0c574f736c476489d09f560a203 AS runtime RUN apt-get update \ + && apt-get install -y --no-install-recommends ca-certificates \ + && sed -i "s|http://deb.debian.org|https://deb.debian.org|g" /etc/apt/sources.list.d/debian.sources \ + && apt-get update \ && apt-get install -y --no-install-recommends ca-certificates fd-find git openssh-client ripgrep \ && ln -s /usr/bin/fdfind /usr/local/bin/fd \ && rm -rf /var/lib/apt/lists/* -COPY --from=build /opt/diffex /opt/diffex +COPY --from=build /opt/runtime /opt/diffex RUN ln -s /opt/diffex/packages/coding-agent-server/dist/cli.js /usr/local/bin/diffex-runtime-server WORKDIR /workspace +ARG SOURCE_COMMIT LABEL org.opencontainers.image.source="https://github.com/diffexai/diffex" -LABEL org.opencontainers.image.description="Pi-based hosted Diffex coding-agent runtime" +LABEL org.opencontainers.image.revision=$SOURCE_COMMIT +LABEL org.opencontainers.image.licenses="Apache-2.0 AND MIT" +LABEL org.opencontainers.image.description="Hosted Diffex coding-agent runtime" + +CMD ["diffex-runtime-server"] diff --git a/docs/cloud-runtime.md b/docs/cloud-runtime.md index 5c03552e3..2626023bc 100644 --- a/docs/cloud-runtime.md +++ b/docs/cloud-runtime.md @@ -24,10 +24,11 @@ Only Daytona's opaque placeholder enters the sandbox environment, and the real p ## Runtime Image -`Dockerfile.runtime` produces the AMD64 image used to create the Daytona sandbox snapshot. -The runtime-image workflow is disabled during the identity migration, including manual dispatches. -Re-enable the workflow for validation and immutable `ghcr.io/diffexai/diffex-runtime:sha-` publication after reviewing the image and its consumers. -Keep production on its existing tested image and Daytona snapshot until the replacement image and snapshot pass release validation. +`Dockerfile.runtime` produces a Linux amd64 candidate with a digest-pinned Node base image, compiled workspace outputs, user documentation, and legal notices. +The Runtime Image workflow now permits manual validation and retains the tested image privately. +Registry publication and automatic main-branch publication remain disabled. +The owner confirmed on 2026-09-13 that no staging deployment or Daytona consumer is configured, so no existing production digest or snapshot can be assumed. +See [runtime validation and promotion prerequisites](release/runtime-validation.md) for local commands and the pending hosted acceptance. Before a cloud release, register GHCR with Daytona, create a Daytona snapshot from that exact immutable image tag or digest, and set `DAYTONA_SANDBOX_SNAPSHOT` to the resulting snapshot name. Do not use a floating image tag for production. diff --git a/docs/release/artifact-acceptance.md b/docs/release/artifact-acceptance.md index d4b68059b..b0b86c6e0 100644 --- a/docs/release/artifact-acceptance.md +++ b/docs/release/artifact-acceptance.md @@ -28,6 +28,21 @@ Acceptance executes file discovery and content search, records their versions, a Native runners install those prerequisites through apt or Homebrew before package acceptance. This validates the npm installation with controlled system helpers; it does not establish the separate automatic-helper-download lifecycle. +## Installed lifecycle acceptance + +The acceptance command also runs `scripts/npm-lifecycle.mjs` against the exact reviewed CLI tarball. +It installs globally into an isolated prefix, checks PATH, version, help, and models, then invokes the installed self-updater with real npm subprocesses. +Only discovery and the Diffex-scoped registry are controlled fixtures; dependency downloads use npmjs.org. +An isolated version-metadata predecessor exercises upgrade to the reviewed candidate without publishing a fake release. +Fresh caches require actual target downloads. +Malformed and foreign metadata, redirects, outages, no release, and checksum mismatches are rejected without changing the installed prefix, including force. +An interrupted npm download demonstrates the documented pinned repair and, when npm reports `ENOTEMPTY`, relocation of its exact stale backup before retrying. +Interruption itself can mutate npm's prefix and is not claimed to preserve the executable. +Uninstall verifies executable removal and retained global and project data. +The same harness verifies Bun global installation and PATH, rejects a foreign release identity without mutation, upgrades with the real Bun subprocess, checks the installed runtime hashes, and uninstalls while retaining user and project data. +An isolated Bun global scope registry ensures that the upgrade receives the reviewed candidate bytes. +`npm-lifecycle.json` records commands, identities, results, limitations, and cleanup. + ## Supported distribution scope The approved channel remains npm, installed with npm or Bun and executed on Node 22.19.0 or newer. diff --git a/docs/release/distribution-policy.json b/docs/release/distribution-policy.json index cf7ffd738..e36671f26 100644 --- a/docs/release/distribution-policy.json +++ b/docs/release/distribution-policy.json @@ -35,7 +35,7 @@ "HTML export templates and their reviewed marked/highlight.js bundles", "runtime theme and model JSON", "SQLite migration SQL", - "documentation including inline examples", + "user-facing documentation explicitly listed in scripts/distribution-docs.json", "Diffex-owned TUI native prebuilds using system APIs" ], "publicHosting": "npmjs", @@ -55,5 +55,6 @@ "protocol", "telemetry", "tui" - ] + ], + "developmentDocumentation": "exclude from all distribution artifacts, including bootstrap and hosted runtime; keep private development records in the source repository" } diff --git a/docs/release/runtime-validation.md b/docs/release/runtime-validation.md new file mode 100644 index 000000000..997f6784d --- /dev/null +++ b/docs/release/runtime-validation.md @@ -0,0 +1,74 @@ +# Hosted runtime candidate validation + +The owner confirmed on 2026-09-13 that no staging deployment or Daytona consumer is configured. +Local runtime validation is the available acceptance boundary. +It does not promote a hosted deployment or establish a prior-runtime rollback baseline. +The release remains npm-only; this image is not an interactive CLI installation channel. + +## Build and inspect locally + +Run from the private source checkout with Docker running: + +```sh +DIFFEX_RUNTIME_COMMIT="$(git rev-parse HEAD)" +docker build --platform linux/amd64 \ + --build-arg "SOURCE_COMMIT=$DIFFEX_RUNTIME_COMMIT" \ + --file Dockerfile.runtime --tag diffex-runtime:candidate . +docker run --rm --platform linux/amd64 --network none \ + --mount "type=bind,src=$PWD/scripts/fixtures,dst=/acceptance,readonly" \ + --entrypoint node diffex-runtime:candidate /acceptance/runtime-smoke.mjs +docker save --output /tmp/diffex-runtime-candidate.tar diffex-runtime:candidate +python3 scripts/check-runtime-image.py /tmp/diffex-runtime-candidate.tar +docker inspect diffex-runtime:candidate +shasum -a 256 /tmp/diffex-runtime-candidate.tar +``` + +Use a clean committed source for retained acceptance and record its full commit. +Dirty-checkout results must also identify the uncommitted diff and cannot be attributed solely to the base commit label. +The Dockerfile pins the Node 24.16.0 base manifest digest and uses Linux amd64. +It hydrates validated model data, compiles workspace outputs, prunes development dependencies, and stages the required hosted runtime dependency closure. +The image carries compiled files, native runtime assets, the explicit user-documentation allowlist, and legal notices. +Project build inputs and private development documentation stay in the build stage. +Third-party redistribution materials remain with dependencies. + +The fixture is mounted read-only for testing and never copied into a distribution layer. +It launches the actual server executable, uses a mock model provider and usage endpoint, verifies anonymous and incorrect-credential denial, rejects another session ID, exchanges a prompt and reply, and restarts using persisted session data. +External networking is disabled by Docker. +The test removes its directories and the container exits cleanly. +The layer checker examines every saved image layer so deleting an internal document in a later layer cannot hide its inclusion. + +The manually dispatched Runtime Image workflow runs the same build, smoke, and layer checks on Linux amd64. +It retains the tested Docker archive, local image ID, archive checksum, and evidence in private Actions storage for 14 days. +It has read-only repository permissions and no registry publishing job. +There is no automatic main-branch image publication. + +## Local evidence + +On 2026-09-13, the development candidate passed the Linux amd64 build and smoke on Docker running on Darwin arm64. +The initial local image ID was `sha256:cfd4495ec712460a1400a1f6dda7f4352b7e48a659bed79505cf0c2f3714aad9`. +Its source label identifies base commit `06830a99220c3dad14292830171c433c916d49b1`; the branch's uncommitted runtime and packaging changes were also build inputs. +Layer inspection accepted 1,426 project files across 12 runtime workspaces and all 30 approved documentation assets. +This development image is not a registry release digest or a promoted artifact. +Final committed-candidate workflow evidence belongs in the implementation PR. + +The npm lifecycle harness also passed on Darwin arm64 with Node 22.19.0, npm 10.9.3, and Bun 1.3.11. +It used the local candidate tarball, real npm installation processes, controlled release discovery, and a loopback registry for the canonical package. +Malformed release inputs and checksum mismatch preserved the installed prefix. +An interrupted download changed npm's prefix and required moving the exact stale backup before a pinned repair succeeded. +Bun global installation, PATH, real self-update, installed-byte verification, foreign-identity rejection, and removal also passed. +Removal retained user and project data, and all test-owned directories were removed. +Its version-metadata predecessor is a simulation; public-to-public upgrade acceptance remains separate. + +## Registry and deployment prerequisites + +Keep the future image private at `ghcr.io/diffexai/diffex-runtime` and configure scoped publisher and pull identities before publication. +Review the base-image and dependency license inventory, generate OCI SBOM and provenance attestations, and retain the image's registry manifest digest with its exact source and model-data inputs. +Docker's local exporter does not retain attestations, so the local archive does not claim SBOM/provenance acceptance. +Use a registry or OCI exporter for attested artifacts as described in [Docker's attestation documentation](https://docs.docker.com/build/metadata/attestations/). + +The first hosted acceptance must pull the exact candidate from the intended staging environment, create a Daytona snapshot from its digest, and configure the cloud consumer explicitly. +Record the cloud commit, runtime commit, registry digest, snapshot, deployment owner, architecture, pull identity, and approved credential source without credential values. +Exercise real cloud allocation, attachment, streaming, reconnect, stop, cross-actor denial, and cleanup. +Only then retain the tested digest and snapshot as the first recovery baseline. +Every replacement must keep its previous tested configuration available, demonstrate restoration, and verify cleanup before retirement. +Registry publication, attestations, real provider inference, deployment promotion, and prior-runtime rollback are not completed by local testing. diff --git a/packages/coding-agent/docs/installation.md b/packages/coding-agent/docs/installation.md new file mode 100644 index 000000000..2914fa258 --- /dev/null +++ b/packages/coding-agent/docs/installation.md @@ -0,0 +1,114 @@ +# Install, update, and remove Diffex + +Diffex is distributed through npmjs.org as `@diffexai/diffex`. +Use Node.js 22.19.0 or newer and npm on macOS or Linux, on arm64 or x64. +Windows, Android/Termux, standalone binaries, and additional installation channels are not accepted release targets. +The hosted runtime image is for the cloud service. + +## Install and verify + +```sh +node --version +npm --version +npm install -g --ignore-scripts --registry=https://registry.npmjs.org @diffexai/diffex +diffex --version +diffex --help +diffex --list-models +``` + +Only the CLI package is needed; its internal Diffex modules are included. +`--ignore-scripts` disables dependency installation scripts. +Npm verifies downloaded package bytes against registry integrity metadata. +For a reproducible installation, replace the package argument with `@diffexai/diffex@`, using an exact version you have checked on npmjs.org. + +The executable directory is `$(npm prefix -g)/bin` on supported platforms. +If `diffex` is not found, add that directory to your shell's PATH: + +```sh +export PATH="$(npm prefix -g)/bin:$PATH" +command -v diffex +diffex --version +``` + +Persist the PATH setting in your shell configuration if necessary. +Use a Node version manager or a user-writable npm prefix when the global directory is not writable. +Changing Node installations can change the npm prefix; check `command -v diffex` after switching. + +Start `diffex` in your project directory, use `/login` to configure your provider, and send a short prompt. +See [Quickstart](quickstart.md) and [Providers](providers.md) for authentication and execution. + +## Upgrade + +```sh +diffex update --self +diffex --version +``` + +Self-update validates the response from `https://api.diffex.ai/api/latest-version` and installs exactly `@diffexai/diffex@` using the detected package manager. +It rejects malformed metadata, foreign package names, redirects, and unavailable discovery before starting installation. +`--force` permits reinstalling a verified release; it does not bypass identity checks. +The discovery service can report no generally announced release even when a bootstrap package exists on npm. + +To install a reviewed npm version directly, including when discovery has no release: + +```sh +npm install -g --ignore-scripts --registry=https://registry.npmjs.org '@diffexai/diffex@' +diffex --version +``` + +Replace `` before running the command. +Use your original npm prefix if it differs from the active default. +Legacy clients that still trust Pi cannot migrate just by contacting the Diffex endpoint. +Install the canonical package directly with the command above, then verify `command -v diffex` and `diffex --version`. + +## Recover an interrupted installation + +Npm owns replacement of an npm installation. +If an update was interrupted or `diffex` no longer starts, repeat the pinned npm install command with the same verified version and prefix, then check `diffex --version` and `diffex --help`. +Do not remove your user-data directory to repair an executable installation. +Npm can begin filesystem replacement before a download finishes. +If retrying reports `ENOTEMPTY` while renaming to a `.diffex-...` backup, move the exact destination shown by npm to a separate backup directory and retry: + +```sh +DIFFEX_RECOVERY_FROM='/exact/destination/from/npm/error/.diffex-...' +DIFFEX_RECOVERY_DIR="$(mktemp -d)" +mv "$DIFFEX_RECOVERY_FROM" "$DIFFEX_RECOVERY_DIR/" +npm install -g --ignore-scripts --registry=https://registry.npmjs.org '@diffexai/diffex@' +diffex --version +diffex --help +``` + +Replace both placeholders with the reported Diffex backup path and verified version. +Move only that package backup, preserving it until the repaired installation works. +This procedure does not move or delete your settings or sessions. + +## Remove + +```sh +npm uninstall -g --ignore-scripts @diffexai/diffex +command -v diffex +``` + +Use the npm prefix that installed the executable. +The final command should find no executable unless another installation remains on PATH. +If your shell caches command locations, open a new terminal or refresh its command cache. + +Removal retains settings, credentials, sessions, installed extensions, and harness data under `~/.diffex/agent/`, or the directory selected by `DIFFEX_CODING_AGENT_DIR`. +Project `.diffex` directories and any separately configured session directory are also retained. +Keep this data for reinstallation; delete it separately only if you intend to erase it. + +## Bun package installations + +The same npm package can also be installed by Bun while running on Node: + +```sh +bun install -g --ignore-scripts @diffexai/diffex +diffex --version +diffex update --self +bun uninstall -g @diffexai/diffex +``` + +Use `bun pm bin -g` to locate Bun's global executable directory and add it to PATH. +Pin direct upgrades to the reviewed `@diffexai/diffex@` in the install command. +Use the manager that owns the installation for repair and removal. +User data follows the same retention policy. diff --git a/packages/coding-agent/docs/quickstart.md b/packages/coding-agent/docs/quickstart.md index fc4d6e09c..6c9849602 100644 --- a/packages/coding-agent/docs/quickstart.md +++ b/packages/coding-agent/docs/quickstart.md @@ -4,33 +4,15 @@ This page gets you from install to a useful first Diffex session. ## Install -Diffex is distributed as an npm package: +Use Node.js 22.19.0 or newer on macOS or Linux, on arm64 or x64. +Install the canonical package from npmjs.org: ```bash -npm install -g --ignore-scripts @diffexai/diffex +npm install -g --ignore-scripts --registry=https://registry.npmjs.org @diffexai/diffex +diffex --version ``` -`--ignore-scripts` disables dependency lifecycle scripts during install. Diffex does not require install scripts for normal npm installs. - -### Uninstall - -Use the package manager that installed Diffex: - -```bash -# npm -npm uninstall -g @diffexai/diffex - -# pnpm -pnpm remove -g @diffexai/diffex - -# Yarn -yarn global remove @diffexai/diffex - -# Bun -bun uninstall -g @diffexai/diffex -``` - -Uninstalling Diffex leaves settings, credentials, sessions, and installed Diffex packages in `~/.diffex/agent/`. +See [Installation](installation.md) for PATH setup, pinned versions, updates, recovery, removal, and retained user data. Then start Diffex in the project directory you want it to work on: @@ -175,4 +157,5 @@ Use `--mode json` for JSON event output or `--mode rpc` for process integration. - [Keybindings](keybindings.md) - shortcuts and customization. - [Diffex Packages](packages.md) - install shared extensions, skills, prompts, and themes. -Platform notes: [Windows](windows.md), [Termux](termux.md), [tmux](tmux.md), [Terminal setup](terminal-setup.md), [Shell aliases](shell-aliases.md). +Terminal notes: [tmux](tmux.md), [Terminal setup](terminal-setup.md), [Shell aliases](shell-aliases.md). +Windows and Android/Termux are not accepted release targets. diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index 02c527050..6c6f9e843 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -30,7 +30,36 @@ }, "files": [ "dist", - "docs", + "docs/compaction.md", + "docs/containerization.md", + "docs/custom-provider.md", + "docs/environment-variables.md", + "docs/evolution.md", + "docs/extensions.md", + "docs/installation.md", + "docs/json.md", + "docs/keybindings.md", + "docs/llama-cpp.md", + "docs/models.md", + "docs/packages.md", + "docs/prompt-templates.md", + "docs/providers.md", + "docs/quickstart.md", + "docs/rpc.md", + "docs/sdk.md", + "docs/security.md", + "docs/session-format.md", + "docs/sessions.md", + "docs/settings.md", + "docs/shell-aliases.md", + "docs/skills.md", + "docs/terminal-setup.md", + "docs/themes.md", + "docs/tmux.md", + "docs/tui.md", + "docs/usage.md", + "docs/images/interactive-mode.png", + "docs/images/tree-view.png", "CHANGELOG.md", "npm-shrinkwrap.json", "LICENSE", @@ -45,7 +74,7 @@ "build": "tsgo -p tsconfig.build.json && shx chmod +x dist/cli.js dist/rpc-entry.js && npm run copy-assets", "build:binary": "node -e \"throw new Error('Standalone distribution is deferred pending runtime redistribution review')\"", "copy-assets": "shx mkdir -p dist/modes/interactive/theme && shx cp src/modes/interactive/theme/*.json dist/modes/interactive/theme/ && shx mkdir -p dist/core/export-html/vendor && shx cp src/core/export-html/template.html src/core/export-html/template.css src/core/export-html/template.js dist/core/export-html/ && shx cp src/core/export-html/vendor/*.js dist/core/export-html/vendor/ && shx rm -rf dist/vendor && shx mkdir -p dist/vendor && shx cp -r ../harness-trace/dist dist/vendor/harness-trace && shx cp -r ../evolution/dist dist/vendor/evolution", - "copy-binary-assets": "shx cp package.json dist/ && shx cp README.md dist/ && shx cp CHANGELOG.md dist/ && shx mkdir -p dist/theme && shx cp src/modes/interactive/theme/*.json dist/theme/ && shx mkdir -p dist/export-html/vendor && shx cp src/core/export-html/template.html dist/export-html/ && shx cp src/core/export-html/vendor/*.js dist/export-html/vendor/ && shx cp -r docs dist/ && shx cp ../../node_modules/@silvia-odwyer/photon-node/photon_rs_bg.wasm dist/ && shx cp LICENSE NOTICE THIRD-PARTY-NOTICES.txt distribution-components.json dist/", + "copy-binary-assets": "shx cp package.json dist/ && shx cp README.md dist/ && shx cp CHANGELOG.md dist/ && shx mkdir -p dist/theme && shx cp src/modes/interactive/theme/*.json dist/theme/ && shx mkdir -p dist/export-html/vendor && shx cp src/core/export-html/template.html dist/export-html/ && shx cp src/core/export-html/vendor/*.js dist/export-html/vendor/ && node ../../scripts/copy-distribution-docs.mjs . dist && shx cp ../../node_modules/@silvia-odwyer/photon-node/photon_rs_bg.wasm dist/ && shx cp LICENSE NOTICE THIRD-PARTY-NOTICES.txt distribution-components.json dist/", "test": "vitest --run", "shrinkwrap": "node ../../scripts/generate-coding-agent-shrinkwrap.mjs", "prepublishOnly": "npm run clean && npm run build && npm run shrinkwrap", diff --git a/scripts/artifact-acceptance.mjs b/scripts/artifact-acceptance.mjs index c2b4cfd0a..b48f92533 100644 --- a/scripts/artifact-acceptance.mjs +++ b/scripts/artifact-acceptance.mjs @@ -320,6 +320,17 @@ export function acceptArtifacts(artifactDirectory, outputDirectory, { managers = helpers, }); } + const cli = records.find((record) => record.name === "@diffexai/diffex"); + execFileSync( + process.execPath, + [ + join(root, "scripts/npm-lifecycle.mjs"), + join(tarballDirectory, cli.filename), + outputDirectory, + ...(managers.includes("bun") ? [] : ["--skip-bun"]), + ], + { cwd: root, stdio: "inherit", timeout: 600000 }, + ); verifyTarballs(tarballDirectory, records, packages); writeFileSync( join(outputDirectory, "SHA256SUMS"), diff --git a/scripts/check-distribution-payload.py b/scripts/check-distribution-payload.py index f4f299df5..22f871bdd 100644 --- a/scripts/check-distribution-payload.py +++ b/scripts/check-distribution-payload.py @@ -15,6 +15,9 @@ from pathlib import Path, PurePosixPath +DOCUMENTATION = set(json.loads(Path(__file__).with_name('distribution-docs.json').read_text())) +LEGAL_DOCUMENTS = {'LICENSE', 'NOTICE', 'THIRD-PARTY-NOTICES.txt'} + MAX_BYTES = 1024 * 1024 * 1024 MAX_FILES = 100000 MAX_DEPTH = 8 @@ -40,6 +43,15 @@ def check_path(name, directory=False): fail(name, 'prohibited source or private path') if any(part.lower() == 'agents.md' for part in path.parts): fail(name, 'prohibited repository instructions') + if not directory: + if 'docs' in path.parts: + local = '/'.join(path.parts[path.parts.index('docs'):]) + if local not in DOCUMENTATION: + fail(name, 'unapproved distribution documentation') + elif path.suffix.lower() in {'.md', '.mdx', '.rst', '.txt', '.pdf'} and path.name not in LEGAL_DOCUMENTS: + # Only the reviewed public README and release history belong at the distribution root. + if not (path.name in {'README.md', 'CHANGELOG.md'} and len(path.parts) <= 3 and 'dist' not in path.parts): + fail(name, 'unapproved distribution documentation') declaration = name.endswith(('.d.ts', '.d.mts', '.d.cts')) if path.suffix in SOURCE_SUFFIXES and not declaration: fail(name, 'prohibited source file') @@ -150,6 +162,9 @@ def inspect_archive(file, license_file, expected_name=None, expected_version=Non if hashlib.sha256(entries[root + item['path']]).hexdigest() != item['sha256']: fail(str(file), 'runtime file hash mismatch') if metadata.get('name') == '@diffexai/diffex': + for document in DOCUMENTATION | {'README.md'}: + if not entries.get(root + document, b'').strip(): + fail(str(file), 'missing required user documentation: ' + document) for field in ['dependencies', 'optionalDependencies', 'peerDependencies']: if any(name.startswith('@diffexai/') for name in metadata.get(field, {})): fail(str(file), 'CLI must not fetch separate Diffex packages') diff --git a/scripts/check-runtime-image.py b/scripts/check-runtime-image.py new file mode 100644 index 000000000..c5871067b --- /dev/null +++ b/scripts/check-runtime-image.py @@ -0,0 +1,78 @@ +"""Inspect project payloads in every saved Docker image layer, including deleted files.""" +import argparse +import importlib.util +import json +import tarfile +import sys +from pathlib import Path, PurePosixPath + +sys.dont_write_bytecode = True + +spec = importlib.util.spec_from_file_location('payload', Path(__file__).with_name('check-distribution-payload.py')) +payload = importlib.util.module_from_spec(spec) +spec.loader.exec_module(payload) + + +def inspect_image(filename): + documents = set() + workspaces = set() + notices = set() + files = 0 + identity = None + with tarfile.open(filename) as image: + manifest = json.load(image.extractfile('manifest.json')) + if len(manifest) != 1: + raise ValueError('Expected exactly one runtime image') + config = json.load(image.extractfile(manifest[0]['Config'])) + if config['architecture'] != 'amd64' or config['os'] != 'linux': + raise ValueError('Expected Linux amd64 runtime') + for layer in manifest[0]['Layers']: + with tarfile.open(fileobj=image.extractfile(layer), mode='r|') as archive: + for entry in archive: + name = entry.name.removeprefix('./') + if not name.startswith('opt/diffex/') or not entry.isfile(): + continue + if PurePosixPath(name).name.startswith('.wh.'): + continue + local = name.removeprefix('opt/diffex/') + if local.startswith('node_modules/'): + continue # Third-party redistribution notices remain with dependencies. + if local == 'runtime-distribution.json': + identity = json.load(archive.extractfile(entry)) + continue + if not local.startswith('packages/'): + if local not in {'README.md', 'LICENSE', 'THIRD-PARTY-NOTICES.txt'}: + raise ValueError(f'Unexpected project root file: {local}') + continue + parts = PurePosixPath(local).parts + offset = 3 if parts[1] == 'session-backends' else 2 + workspace = '/'.join(parts[:offset]) + relative = '/'.join(parts[offset:]) + if relative.startswith('node_modules/'): + continue + workspaces.add(workspace) + if relative not in {'package.json', 'LICENSE', 'NOTICE', 'THIRD-PARTY-NOTICES.txt', 'distribution-components.json', 'distribution-files.json'} and not relative.startswith(('dist/', 'native/', 'docs/')): + raise ValueError(f'Unexpected runtime workspace file: {local}') + content = archive.extractfile(entry).read() + payload.scan_bytes(content, 'package/' + relative, {'bytes': 0, 'files': 0}) + if relative.startswith('docs/'): + documents.add(relative) + if relative in {'LICENSE', 'THIRD-PARTY-NOTICES.txt'} and content.strip(): + notices.add((workspace, relative)) + files += 1 + if not identity or not workspaces or documents != payload.DOCUMENTATION: + raise ValueError('Missing runtime identity, workspaces, or approved user documentation') + for workspace in workspaces: + for notice in ['LICENSE', 'THIRD-PARTY-NOTICES.txt']: + if (workspace, notice) not in notices: + raise ValueError(f'Missing runtime notice: {workspace}/{notice}') + labels = config['config'].get('Labels', {}) + if labels.get('org.opencontainers.image.revision') != identity['sourceCommit']: + raise ValueError('Runtime source identity mismatch') + return {'status': 'passed', 'projectFiles': files, 'workspaces': len(workspaces), 'documentation': sorted(documents), 'sourceCommit': identity['sourceCommit'], 'architecture': config['architecture']} + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('image') + print(json.dumps(inspect_image(parser.parse_args().image))) diff --git a/scripts/cli-distribution.mjs b/scripts/cli-distribution.mjs index e40b44b20..9b03d47cb 100644 --- a/scripts/cli-distribution.mjs +++ b/scripts/cli-distribution.mjs @@ -1,7 +1,12 @@ import { readFileSync } from "node:fs"; +import { fileURLToPath } from "node:url"; import { join, relative } from "node:path"; import { findPackageDirectories } from "./package-workspaces.mjs"; +export const distributionDocs = JSON.parse( + readFileSync(fileURLToPath(new URL("./distribution-docs.json", import.meta.url)), "utf8"), +); + export function cliDistribution(root = ".") { const workspaces = new Map( findPackageDirectories(join(root, "packages")) @@ -58,9 +63,18 @@ export function cliDistribution(root = ".") { manifest.diffexConfig = { ...manifest.diffexConfig, production: true }; delete manifest.repository; delete manifest.bugs; - manifest.files = manifest.files.filter( - (file) => !["notice", "contributing.md"].includes(file.split("/")[0].toLowerCase()), - ); + manifest.files = [ + "dist", + ...distributionDocs, + "README.md", + "CHANGELOG.md", + "npm-shrinkwrap.json", + "LICENSE", + "THIRD-PARTY-NOTICES.txt", + "distribution-components.json", + "distribution-files.json", + "release", + ]; manifest.dependencies = Object.fromEntries(Object.entries(dependencies).sort(([a], [b]) => a.localeCompare(b))); manifest.optionalDependencies = Object.fromEntries( Object.entries(optionalDependencies).sort(([a], [b]) => a.localeCompare(b)), diff --git a/scripts/copy-distribution-docs.mjs b/scripts/copy-distribution-docs.mjs new file mode 100644 index 000000000..9235f2d4c --- /dev/null +++ b/scripts/copy-distribution-docs.mjs @@ -0,0 +1,10 @@ +import { cpSync, mkdirSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { distributionDocs } from "./cli-distribution.mjs"; + +const [source, target] = process.argv.slice(2); +if (!source || !target) throw new Error("Usage: copy-distribution-docs.mjs SOURCE TARGET"); +for (const file of distributionDocs) { + mkdirSync(dirname(join(target, file)), { recursive: true }); + cpSync(join(source, file), join(target, file)); +} diff --git a/scripts/distribution-docs.json b/scripts/distribution-docs.json new file mode 100644 index 000000000..1b14ca6fa --- /dev/null +++ b/scripts/distribution-docs.json @@ -0,0 +1,32 @@ +[ + "docs/compaction.md", + "docs/containerization.md", + "docs/custom-provider.md", + "docs/environment-variables.md", + "docs/evolution.md", + "docs/extensions.md", + "docs/installation.md", + "docs/json.md", + "docs/keybindings.md", + "docs/llama-cpp.md", + "docs/models.md", + "docs/packages.md", + "docs/prompt-templates.md", + "docs/providers.md", + "docs/quickstart.md", + "docs/rpc.md", + "docs/sdk.md", + "docs/security.md", + "docs/session-format.md", + "docs/sessions.md", + "docs/settings.md", + "docs/shell-aliases.md", + "docs/skills.md", + "docs/terminal-setup.md", + "docs/themes.md", + "docs/tmux.md", + "docs/tui.md", + "docs/usage.md", + "docs/images/interactive-mode.png", + "docs/images/tree-view.png" +] diff --git a/scripts/distribution-payload.test.mjs b/scripts/distribution-payload.test.mjs index 779bf6de3..b9eda0ff6 100644 --- a/scripts/distribution-payload.test.mjs +++ b/scripts/distribution-payload.test.mjs @@ -6,6 +6,7 @@ import { tmpdir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { test } from "node:test"; +import { distributionDocs } from "./cli-distribution.mjs"; import { createDistributionManifest } from "./release-distribution-manifest.mjs"; import { getPublicWorkspacePackages } from "./release-packages.mjs"; @@ -16,6 +17,8 @@ const version = getPublicWorkspacePackages()[0].version; function fixture(extra, omit) { const directory = mkdtempSync(join(tmpdir(), "diffex-payload-test-")); const files = { + ...Object.fromEntries(distributionDocs.map((path) => [path, "Reviewed user guide"])), + "README.md": "Diffex installation and usage", "package.json": JSON.stringify({ name: "@diffexai/diffex", version, license: "Apache-2.0" }), LICENSE: readFileSync(license), NOTICE: "Diffex modifications copyright 2026 Luntraa, Inc. Pi-derived portions retain MIT terms in LICENSE.\n", @@ -72,7 +75,7 @@ test("accepts a tarball with readable JavaScript, declarations, and complete not "native/darwin/prebuilds/darwin-arm64/helper.node": Buffer.from("native fixture"), "dist/vendor/evolution/index.js": "export const value = 1;", "dist/vendor/harness-trace/index.d.ts": "export declare const value: number;", - "docs/context-files.md": "Diffex loads AGENTS.md from the user's project.\n", + "docs/usage.md": "Diffex loads AGENTS.md from the user's project.\n", }); assert.equal(result.status, 0, result.stderr); }); @@ -96,6 +99,10 @@ test("accepts consolidated attribution without a separate NOTICE file", () => { }); for (const [name, content, reason] of [ + ["docs/development.md", "Private build instructions", /unapproved distribution documentation/], + ["docs/new-internal-audit.md", "Private evidence", /unapproved distribution documentation/], + ["dist/internal-handoff.txt", "Private handoff", /unapproved distribution documentation/], + ["dist/node_modules/@diffexai/diffex-ai/docs/audit.md", "Private audit", /unapproved distribution documentation/], ["dist/implementation.ts", "export const value = 1", /prohibited source file/], ["dist/renamed.js", "export interface Hidden {", /TypeScript implementation content/], ["dist/native.c", "int main() {}", /prohibited source file/], @@ -107,7 +114,11 @@ for (const [name, content, reason] of [ ["examples/example.ts", "export const value = 1", /prohibited source or private path/], ["AGENTS.md", "Private repository instructions", /prohibited repository instructions/], ["docs/agents.MD", "Private repository instructions", /prohibited repository instructions/], - ["dist/node_modules/@diffexai/diffex-ai/AGENTS.md", "Private repository instructions", /prohibited repository instructions/], + [ + "dist/node_modules/@diffexai/diffex-ai/AGENTS.md", + "Private repository instructions", + /prohibited repository instructions/, + ], ]) { test(`rejects prohibited content in ${name}`, () => { const result = inspect({ [name]: content }); @@ -117,7 +128,7 @@ for (const [name, content, reason] of [ } test("inspects a nested archive even when renamed to a binary resource", () => { - const nested = fixture({ "dist/private.ts": "export const hidden = 1" }); + const nested = fixture({ "dist/private.ts": "export const hidden = 1" }, "README.md"); try { const result = inspect({ "dist/resource.bin": readFileSync(nested.archive) }); assert.notEqual(result.status, 0); @@ -128,7 +139,7 @@ test("inspects a nested archive even when renamed to a binary resource", () => { }); test("rejects repository instructions inside a nested archive", () => { - const nested = fixture({ "docs/AGENTS.md": "Private repository instructions" }); + const nested = fixture({ "docs/AGENTS.md": "Private repository instructions" }, "README.md"); try { const result = inspect({ "dist/resource.bin": readFileSync(nested.archive) }); assert.notEqual(result.status, 0); @@ -178,3 +189,11 @@ test("rejects duplicate members before interpreting package metadata", () => { rmSync(directory, { recursive: true, force: true }); } }); + +for (const guide of ["README.md", "docs/installation.md"]) { + test(`rejects a CLI artifact missing required user documentation ${guide}`, () => { + const result = inspect({}, guide); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /missing required user documentation/); + }); +} diff --git a/scripts/fixtures/installed-update-guard.mjs b/scripts/fixtures/installed-update-guard.mjs new file mode 100644 index 000000000..fdf404df8 --- /dev/null +++ b/scripts/fixtures/installed-update-guard.mjs @@ -0,0 +1,28 @@ +// Private acceptance fixture; the runner prepends a static import of the installed HTTP dispatcher. +import { appendFileSync, readFileSync } from "node:fs"; +import childProcess from "node:child_process"; +import { syncBuiltinESMExports } from "node:module"; +import { Socket } from "node:net"; + +const record = (event) => appendFileSync(process.env.DIFFEX_LIFECYCLE_EVENTS, `${JSON.stringify(event)}\n`); +globalThis.fetch = async (input, init) => { + const url = String(input); + record({ type: "discovery", url, redirect: init?.redirect }); + if (url !== "https://api.diffex.ai/api/latest-version" || init?.redirect !== "manual") + throw new Error("Unexpected lifecycle discovery request"); + const fixture = JSON.parse(readFileSync(process.env.DIFFEX_LIFECYCLE_RESPONSE, "utf8")); + if (fixture.failure) throw new Error("Controlled discovery outage"); + return new Response(fixture.body, { + status: fixture.status, + headers: { "content-type": "application/json", ...fixture.headers }, + }); +}; +Socket.prototype.connect = function () { + throw new Error("Installed CLI attempted unexpected network access"); +}; +const spawn = childProcess.spawn; +childProcess.spawn = (command, args, options) => { + record({ type: "process", command, args }); + return spawn(command, args, options); +}; +syncBuiltinESMExports(); diff --git a/scripts/fixtures/runtime-fixture.mjs b/scripts/fixtures/runtime-fixture.mjs new file mode 100644 index 000000000..635c78ad6 --- /dev/null +++ b/scripts/fixtures/runtime-fixture.mjs @@ -0,0 +1,68 @@ +// Mounted into acceptance containers only; never included in an image layer. +import { appendFileSync } from "node:fs"; +import { Socket } from "node:net"; +import { ModelRuntime } from "/opt/diffex/packages/coding-agent/dist/index.js"; +import { createAssistantMessageEventStream } from "/opt/diffex/packages/ai/dist/index.js"; +import "/opt/diffex/packages/coding-agent/dist/core/http-dispatcher.js"; + +const connect = Socket.prototype.connect; +Socket.prototype.connect = function (...args) { + const options = Array.isArray(args[0]) ? args[0][0] : args[0]; + const host = typeof options === "object" ? options.host : args[1]; + if (host !== "127.0.0.1" && host !== "::1") throw new Error("Runtime smoke blocked external socket"); + return connect.apply(this, args); +}; +globalThis.fetch = async (input) => { + if (String(input) !== "http://127.0.0.1:1/usage") throw new Error("Runtime smoke blocked external fetch"); + appendFileSync(process.env.DIFFEX_SMOKE_EVENTS, "usage\n"); + return Response.json({ accepted: true }); +}; +const create = ModelRuntime.create.bind(ModelRuntime); +ModelRuntime.create = async (...args) => { + const runtime = await create(...args); + runtime.registerProvider("runtime-fixture", { + baseUrl: "https://fixture.invalid", + apiKey: "local-fixture", + api: "runtime-fixture", + models: [ + { + id: "fixture", + name: "Runtime fixture", + reasoning: false, + input: ["text"], + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 }, + contextWindow: 8192, + maxTokens: 128, + }, + ], + streamSimple(model) { + appendFileSync(process.env.DIFFEX_SMOKE_EVENTS, "inference\n"); + const stream = createAssistantMessageEventStream(); + const message = { + role: "assistant", + content: [{ type: "text", text: "runtime-fixture-ok" }], + api: model.api, + provider: model.provider, + model: model.id, + usage: { + input: 1, + output: 1, + cacheRead: 0, + cacheWrite: 0, + totalTokens: 2, + cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 }, + }, + stopReason: "stop", + timestamp: Date.now(), + }; + queueMicrotask(() => { + stream.push({ type: "start", partial: message }); + stream.push({ type: "text_delta", contentIndex: 0, delta: "runtime-fixture-ok", partial: message }); + stream.push({ type: "done", reason: "stop", message }); + stream.end(message); + }); + return stream; + }, + }); + return runtime; +}; diff --git a/scripts/fixtures/runtime-smoke.mjs b/scripts/fixtures/runtime-smoke.mjs new file mode 100644 index 000000000..e9414a763 --- /dev/null +++ b/scripts/fixtures/runtime-smoke.mjs @@ -0,0 +1,141 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { once } from "node:events"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { DiffexClient } from "/opt/diffex/packages/client/dist/index.js"; +import { createWebSocketTransportFactory } from "/opt/diffex/packages/client/dist/websocket.js"; +import { WebSocket } from "/opt/diffex/node_modules/ws/wrapper.mjs"; + +const root = mkdtempSync(join(tmpdir(), "diffex-runtime-smoke-")); +for (const name of ["workspace", "agent", "state", "sessions"]) mkdirSync(join(root, name)); +const events = join(root, "events"); +writeFileSync(events, ""); +const secret = "runtime-local-smoke-fixture-32-bytes"; +const env = { + PATH: process.env.PATH, + HOME: root, + DIFFEX_OFFLINE: "1", + HOST: "127.0.0.1", + PORT: "8787", + DIFFEX_SESSION_ID: "runtime-smoke-session", + DIFFEX_WORKSPACE: join(root, "workspace"), + DIFFEX_AGENT_DIR: join(root, "agent"), + DIFFEX_CODING_AGENT_DIR: join(root, "agent"), + DIFFEX_RUNTIME_SHARED_SECRET: secret, + DIFFEX_RUNTIME_STATE_DIR: join(root, "state"), + DIFFEX_SESSION_DIR: join(root, "sessions"), + DIFFEX_OWNER_USER_ID: "runtime-smoke-user", + DIFFEX_ORG_ID: "runtime-smoke-org", + DIFFEX_MODEL_PROVIDER: "runtime-fixture", + DIFFEX_MODEL_ID: "fixture", + DIFFEX_MODEL_API_KEY: "local-unused-fixture", + DIFFEX_USAGE_ENDPOINT: "http://127.0.0.1:1/usage", + DIFFEX_SMOKE_EVENTS: events, +}; +const transport = (credential) => + createWebSocketTransportFactory({ + url: "ws://127.0.0.1:8787/runtime", + webSocket: class extends WebSocket { + constructor(url, protocols) { + super(url, protocols, { headers: credential ? { authorization: `Bearer ${credential}` } : {} }); + } + }, + }); +let child; +let completion; +const watchdog = setTimeout(() => child?.kill("SIGKILL"), 60000); +async function start() { + child = spawn( + process.execPath, + ["--import", "/acceptance/runtime-fixture.mjs", "/opt/diffex/packages/coding-agent-server/dist/cli.js"], + { cwd: env.DIFFEX_WORKSPACE, env, stdio: ["ignore", "pipe", "pipe"] }, + ); + completion = once(child, "exit"); + let output = ""; + child.stderr.on("data", (chunk) => { + output += chunk; + }); + await new Promise((done, reject) => { + child.once("error", reject); + child.once("exit", () => reject(new Error(`Runtime exited before readiness: ${output}`))); + child.stdout.on("data", (chunk) => { + output += chunk; + if (output.includes("Diffex runtime server listening")) done(); + }); + }); +} +async function stop() { + child.kill("SIGTERM"); + const [code, signal] = await completion; + assert.equal(signal, null); + assert.equal(code, 0); +} +try { + await start(); + await assert.rejects(DiffexClient.connect({ transportFactory: transport() })); + await assert.rejects(DiffexClient.connect({ transportFactory: transport("wrong-fixture") })); + const client = await DiffexClient.connect({ transportFactory: transport(secret) }); + try { + assert.deepEqual( + (await client.listSessions()).map((session) => session.id), + [env.DIFFEX_SESSION_ID], + ); + await assert.rejects(client.attachSession("another-session")); + const session = await client.attachSession(env.DIFFEX_SESSION_ID); + try { + const result = await session.prompt("Reply with the fixture confirmation."); + assert( + result.transcript.some( + (item) => + item.role === "assistant" && + item.content.some((part) => part.type === "text" && part.text.includes("runtime-fixture-ok")), + ), + ); + } finally { + await session.dispose(); + } + } finally { + await client.dispose(); + } + await stop(); + await start(); + const restored = await DiffexClient.connect({ transportFactory: transport(secret) }); + try { + const session = await restored.attachSession(env.DIFFEX_SESSION_ID); + assert(session.snapshot.transcript.some((item) => item.role === "assistant")); + await session.dispose(); + } finally { + await restored.dispose(); + } + await stop(); + assert(readFileSync(events, "utf8").includes("inference")); + assert(readFileSync(events, "utf8").includes("usage")); + console.log( + JSON.stringify({ + status: "passed", + platform: process.platform, + architecture: process.arch, + node: process.version, + startup: true, + anonymousRejected: true, + wrongCredentialRejected: true, + foreignSessionRejected: true, + promptReply: "mock-provider", + usage: "mock-endpoint", + restartPersistence: true, + shutdown: true, + externalNetwork: "disabled by container", + sourceCommit: JSON.parse(readFileSync("/opt/diffex/runtime-distribution.json", "utf8")).sourceCommit, + }), + ); +} finally { + clearTimeout(watchdog); + if (child && child.exitCode === null && child.signalCode === null) { + child.kill("SIGKILL"); + await completion; + } + rmSync(root, { recursive: true, force: true }); + assert(!existsSync(root)); +} diff --git a/scripts/npm-lifecycle.mjs b/scripts/npm-lifecycle.mjs new file mode 100644 index 000000000..828224160 --- /dev/null +++ b/scripts/npm-lifecycle.mjs @@ -0,0 +1,333 @@ +import assert from "node:assert/strict"; +import { execFile, spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + realpathSync, + readdirSync, + readlinkSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { createServer } from "node:http"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; +import { promisify } from "node:util"; +import { isolatedEnvironment } from "./artifact-acceptance.mjs"; + +const run = promisify(execFile); +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const [archiveArgument, evidenceArgument] = process.argv.slice(2); +assert(archiveArgument && evidenceArgument, "Usage: npm-lifecycle.mjs REVIEWED_TARBALL EVIDENCE_DIRECTORY"); +const archive = resolve(archiveArgument); +const output = resolve(evidenceArgument); +const bytes = readFileSync(archive); +const metadata = JSON.parse((await run("tar", ["-xOf", archive, "package/package.json"])).stdout); +assert.equal(metadata.name, "@diffexai/diffex"); +await run( + "python3", + [ + join(root, "scripts/check-distribution-payload.py"), + archive, + "--license", + join(root, "LICENSE"), + "--name", + metadata.name, + "--version", + metadata.version, + ], + { maxBuffer: 20 * 1024 * 1024 }, +); +const inventory = JSON.parse( + (await run("tar", ["-xOf", archive, "package/distribution-files.json"], { maxBuffer: 10 * 1024 * 1024 })).stdout, +); +function verifyInstalled(directory) { + assert.deepEqual(JSON.parse(readFileSync(join(directory, "distribution-files.json"), "utf8")), inventory); + for (const file of inventory.files) + assert.equal( + createHash("sha256") + .update(readFileSync(join(directory, file.path))) + .digest("hex"), + file.sha256, + `Installed runtime bytes changed: ${file.path}`, + ); +} +const temporary = mkdtempSync(join(tmpdir(), "diffex npm lifecycle ")); +const env = isolatedEnvironment(temporary); +delete env.DIFFEX_OFFLINE; +env.npm_config_fetch_retries = "0"; +env.npm_config_audit = "false"; +env.npm_config_fund = "false"; +const prefix = env.npm_config_prefix; +env.PATH = `${join(prefix, "bin")}:${env.PATH}`; +const packageRoot = join(prefix, "lib/node_modules/@diffexai/diffex"); +const executable = join(prefix, "bin/diffex"); +const options = { cwd: join(temporary, "project"), env, timeout: 180000, maxBuffer: 10 * 1024 * 1024 }; +const events = join(temporary, "events.jsonl"); +const response = join(temporary, "response.json"); +env.DIFFEX_LIFECYCLE_EVENTS = events; +env.DIFFEX_LIFECYCLE_RESPONSE = response; +const evidence = { + sourceCommit: "", + platform: process.platform, + arch: process.arch, + node: process.version, + version: metadata.version, + sha256: createHash("sha256").update(bytes).digest("hex"), + commands: [], + checks: [], + status: "failed", + cleanup: false, + limitation: + "Real npm installs reviewed bytes; discovery and scoped registry use controlled fixtures. The predecessor changes only isolated installed version metadata. No public-to-public upgrade or real provider inference is claimed.", +}; +let mode = "valid"; +let interrupt; +let tarballRequests = 0; +const server = createServer((request, reply) => { + const path = decodeURIComponent(request.url); + if (path === "/@diffexai/diffex") { + const pkg = { + ...metadata, + dist: { + tarball: `http://127.0.0.1:${server.address().port}/diffex.tgz`, + integrity: `sha512-${createHash("sha512") + .update(mode === "corrupt" ? "wrong bytes" : bytes) + .digest("base64")}`, + }, + }; + reply.setHeader("content-type", "application/json"); + reply.end( + JSON.stringify({ + name: metadata.name, + "dist-tags": { latest: metadata.version }, + versions: { [metadata.version]: pkg }, + }), + ); + } else if (path === "/diffex.tgz") { + tarballRequests++; + if (mode === "interrupt") { + interrupt?.(); + return; + } + reply.end(bytes); + } else { + reply.writeHead(404).end(); + } +}); +await new Promise((done) => server.listen(0, "127.0.0.1", done)); + +async function command(command, args, extra = {}) { + try { + console.log(`Lifecycle: ${command} ${args.join(" ")}`); + const result = await run(command, args, { ...options, ...extra }); + evidence.commands.push({ command, args, exitCode: 0 }); + return result.stdout; + } catch (error) { + evidence.commands.push({ command, args, exitCode: error.code, signal: error.signal }); + throw error; + } +} +function snapshot(directory) { + const result = {}; + function visit(current, local = "") { + for (const entry of readdirSync(current, { withFileTypes: true })) { + const file = join(current, entry.name); + const key = `${local}${entry.name}`; + if (entry.isDirectory()) visit(file, `${key}/`); + else + result[key] = entry.isSymbolicLink() + ? readlinkSync(file) + : createHash("sha256").update(readFileSync(file)).digest("hex"); + } + } + visit(directory); + return createHash("sha256") + .update(JSON.stringify(Object.entries(result).sort(([a], [b]) => a.localeCompare(b)))) + .digest("hex"); +} +function fixture(value) { + writeFileSync(response, JSON.stringify(value)); + writeFileSync(events, ""); +} +const valid = { status: 200, body: JSON.stringify({ packageName: metadata.name, version: metadata.version }) }; +const guard = join(temporary, "guard.mjs"); +const update = (force = false) => + command(process.execPath, ["--import", guard, executable, "update", "--self", ...(force ? ["--force"] : [])]); +try { + writeFileSync(env.npm_config_userconfig, `@diffexai:registry=http://127.0.0.1:${server.address().port}/\n`); + evidence.npm = (await command("npm", ["--version"])).trim(); + await command("npm", ["install", "-g", "--ignore-scripts", "--registry=https://registry.npmjs.org", archive]); + verifyInstalled(packageRoot); + evidence.sourceCommit = JSON.parse(readFileSync(join(packageRoot, "distribution-files.json"), "utf8")).sourceCommit; + assert.equal((await command("diffex", ["--version"])).trim(), metadata.version); + assert((await command("diffex", ["--help"])).includes("diffex")); + await command("diffex", ["--list-models"], { env: { ...env, DIFFEX_OFFLINE: "1" } }); + writeFileSync( + guard, + `import ${JSON.stringify(pathToFileURL(join(packageRoot, "dist/core/http-dispatcher.js")).href)};\n${readFileSync(join(root, "scripts/fixtures/installed-update-guard.mjs"), "utf8")}`, + ); + const retained = join(env.DIFFEX_CODING_AGENT_DIR, "lifecycle-retained.json"); + writeFileSync(retained, '{"retained":"session and settings fixture"}\n'); + const projectData = join(options.cwd, ".diffex"); + mkdirSync(projectData); + writeFileSync(join(projectData, "retained.txt"), "project data"); + const before = snapshot(prefix); + for (const invalid of [ + { status: 200, body: "{" }, + { status: 200, body: JSON.stringify({ packageName: "@foreign/cli", version: metadata.version }) }, + { status: 200, body: JSON.stringify({ packageName: metadata.name, version: "1.2.3-beta.1" }) }, + { status: 302, body: "{}", headers: { location: "https://foreign.invalid" } }, + { status: 503, body: "{}" }, + { status: 404, body: '{"error":"no_published_release"}' }, + ]) { + fixture(invalid); + await assert.rejects(update(true)); + assert.deepEqual(snapshot(prefix), before); + const recorded = readFileSync(events, "utf8"); + assert(!recorded.includes('"type":"process"')); + evidence.checks.push({ invalid, installationUnchanged: true }); + } + fixture(valid); + await update(); + assert.deepEqual(snapshot(prefix), before); + assert(!readFileSync(events, "utf8").includes('"type":"process"')); + // Simulate an older installed client without publishing or changing reviewed candidate bytes. + writeFileSync(join(packageRoot, "package.json"), JSON.stringify({ ...metadata, version: "0.0.0" })); + env.npm_config_cache = join(temporary, "cache-upgrade"); + await update(); + assert.equal((await command("diffex", ["--version"])).trim(), metadata.version); + assert(readFileSync(events, "utf8").includes(`${metadata.name}@${metadata.version}`)); + assert(tarballRequests > 0, "Real npm must download the target tarball"); + evidence.checks.push({ upgrade: "0.0.0 metadata fixture to reviewed candidate", realNpm: true }); + for (const fault of ["corrupt", "interrupt"]) { + mode = fault; + env.npm_config_cache = join(temporary, `cache-${fault}`); + fixture(valid); + const stable = snapshot(prefix); + if (fault === "corrupt") await assert.rejects(update(true), /integrity|checksum|EINTEGRITY/i); + else { + const child = spawn(process.execPath, ["--import", guard, executable, "update", "--self", "--force"], { + ...options, + detached: true, + stdio: "ignore", + }); + let interrupted = false; + interrupt = () => { + interrupted = true; + process.kill(-child.pid, "SIGTERM"); + }; + const deadline = setTimeout(() => process.kill(-child.pid, "SIGKILL"), 30000); + try { + await new Promise((done, reject) => { + child.on("close", done); + child.on("error", reject); + }); + } finally { + clearTimeout(deadline); + } + assert(interrupted, "Interrupted update must reach the real npm archive download"); + evidence.commands.push({ + command: "diffex update --self --force", + signal: "SIGTERM", + boundary: "npm archive download", + }); + } + const installationUnchanged = snapshot(prefix) === stable; + if (fault === "corrupt") assert(installationUnchanged, "Integrity failure mutated the prefix"); + mode = "valid"; + env.npm_config_cache = join(temporary, `cache-repair-${fault}`); + const repairArgs = [ + "install", + "-g", + "--ignore-scripts", + "--registry=https://registry.npmjs.org", + `${metadata.name}@${metadata.version}`, + ]; + let movedBackup = false; + try { + await command("npm", repairArgs); + } catch (error) { + if (fault !== "interrupt" || !error.stderr?.includes("ENOTEMPTY")) throw error; + const backup = error.stderr.match(/npm error dest (.+)/)?.[1]; + assert(backup && /^\.diffex-[a-zA-Z0-9]+$/.test(backup.split("/").pop())); + assert.equal(realpathSync(dirname(backup)), realpathSync(dirname(packageRoot))); + renameSync(backup, join(temporary, "retained-npm-recovery-backup")); + movedBackup = true; + await command("npm", repairArgs); + } + assert.equal((await command("diffex", ["--version"])).trim(), metadata.version); + verifyInstalled(packageRoot); + evidence.checks.push({ fault, installationUnchanged, movedBackup, pinnedRepair: "passed" }); + } + fixture(valid); + await update(true); + await command("npm", ["uninstall", "-g", "--ignore-scripts", metadata.name]); + assert(!existsSync(executable)); + assert(!existsSync(packageRoot)); + assert.equal(readFileSync(retained, "utf8"), '{"retained":"session and settings fixture"}\n'); + assert.equal(readFileSync(join(projectData, "retained.txt"), "utf8"), "project data"); + evidence.checks.push({ uninstall: "passed", userDataRetained: true, projectDataRetained: true }); + if (!process.argv.includes("--skip-bun")) { + env.BUN_INSTALL = join(temporary, "bun"); + env.BUN_INSTALL_BIN = join(env.BUN_INSTALL, "bin"); + env.BUN_INSTALL_GLOBAL_DIR = join(env.BUN_INSTALL, "install/global"); + env.BUN_INSTALL_CACHE_DIR = join(temporary, "bun-cache"); + env.PATH = `${env.BUN_INSTALL_BIN}:${env.PATH}`; + writeFileSync( + join(env.XDG_CONFIG_HOME, ".bunfig.toml"), + `[install.scopes]\ndiffexai = "http://127.0.0.1:${server.address().port}/"\n`, + ); + evidence.bun = (await command("bun", ["--version"])).trim(); + await command("bun", ["install", "-g", "--ignore-scripts", archive]); + assert.equal(realpathSync((await command("bun", ["pm", "bin", "-g"])).trim()), realpathSync(env.BUN_INSTALL_BIN)); + const bunPackage = join(env.BUN_INSTALL_GLOBAL_DIR, "node_modules/@diffexai/diffex"); + const bunExecutable = join(env.BUN_INSTALL_BIN, "diffex"); + verifyInstalled(bunPackage); + assert.equal((await command("diffex", ["--version"])).trim(), metadata.version); + assert((await command("diffex", ["--help"])).includes("diffex")); + writeFileSync( + guard, + `import ${JSON.stringify(pathToFileURL(join(bunPackage, "dist/core/http-dispatcher.js")).href)};\n${readFileSync(join(root, "scripts/fixtures/installed-update-guard.mjs"), "utf8")}`, + ); + fixture({ status: 200, body: JSON.stringify({ packageName: "@foreign/cli", version: metadata.version }) }); + const stable = snapshot(env.BUN_INSTALL_GLOBAL_DIR); + await assert.rejects( + command(process.execPath, ["--import", guard, bunExecutable, "update", "--self", "--force"]), + ); + assert.equal(snapshot(env.BUN_INSTALL_GLOBAL_DIR), stable); + fixture(valid); + writeFileSync(join(bunPackage, "package.json"), JSON.stringify({ ...metadata, version: "0.0.0" })); + await command(process.execPath, ["--import", guard, bunExecutable, "update", "--self"]); + assert.equal((await command("diffex", ["--version"])).trim(), metadata.version); + verifyInstalled(bunPackage); + assert(readFileSync(events, "utf8").includes('"command":"bun"')); + await command("bun", ["uninstall", "-g", metadata.name]); + assert(!existsSync(bunExecutable)); + assert(!existsSync(bunPackage)); + assert.equal(readFileSync(retained, "utf8"), '{"retained":"session and settings fixture"}\n'); + assert.equal(readFileSync(join(projectData, "retained.txt"), "utf8"), "project data"); + evidence.checks.push({ + manager: "bun", + install: "passed", + upgrade: "metadata fixture to reviewed candidate", + foreignIdentityRejected: true, + uninstall: "passed", + userDataRetained: true, + }); + } + evidence.status = "passed"; +} finally { + server.closeAllConnections(); + await new Promise((done) => server.close(done)); + rmSync(temporary, { recursive: true, force: true }); + evidence.cleanup = !existsSync(temporary); + mkdirSync(output, { recursive: true }); + writeFileSync(join(output, "npm-lifecycle.json"), `${JSON.stringify(evidence, null, 2)}\n`); +} +console.log(`npm lifecycle ${evidence.status}: ${metadata.version}`); diff --git a/scripts/prepare-distribution.test.mjs b/scripts/prepare-distribution.test.mjs index bd2b394cc..edaf7083c 100644 --- a/scripts/prepare-distribution.test.mjs +++ b/scripts/prepare-distribution.test.mjs @@ -1,11 +1,21 @@ import assert from "node:assert/strict"; import { execFileSync, spawnSync } from "node:child_process"; -import { cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { + cpSync, + existsSync, + mkdirSync, + mkdtempSync, + readdirSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { test } from "node:test"; import { PROJECT_CHANGELOGS } from "./prepare-distribution.mjs"; -import { cliDistribution } from "./cli-distribution.mjs"; +import { cliDistribution, distributionDocs } from "./cli-distribution.mjs"; test("packed CLI exposes the product homepage and preserves referenced documentation", () => { const root = mkdtempSync(join(tmpdir(), "diffex-cli-presentation-")); @@ -20,9 +30,11 @@ test("packed CLI exposes the product homepage and preserves referenced documenta "release-distribution-manifest.mjs", "package-workspaces.mjs", "check-distribution-payload.py", + "distribution-docs.json", "cli-distribution.mjs", "stage-cli-distribution.mjs", - ]) copy(`scripts/${script}`); + ]) + copy(`scripts/${script}`); for (const file of [ "LICENSE", "docs/release/distribution-policy.json", @@ -31,7 +43,8 @@ test("packed CLI exposes the product homepage and preserves referenced documenta "packages/coding-agent/npm-shrinkwrap.json", "packages/coding-agent/install-lock/package.json", "packages/coding-agent/install-lock/package-lock.json", - ]) copy(file); + ]) + copy(file); const directories = [ "packages/coding-agent", "packages/evolution", @@ -40,8 +53,7 @@ test("packed CLI exposes the product homepage and preserves referenced documenta ]; for (const directory of directories) { copy(`${directory}/package.json`); - for (const folder of ["src", "dist", "docs"]) - mkdirSync(join(root, directory, folder), { recursive: true }); + for (const folder of ["src", "dist", "docs"]) mkdirSync(join(root, directory, folder), { recursive: true }); writeFileSync(join(root, directory, "src/index.ts"), "export const value = 1;\n"); writeFileSync(join(root, directory, "dist/index.js"), "export const value = 1;\n"); for (const file of ["README.md", "CONTRIBUTING.md", "docs/skills.md", "CHANGELOG.md"]) @@ -56,7 +68,11 @@ test("packed CLI exposes the product homepage and preserves referenced documenta "invocation-traces.md", "images/doom-extension.png", "images/exy.png", + "new-internal-audit.md", + "windows.md", + "termux.md", ]; + writeFileSync(join(pkg, "docs/new-internal-audit.md"), "Private audit must never ship"); const metadata = JSON.parse(readFileSync(join(pkg, "package.json"), "utf8")); metadata.files.push("CONTRIBUTING.md"); writeFileSync(join(pkg, "package.json"), JSON.stringify(metadata)); @@ -80,30 +96,44 @@ test("packed CLI exposes the product homepage and preserves referenced documenta }), ); const archive = join(root, "stage", packed.filename); - const published = JSON.parse(execFileSync("tar", ["-xOf", archive, "package/package.json"], { encoding: "utf8" })); + const published = JSON.parse( + execFileSync("tar", ["-xOf", archive, "package/package.json"], { encoding: "utf8" }), + ); assert.equal(published.homepage, "https://diffex.ai/"); assert.equal(published.repository, undefined); assert.equal(published.bugs, undefined); assert.equal(published.diffexConfig.production, true); - assert.equal(execFileSync("tar", ["-xOf", archive, "package/README.md"]).length, 0); + assert.match( + execFileSync("tar", ["-xOf", archive, "package/README.md"], { encoding: "utf8" }), + /docs\/installation\.md/, + ); for (const file of readdirSync(join(pkg, "docs"), { recursive: true })) { const source = join(pkg, "docs", file); if (!statSync(source).isFile()) continue; const path = `docs/${file.replaceAll("\\", "/")}`; - if (excludedDocs.includes(file.replaceAll("\\", "/"))) { + if (!distributionDocs.includes(path)) { assert(!packed.files.some((entry) => entry.path === path), `${path} must not ship`); } else { - assert.deepEqual(execFileSync("tar", ["-xOf", archive, `package/${path}`]), readFileSync(source), `${path} must remain intact`); + assert.deepEqual( + execFileSync("tar", ["-xOf", archive, `package/${path}`]), + readFileSync(source), + `${path} must remain intact`, + ); } } for (const file of excludedDocs) assert(existsSync(join(pkg, "docs", file)), `${file} must remain in source`); for (const { path } of packed.files) assert(!path.split("/").some((part) => ["notice", "contributing.md"].includes(part.toLowerCase())), path); - const attribution = execFileSync("tar", ["-xOf", archive, "package/THIRD-PARTY-NOTICES.txt"], { encoding: "utf8" }); + const attribution = execFileSync("tar", ["-xOf", archive, "package/THIRD-PARTY-NOTICES.txt"], { + encoding: "utf8", + }); assert(attribution.startsWith(readFileSync(join(pkg, "NOTICE"), "utf8"))); assert.match(attribution, /highlight\.js@11\.9\.0/); assert.match(attribution, /marked@18\.0\.5/); - const result = spawnSync("python3", ["scripts/check-distribution-payload.py", archive], { cwd: root, encoding: "utf8" }); + const result = spawnSync("python3", ["scripts/check-distribution-payload.py", archive], { + cwd: root, + encoding: "utf8", + }); assert.equal(result.status, 0, result.stderr); } finally { rmSync(root, { recursive: true, force: true }); @@ -120,6 +150,7 @@ test("actual npm tarball carries notices and excludes debug sources after prepar "release-distribution-manifest.mjs", "package-workspaces.mjs", "check-distribution-payload.py", + "distribution-docs.json", ]) cpSync(`scripts/${script}`, join(root, "scripts", script)); cpSync("LICENSE", join(root, "LICENSE")); diff --git a/scripts/runtime-image.test.mjs b/scripts/runtime-image.test.mjs new file mode 100644 index 000000000..5801ef437 --- /dev/null +++ b/scripts/runtime-image.test.mjs @@ -0,0 +1,67 @@ +import assert from "node:assert/strict"; +import { execFileSync, spawnSync } from "node:child_process"; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import test from "node:test"; +import { distributionDocs } from "./cli-distribution.mjs"; + +test("image inspection rejects development documents in an earlier layer even if later removed", () => { + const directory = mkdtempSync(join(tmpdir(), "diffex runtime layers ")); + try { + const write = (file, content) => { + const path = join(directory, file); + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, content); + }; + const layer = "layer/opt/diffex/"; + const workspace = `${layer}packages/coding-agent/`; + write(`${layer}runtime-distribution.json`, JSON.stringify({ sourceCommit: "a".repeat(40) })); + write(`${workspace}LICENSE`, readFileSync("LICENSE")); + write(`${workspace}THIRD-PARTY-NOTICES.txt`, "Reviewed notice"); + for (const doc of distributionDocs) write(workspace + doc, "User-facing guide"); + write( + "config.json", + JSON.stringify({ + architecture: "amd64", + os: "linux", + config: { Labels: { "org.opencontainers.image.revision": "a".repeat(40) } }, + }), + ); + write("manifest.json", JSON.stringify([{ Config: "config.json", Layers: ["layer.tar", "cleanup.tar"] }])); + write("cleanup/opt/diffex/packages/coding-agent/docs/.wh.development.md", ""); + execFileSync("tar", ["-cf", join(directory, "cleanup.tar"), "-C", join(directory, "cleanup"), "opt"], { + env: { ...process.env, COPYFILE_DISABLE: "1" }, + }); + const inspect = () => { + execFileSync("tar", ["-cf", join(directory, "layer.tar"), "-C", join(directory, "layer"), "opt"], { + env: { ...process.env, COPYFILE_DISABLE: "1" }, + }); + execFileSync( + "tar", + [ + "-cf", + join(directory, "image.tar"), + "-C", + directory, + "manifest.json", + "config.json", + "layer.tar", + "cleanup.tar", + ], + { env: { ...process.env, COPYFILE_DISABLE: "1" } }, + ); + return spawnSync("python3", [resolve("scripts/check-runtime-image.py"), join(directory, "image.tar")], { + encoding: "utf8", + }); + }; + const valid = inspect(); + assert.equal(valid.status, 0, valid.stderr); + write(`${workspace}docs/development.md`, "Private development handoff"); + const invalid = inspect(); + assert.notEqual(invalid.status, 0); + assert.match(invalid.stderr, /unapproved distribution documentation/); + } finally { + rmSync(directory, { recursive: true, force: true }); + } +}); diff --git a/scripts/stage-cli-distribution.mjs b/scripts/stage-cli-distribution.mjs index 630e0289d..e743fd032 100644 --- a/scripts/stage-cli-distribution.mjs +++ b/scripts/stage-cli-distribution.mjs @@ -1,32 +1,22 @@ import { cpSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { createHash } from "node:crypto"; -import { join, relative } from "node:path"; +import { dirname, join } from "node:path"; import { cliDistribution } from "./cli-distribution.mjs"; import { preparePackageDistribution } from "./prepare-distribution.mjs"; export function stageCliDistribution(directory, stage) { const { manifest, bundled } = cliDistribution(); mkdirSync(stage, { recursive: true }); - const notices = [ - "LICENSE", - "THIRD-PARTY-NOTICES.txt", - "distribution-components.json", - "distribution-files.json", - ]; - const excludedDocs = new Set([ - "docs/development.md", - "docs/docs.json", - "docs/index.md", - "docs/invocation-traces.md", - "docs/images/doom-extension.png", - "docs/images/exy.png", - ]); - for (const file of manifest.files) - cpSync(join(directory, file), join(stage, file), { - recursive: true, - filter: (source) => !excludedDocs.has(relative(directory, source).replaceAll("\\", "/")), - }); - writeFileSync(join(stage, "README.md"), ""); + const notices = ["LICENSE", "THIRD-PARTY-NOTICES.txt", "distribution-components.json", "distribution-files.json"]; + for (const file of manifest.files) { + if (file === "README.md") continue; + mkdirSync(dirname(join(stage, file)), { recursive: true }); + cpSync(join(directory, file), join(stage, file), { recursive: true }); + } + writeFileSync( + join(stage, "README.md"), + "# Diffex\n\nCoding agent for your terminal.\nSee [installation, updates, and removal](docs/installation.md), [quickstart](docs/quickstart.md), and [usage](docs/usage.md).\n\nDiffex is licensed under Apache-2.0; Pi-derived portions retain their MIT terms.\nSee LICENSE and THIRD-PARTY-NOTICES.txt.\n", + ); const inventory = JSON.parse(readFileSync(join(directory, "distribution-files.json"), "utf8")); const components = JSON.parse(readFileSync(join(directory, "distribution-components.json"), "utf8")).filter( (component) => !bundled.some((pkg) => pkg.manifest.name === component.name), diff --git a/scripts/stage-runtime-distribution.mjs b/scripts/stage-runtime-distribution.mjs new file mode 100644 index 000000000..93681a590 --- /dev/null +++ b/scripts/stage-runtime-distribution.mjs @@ -0,0 +1,79 @@ +import { cpSync, existsSync, lstatSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs"; +import { dirname, isAbsolute, join, relative, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; +import { distributionDocs } from "./cli-distribution.mjs"; +import { findPackageDirectories } from "./package-workspaces.mjs"; +import { preparePackageDistribution } from "./prepare-distribution.mjs"; + +const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); +const output = resolve(process.argv[2] ?? ""); +if (!process.argv[2] || existsSync(output)) throw new Error("Specify a new runtime output directory"); +const local = relative(root, output); +if (!local.startsWith("..") && !isAbsolute(local)) throw new Error("Runtime output must be outside the source tree"); +process.chdir(root); +const workspaces = new Map( + findPackageDirectories().map((directory) => { + const manifest = JSON.parse(readFileSync(join(directory, "package.json"), "utf8")); + return [manifest.name, { directory, manifest }]; + }), +); +const selected = new Map(); +function visit(name) { + if (selected.has(name)) return; + const pkg = workspaces.get(name); + if (!pkg) throw new Error(`Missing runtime workspace: ${name}`); + selected.set(name, pkg); + for (const dependency of Object.keys({ ...pkg.manifest.dependencies, ...pkg.manifest.optionalDependencies })) + if (dependency.startsWith("@diffexai/")) visit(dependency); +} +visit("@diffexai/diffex-coding-agent-server"); +visit("@diffexai/diffex-evolution"); +visit("@diffexai/diffex-harness-trace"); +mkdirSync(output); +const notices = [ + "LICENSE", + "NOTICE", + "THIRD-PARTY-NOTICES.txt", + "distribution-components.json", + "distribution-files.json", +]; +for (const pkg of selected.values()) { + const prepared = preparePackageDistribution(pkg.directory); + const target = join(output, pkg.directory); + mkdirSync(target, { recursive: true }); + const manifest = structuredClone(pkg.manifest); + delete manifest.scripts; + delete manifest.devDependencies; + writeFileSync(join(target, "package.json"), `${JSON.stringify(manifest, null, 2)}\n`); + const files = [...prepared.files.map((file) => file.path), ...notices]; + if (pkg.manifest.name === "@diffexai/diffex") files.push(...distributionDocs); + for (const file of files) { + mkdirSync(dirname(join(target, file)), { recursive: true }); + cpSync(join(pkg.directory, file), join(target, file)); + } + if (existsSync(join(pkg.directory, "node_modules"))) + cpSync(join(pkg.directory, "node_modules"), join(target, "node_modules"), { recursive: true }); +} +cpSync(join(root, "node_modules"), join(output, "node_modules"), { + recursive: true, + verbatimSymlinks: true, + filter(source) { + if (!lstatSync(source).isSymbolicLink()) return true; + const resolved = realpathSync(source); + const workspace = [...workspaces.values()].find( + (pkg) => resolved === resolve(pkg.directory) || resolved.startsWith(`${resolve(pkg.directory)}/`), + ); + return !workspace || selected.has(workspace.manifest.name); + }, +}); +cpSync(join(root, "LICENSE"), join(output, "LICENSE")); +cpSync(join(output, "packages/coding-agent/THIRD-PARTY-NOTICES.txt"), join(output, "THIRD-PARTY-NOTICES.txt")); +writeFileSync( + join(output, "README.md"), + "# Diffex hosted runtime\n\nStart diffex-runtime-server with the session, workspace, internal authentication, model, and usage configuration supplied by your control plane.\nThis image is not an interactive CLI installation channel.\nSee packages/coding-agent/docs for user-facing runtime behavior and LICENSE and THIRD-PARTY-NOTICES.txt for legal terms.\n", +); +writeFileSync( + join(output, "runtime-distribution.json"), + `${JSON.stringify({ sourceCommit: process.env.DIFFEX_SOURCE_COMMIT, packages: [...selected.keys()], documentation: distributionDocs }, null, 2)}\n`, +); +console.log(`Prepared ${selected.size} private hosted runtime workspaces`); From 53777b7ca31a7067de3c90357702dc29b0711309 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sun, 13 Sep 2026 16:45:07 -0400 Subject: [PATCH 2/4] fix(coding-agent): handle npm cleanup errors - Match recovery paths to actual isolated backup directories so npm logging variations cannot block repair. - Reject unrelated, ambiguous, and symlinked paths before moving a retained backup. --- AGENTS.md | 2 +- scripts/npm-lifecycle.mjs | 5 ++--- scripts/npm-recovery.mjs | 17 +++++++++++++++++ scripts/npm-recovery.test.mjs | 33 +++++++++++++++++++++++++++++++++ 4 files changed, 53 insertions(+), 4 deletions(-) create mode 100644 scripts/npm-recovery.mjs create mode 100644 scripts/npm-recovery.test.mjs diff --git a/AGENTS.md b/AGENTS.md index a2b2acabd..d896780d1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -174,4 +174,4 @@ If the user's instructions conflict with any rule in this document, ask for expl - 2026-09-13: Guessed release helper paths and reused npm caches obscured validation -> Discover paths before reading and use a fresh cache when proving that an update downloads artifact bytes. -- 2026-09-13: Assumed interrupted npm downloads preserve the prefix -> Test real npm processes, retain user data, and recover ENOTEMPTY by moving only the exact stale package backup before a pinned reinstall. +- 2026-09-13: Assumed interrupted npm downloads preserve the prefix -> Test real npm processes, retain user data, and recover ENOTEMPTY by matching its reported backup against actual isolated directories instead of depending on one npm log format. diff --git a/scripts/npm-lifecycle.mjs b/scripts/npm-lifecycle.mjs index 828224160..81d00bfbe 100644 --- a/scripts/npm-lifecycle.mjs +++ b/scripts/npm-lifecycle.mjs @@ -19,6 +19,7 @@ import { dirname, join, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { isolatedEnvironment } from "./artifact-acceptance.mjs"; +import { findNpmRecoveryBackup } from "./npm-recovery.mjs"; const run = promisify(execFile); const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); @@ -254,9 +255,7 @@ try { await command("npm", repairArgs); } catch (error) { if (fault !== "interrupt" || !error.stderr?.includes("ENOTEMPTY")) throw error; - const backup = error.stderr.match(/npm error dest (.+)/)?.[1]; - assert(backup && /^\.diffex-[a-zA-Z0-9]+$/.test(backup.split("/").pop())); - assert.equal(realpathSync(dirname(backup)), realpathSync(dirname(packageRoot))); + const backup = findNpmRecoveryBackup(dirname(packageRoot), error.stderr); renameSync(backup, join(temporary, "retained-npm-recovery-backup")); movedBackup = true; await command("npm", repairArgs); diff --git a/scripts/npm-recovery.mjs b/scripts/npm-recovery.mjs new file mode 100644 index 000000000..f8d935cd8 --- /dev/null +++ b/scripts/npm-recovery.mjs @@ -0,0 +1,17 @@ +import assert from "node:assert/strict"; +import { readdirSync, realpathSync } from "node:fs"; +import { join } from "node:path"; + +export function findNpmRecoveryBackup(scope, errorOutput) { + assert(errorOutput.includes("ENOTEMPTY"), "Recovery requires npm ENOTEMPTY evidence"); + const candidates = readdirSync(scope, { withFileTypes: true }) + .filter((entry) => entry.isDirectory() && /^\.diffex-[a-zA-Z0-9]+$/.test(entry.name)) + .map((entry) => join(scope, entry.name)) + .filter((path) => errorOutput.includes(path) || errorOutput.includes(realpathSync(path))); + assert.equal( + candidates.length, + 1, + `Expected one reported Diffex backup inside the isolated prefix:\n${errorOutput}`, + ); + return candidates[0]; +} diff --git a/scripts/npm-recovery.test.mjs b/scripts/npm-recovery.test.mjs new file mode 100644 index 000000000..29f6d7579 --- /dev/null +++ b/scripts/npm-recovery.test.mjs @@ -0,0 +1,33 @@ +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { findNpmRecoveryBackup } from "./npm-recovery.mjs"; + +test("npm recovery identifies the reported package backup across rename and cleanup errors", () => { + const root = mkdtempSync(join(tmpdir(), "diffex recovery ")); + try { + const scope = join(root, "@diffexai"); + const backup = join(scope, ".diffex-Abcd1234"); + mkdirSync(backup, { recursive: true }); + for (const output of [ + `npm error code ENOTEMPTY\nnpm error dest ${backup}\n`, + `npm error ENOTEMPTY: directory not empty, rename 'diffex' -> '${backup}'`, + `npm error ENOTEMPTY: directory not empty, rmdir '${backup}/dist'`, + ]) + assert.equal(findNpmRecoveryBackup(scope, output), backup); + assert.throws(() => findNpmRecoveryBackup(scope, "ENOTEMPTY at an unrelated path")); + assert.throws(() => findNpmRecoveryBackup(scope, `ENOENT ${backup}`)); + const outside = join(root, ".diffex-Outside"); + mkdirSync(outside); + symlinkSync(outside, join(scope, ".diffex-Link")); + assert.throws(() => findNpmRecoveryBackup(scope, `ENOTEMPTY ${outside}`)); + assert.throws(() => findNpmRecoveryBackup(scope, `ENOTEMPTY ${join(scope, ".diffex-Link")}`)); + const other = join(scope, ".diffex-Other"); + mkdirSync(other); + assert.throws(() => findNpmRecoveryBackup(scope, `ENOTEMPTY ${backup} ${other}`)); + } finally { + rmSync(root, { recursive: true, force: true }); + } +}); From 3e1a87fe4ff1769408a5927e8a2056f124175cc0 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sun, 13 Sep 2026 16:55:28 -0400 Subject: [PATCH 3/4] fix(coding-agent): recover partial installs - Preserve the interrupted CLI directory when npm cannot clean its partially installed dependencies. - Keep user data and unrelated global packages untouched during the pinned recovery. --- AGENTS.md | 2 +- docs/release/artifact-acceptance.md | 2 +- packages/coding-agent/docs/installation.md | 12 ++++++---- scripts/npm-lifecycle.mjs | 4 ++-- scripts/npm-recovery.mjs | 14 ++++++----- scripts/npm-recovery.test.mjs | 27 ++++++++++++++-------- 6 files changed, 38 insertions(+), 23 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index d896780d1..619d74b21 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -174,4 +174,4 @@ If the user's instructions conflict with any rule in this document, ask for expl - 2026-09-13: Guessed release helper paths and reused npm caches obscured validation -> Discover paths before reading and use a fresh cache when proving that an update downloads artifact bytes. -- 2026-09-13: Assumed interrupted npm downloads preserve the prefix -> Test real npm processes, retain user data, and recover ENOTEMPTY by matching its reported backup against actual isolated directories instead of depending on one npm log format. +- 2026-09-13: Assumed interrupted npm downloads preserve the prefix -> Test real npm processes, retain user data, and recover ENOTEMPTY by matching the affected installation or backup against actual isolated directories instead of assuming every failure is a backup rename. diff --git a/docs/release/artifact-acceptance.md b/docs/release/artifact-acceptance.md index b0b86c6e0..9fdfee1e1 100644 --- a/docs/release/artifact-acceptance.md +++ b/docs/release/artifact-acceptance.md @@ -36,7 +36,7 @@ Only discovery and the Diffex-scoped registry are controlled fixtures; dependenc An isolated version-metadata predecessor exercises upgrade to the reviewed candidate without publishing a fake release. Fresh caches require actual target downloads. Malformed and foreign metadata, redirects, outages, no release, and checksum mismatches are rejected without changing the installed prefix, including force. -An interrupted npm download demonstrates the documented pinned repair and, when npm reports `ENOTEMPTY`, relocation of its exact stale backup before retrying. +An interrupted npm download demonstrates the documented pinned repair and, when npm reports `ENOTEMPTY`, relocation of the affected CLI installation or its exact stale backup before retrying. Interruption itself can mutate npm's prefix and is not claimed to preserve the executable. Uninstall verifies executable removal and retained global and project data. The same harness verifies Bun global installation and PATH, rejects a foreign release identity without mutation, upgrades with the real Bun subprocess, checks the installed runtime hashes, and uninstalls while retaining user and project data. diff --git a/packages/coding-agent/docs/installation.md b/packages/coding-agent/docs/installation.md index 2914fa258..cb9b44a90 100644 --- a/packages/coding-agent/docs/installation.md +++ b/packages/coding-agent/docs/installation.md @@ -67,10 +67,13 @@ Npm owns replacement of an npm installation. If an update was interrupted or `diffex` no longer starts, repeat the pinned npm install command with the same verified version and prefix, then check `diffex --version` and `diffex --help`. Do not remove your user-data directory to repair an executable installation. Npm can begin filesystem replacement before a download finishes. -If retrying reports `ENOTEMPTY` while renaming to a `.diffex-...` backup, move the exact destination shown by npm to a separate backup directory and retry: +If retrying reports `ENOTEMPTY` while renaming to a `.diffex-...` backup, move that reported backup directory aside. +If npm instead reports `ENOTEMPTY` while removing a dependency inside `$(npm root -g)/@diffexai/diffex`, move that complete CLI installation directory aside. +This preserves the partially installed package and lets npm install into an empty destination. +Use a separate backup directory and retry: ```sh -DIFFEX_RECOVERY_FROM='/exact/destination/from/npm/error/.diffex-...' +DIFFEX_RECOVERY_FROM='/exact/affected/CLI/installation/or/backup' DIFFEX_RECOVERY_DIR="$(mktemp -d)" mv "$DIFFEX_RECOVERY_FROM" "$DIFFEX_RECOVERY_DIR/" npm install -g --ignore-scripts --registry=https://registry.npmjs.org '@diffexai/diffex@' @@ -78,8 +81,9 @@ diffex --version diffex --help ``` -Replace both placeholders with the reported Diffex backup path and verified version. -Move only that package backup, preserving it until the repaired installation works. +Replace both placeholders with the affected Diffex installation or backup path and the verified version. +Move only the `@diffexai/diffex` package directory or its reported `.diffex-...` backup, preserving it until the repaired installation works. +Do not move the global prefix, its shared `node_modules`, or another package. This procedure does not move or delete your settings or sessions. ## Remove diff --git a/scripts/npm-lifecycle.mjs b/scripts/npm-lifecycle.mjs index 81d00bfbe..589f005dd 100644 --- a/scripts/npm-lifecycle.mjs +++ b/scripts/npm-lifecycle.mjs @@ -19,7 +19,7 @@ import { dirname, join, resolve } from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; import { promisify } from "node:util"; import { isolatedEnvironment } from "./artifact-acceptance.mjs"; -import { findNpmRecoveryBackup } from "./npm-recovery.mjs"; +import { findNpmRecoveryDirectory } from "./npm-recovery.mjs"; const run = promisify(execFile); const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); @@ -255,7 +255,7 @@ try { await command("npm", repairArgs); } catch (error) { if (fault !== "interrupt" || !error.stderr?.includes("ENOTEMPTY")) throw error; - const backup = findNpmRecoveryBackup(dirname(packageRoot), error.stderr); + const backup = findNpmRecoveryDirectory(dirname(packageRoot), error.stderr); renameSync(backup, join(temporary, "retained-npm-recovery-backup")); movedBackup = true; await command("npm", repairArgs); diff --git a/scripts/npm-recovery.mjs b/scripts/npm-recovery.mjs index f8d935cd8..35c4b27b0 100644 --- a/scripts/npm-recovery.mjs +++ b/scripts/npm-recovery.mjs @@ -1,17 +1,19 @@ import assert from "node:assert/strict"; import { readdirSync, realpathSync } from "node:fs"; -import { join } from "node:path"; +import { basename, join } from "node:path"; -export function findNpmRecoveryBackup(scope, errorOutput) { +export function findNpmRecoveryDirectory(scope, errorOutput) { assert(errorOutput.includes("ENOTEMPTY"), "Recovery requires npm ENOTEMPTY evidence"); const candidates = readdirSync(scope, { withFileTypes: true }) - .filter((entry) => entry.isDirectory() && /^\.diffex-[a-zA-Z0-9]+$/.test(entry.name)) + .filter((entry) => entry.isDirectory() && (entry.name === "diffex" || /^\.diffex-[a-zA-Z0-9]+$/.test(entry.name))) .map((entry) => join(scope, entry.name)) .filter((path) => errorOutput.includes(path) || errorOutput.includes(realpathSync(path))); + const backups = candidates.filter((path) => basename(path).startsWith(".diffex-")); + const affected = backups.length ? backups : candidates; assert.equal( - candidates.length, + affected.length, 1, - `Expected one reported Diffex backup inside the isolated prefix:\n${errorOutput}`, + `Expected one reported Diffex installation or backup inside the isolated prefix:\n${errorOutput}`, ); - return candidates[0]; + return affected[0]; } diff --git a/scripts/npm-recovery.test.mjs b/scripts/npm-recovery.test.mjs index 29f6d7579..240b6a2ac 100644 --- a/scripts/npm-recovery.test.mjs +++ b/scripts/npm-recovery.test.mjs @@ -3,30 +3,39 @@ import { mkdirSync, mkdtempSync, rmSync, symlinkSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; -import { findNpmRecoveryBackup } from "./npm-recovery.mjs"; +import { findNpmRecoveryDirectory } from "./npm-recovery.mjs"; -test("npm recovery identifies the reported package backup across rename and cleanup errors", () => { +test("npm recovery identifies only the affected CLI installation or backup across rename and cleanup errors", () => { const root = mkdtempSync(join(tmpdir(), "diffex recovery ")); try { const scope = join(root, "@diffexai"); const backup = join(scope, ".diffex-Abcd1234"); mkdirSync(backup, { recursive: true }); + const installed = join(scope, "diffex"); + mkdirSync(installed); for (const output of [ `npm error code ENOTEMPTY\nnpm error dest ${backup}\n`, - `npm error ENOTEMPTY: directory not empty, rename 'diffex' -> '${backup}'`, + `npm error ENOTEMPTY: directory not empty, rename '${installed}' -> '${backup}'`, `npm error ENOTEMPTY: directory not empty, rmdir '${backup}/dist'`, ]) - assert.equal(findNpmRecoveryBackup(scope, output), backup); - assert.throws(() => findNpmRecoveryBackup(scope, "ENOTEMPTY at an unrelated path")); - assert.throws(() => findNpmRecoveryBackup(scope, `ENOENT ${backup}`)); + assert.equal(findNpmRecoveryDirectory(scope, output), backup); + assert.equal( + findNpmRecoveryDirectory( + scope, + `npm error ENOTEMPTY: directory not empty, rmdir '${installed}/node_modules/@mistralai/mistralai/esm/models/components'`, + ), + installed, + ); + assert.throws(() => findNpmRecoveryDirectory(scope, "ENOTEMPTY at an unrelated path")); + assert.throws(() => findNpmRecoveryDirectory(scope, `ENOENT ${backup}`)); const outside = join(root, ".diffex-Outside"); mkdirSync(outside); symlinkSync(outside, join(scope, ".diffex-Link")); - assert.throws(() => findNpmRecoveryBackup(scope, `ENOTEMPTY ${outside}`)); - assert.throws(() => findNpmRecoveryBackup(scope, `ENOTEMPTY ${join(scope, ".diffex-Link")}`)); + assert.throws(() => findNpmRecoveryDirectory(scope, `ENOTEMPTY ${outside}`)); + assert.throws(() => findNpmRecoveryDirectory(scope, `ENOTEMPTY ${join(scope, ".diffex-Link")}`)); const other = join(scope, ".diffex-Other"); mkdirSync(other); - assert.throws(() => findNpmRecoveryBackup(scope, `ENOTEMPTY ${backup} ${other}`)); + assert.throws(() => findNpmRecoveryDirectory(scope, `ENOTEMPTY ${backup} ${other}`)); } finally { rmSync(root, { recursive: true, force: true }); } From 59045f4a8e192c8db7d393386d3ab82d28b03e59 Mon Sep 17 00:00:00 2001 From: Jay Yu Date: Sun, 13 Sep 2026 17:34:48 -0400 Subject: [PATCH 4/4] fix(coding-agent): exclude separate NOTICE - Keep project attribution in the consolidated third-party notices file. - Reject redundant NOTICE files in npm artifacts and hosted runtime layers. --- AGENTS.md | 2 ++ packages/coding-agent/package.json | 3 +-- scripts/check-distribution-payload.py | 4 +++- scripts/check-runtime-image.py | 2 +- scripts/distribution-payload.test.mjs | 8 ++++---- scripts/prepare-distribution.mjs | 2 +- scripts/prepare-distribution.test.mjs | 5 ++++- scripts/runtime-image.test.mjs | 7 ++++++- scripts/stage-runtime-distribution.mjs | 8 +------- 9 files changed, 23 insertions(+), 18 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 619d74b21..1bd35aeb1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -175,3 +175,5 @@ If the user's instructions conflict with any rule in this document, ask for expl - 2026-09-13: Guessed release helper paths and reused npm caches obscured validation -> Discover paths before reading and use a fresh cache when proving that an update downloads artifact bytes. - 2026-09-13: Assumed interrupted npm downloads preserve the prefix -> Test real npm processes, retain user data, and recover ENOTEMPTY by matching the affected installation or backup against actual isolated directories instead of assuming every failure is a backup rename. + +- 2026-09-13: Runtime staging copied a redundant project NOTICE after attribution was consolidated -> Generate only the consolidated attribution file and reject separate NOTICE payloads. diff --git a/packages/coding-agent/package.json b/packages/coding-agent/package.json index 6c6f9e843..6035cd036 100644 --- a/packages/coding-agent/package.json +++ b/packages/coding-agent/package.json @@ -63,7 +63,6 @@ "CHANGELOG.md", "npm-shrinkwrap.json", "LICENSE", - "NOTICE", "THIRD-PARTY-NOTICES.txt", "distribution-components.json", "distribution-files.json", @@ -74,7 +73,7 @@ "build": "tsgo -p tsconfig.build.json && shx chmod +x dist/cli.js dist/rpc-entry.js && npm run copy-assets", "build:binary": "node -e \"throw new Error('Standalone distribution is deferred pending runtime redistribution review')\"", "copy-assets": "shx mkdir -p dist/modes/interactive/theme && shx cp src/modes/interactive/theme/*.json dist/modes/interactive/theme/ && shx mkdir -p dist/core/export-html/vendor && shx cp src/core/export-html/template.html src/core/export-html/template.css src/core/export-html/template.js dist/core/export-html/ && shx cp src/core/export-html/vendor/*.js dist/core/export-html/vendor/ && shx rm -rf dist/vendor && shx mkdir -p dist/vendor && shx cp -r ../harness-trace/dist dist/vendor/harness-trace && shx cp -r ../evolution/dist dist/vendor/evolution", - "copy-binary-assets": "shx cp package.json dist/ && shx cp README.md dist/ && shx cp CHANGELOG.md dist/ && shx mkdir -p dist/theme && shx cp src/modes/interactive/theme/*.json dist/theme/ && shx mkdir -p dist/export-html/vendor && shx cp src/core/export-html/template.html dist/export-html/ && shx cp src/core/export-html/vendor/*.js dist/export-html/vendor/ && node ../../scripts/copy-distribution-docs.mjs . dist && shx cp ../../node_modules/@silvia-odwyer/photon-node/photon_rs_bg.wasm dist/ && shx cp LICENSE NOTICE THIRD-PARTY-NOTICES.txt distribution-components.json dist/", + "copy-binary-assets": "shx cp package.json dist/ && shx cp README.md dist/ && shx cp CHANGELOG.md dist/ && shx mkdir -p dist/theme && shx cp src/modes/interactive/theme/*.json dist/theme/ && shx mkdir -p dist/export-html/vendor && shx cp src/core/export-html/template.html dist/export-html/ && shx cp src/core/export-html/vendor/*.js dist/export-html/vendor/ && node ../../scripts/copy-distribution-docs.mjs . dist && shx cp ../../node_modules/@silvia-odwyer/photon-node/photon_rs_bg.wasm dist/ && shx cp LICENSE THIRD-PARTY-NOTICES.txt distribution-components.json dist/", "test": "vitest --run", "shrinkwrap": "node ../../scripts/generate-coding-agent-shrinkwrap.mjs", "prepublishOnly": "npm run clean && npm run build && npm run shrinkwrap", diff --git a/scripts/check-distribution-payload.py b/scripts/check-distribution-payload.py index 22f871bdd..b2af21b8c 100644 --- a/scripts/check-distribution-payload.py +++ b/scripts/check-distribution-payload.py @@ -16,7 +16,7 @@ DOCUMENTATION = set(json.loads(Path(__file__).with_name('distribution-docs.json').read_text())) -LEGAL_DOCUMENTS = {'LICENSE', 'NOTICE', 'THIRD-PARTY-NOTICES.txt'} +LEGAL_DOCUMENTS = {'LICENSE', 'THIRD-PARTY-NOTICES.txt'} MAX_BYTES = 1024 * 1024 * 1024 MAX_FILES = 100000 @@ -43,6 +43,8 @@ def check_path(name, directory=False): fail(name, 'prohibited source or private path') if any(part.lower() == 'agents.md' for part in path.parts): fail(name, 'prohibited repository instructions') + if not directory and path.name.upper() == 'NOTICE': + fail(name, 'separate NOTICE file is excluded; attribution belongs in THIRD-PARTY-NOTICES.txt') if not directory: if 'docs' in path.parts: local = '/'.join(path.parts[path.parts.index('docs'):]) diff --git a/scripts/check-runtime-image.py b/scripts/check-runtime-image.py index c5871067b..100be0180 100644 --- a/scripts/check-runtime-image.py +++ b/scripts/check-runtime-image.py @@ -51,7 +51,7 @@ def inspect_image(filename): if relative.startswith('node_modules/'): continue workspaces.add(workspace) - if relative not in {'package.json', 'LICENSE', 'NOTICE', 'THIRD-PARTY-NOTICES.txt', 'distribution-components.json', 'distribution-files.json'} and not relative.startswith(('dist/', 'native/', 'docs/')): + if relative not in {'package.json', 'LICENSE', 'THIRD-PARTY-NOTICES.txt', 'distribution-components.json', 'distribution-files.json'} and not relative.startswith(('dist/', 'native/', 'docs/')): raise ValueError(f'Unexpected runtime workspace file: {local}') content = archive.extractfile(entry).read() payload.scan_bytes(content, 'package/' + relative, {'bytes': 0, 'files': 0}) diff --git a/scripts/distribution-payload.test.mjs b/scripts/distribution-payload.test.mjs index b9eda0ff6..bd2d784f6 100644 --- a/scripts/distribution-payload.test.mjs +++ b/scripts/distribution-payload.test.mjs @@ -21,7 +21,6 @@ function fixture(extra, omit) { "README.md": "Diffex installation and usage", "package.json": JSON.stringify({ name: "@diffexai/diffex", version, license: "Apache-2.0" }), LICENSE: readFileSync(license), - NOTICE: "Diffex modifications copyright 2026 Luntraa, Inc. Pi-derived portions retain MIT terms in LICENSE.\n", "THIRD-PARTY-NOTICES.txt": "Test fixture with no external bundled dependencies.\n", "distribution-components.json": JSON.stringify([{ name: "Diffex", license: "Apache-2.0 AND MIT" }]), "dist/index.js": "export const value = 1;\n", @@ -93,9 +92,10 @@ for (const notice of [ }); } -test("accepts consolidated attribution without a separate NOTICE file", () => { - const result = inspect({}, "NOTICE"); - assert.equal(result.status, 0, result.stderr); +test("rejects a separate NOTICE file because attribution is consolidated", () => { + const result = inspect({ NOTICE: "Duplicate project attribution" }); + assert.notEqual(result.status, 0); + assert.match(result.stderr, /separate NOTICE file/); }); for (const [name, content, reason] of [ diff --git a/scripts/prepare-distribution.mjs b/scripts/prepare-distribution.mjs index a571da182..468c92e45 100644 --- a/scripts/prepare-distribution.mjs +++ b/scripts/prepare-distribution.mjs @@ -96,7 +96,7 @@ export function preparePackageDistribution(directory) { }); } cpSync(join(root, "LICENSE"), join(absolute, "LICENSE")); - writeFileSync(join(absolute, "NOTICE"), notice); + rmSync(join(absolute, "NOTICE"), { force: true }); writeFileSync(join(absolute, "THIRD-PARTY-NOTICES.txt"), notices); for (const component of components) { if (component.notice) diff --git a/scripts/prepare-distribution.test.mjs b/scripts/prepare-distribution.test.mjs index edaf7083c..12818c9e4 100644 --- a/scripts/prepare-distribution.test.mjs +++ b/scripts/prepare-distribution.test.mjs @@ -127,7 +127,8 @@ test("packed CLI exposes the product homepage and preserves referenced documenta const attribution = execFileSync("tar", ["-xOf", archive, "package/THIRD-PARTY-NOTICES.txt"], { encoding: "utf8", }); - assert(attribution.startsWith(readFileSync(join(pkg, "NOTICE"), "utf8"))); + assert.match(attribution, /^Diffex\nCopyright 2026 Luntraa, Inc\./); + assert(!existsSync(join(pkg, "NOTICE"))); assert.match(attribution, /highlight\.js@11\.9\.0/); assert.match(attribution, /marked@18\.0\.5/); const result = spawnSync("python3", ["scripts/check-distribution-payload.py", archive], { @@ -179,6 +180,7 @@ test("actual npm tarball carries notices and excludes debug sources after prepar ], }), ); + writeFileSync(join(pkg, "NOTICE"), "Stale generated attribution"); writeFileSync(join(pkg, "dist/index.js"), "export const value = 1;\n//# sourceMappingURL=index.js.map\n"); writeFileSync(join(pkg, "dist/index.js.map"), '{"sourcesContent":["private TypeScript"]}'); writeFileSync( @@ -190,6 +192,7 @@ test("actual npm tarball carries notices and excludes debug sources after prepar cwd: tmpdir(), env: { ...process.env, DIFFEX_SOURCE_COMMIT: "a".repeat(40) }, }); + assert.equal(existsSync(join(pkg, "NOTICE")), false); assert.equal(existsSync(join(pkg, "dist/index.js.map")), false); assert.match( readFileSync( diff --git a/scripts/runtime-image.test.mjs b/scripts/runtime-image.test.mjs index 5801ef437..0d20fef93 100644 --- a/scripts/runtime-image.test.mjs +++ b/scripts/runtime-image.test.mjs @@ -6,7 +6,7 @@ import { dirname, join, resolve } from "node:path"; import test from "node:test"; import { distributionDocs } from "./cli-distribution.mjs"; -test("image inspection rejects development documents in an earlier layer even if later removed", () => { +test("image inspection rejects separate NOTICE files and development documents in earlier layers", () => { const directory = mkdtempSync(join(tmpdir(), "diffex runtime layers ")); try { const write = (file, content) => { @@ -57,6 +57,11 @@ test("image inspection rejects development documents in an earlier layer even if }; const valid = inspect(); assert.equal(valid.status, 0, valid.stderr); + write(`${workspace}NOTICE`, "Duplicate project attribution"); + const notice = inspect(); + assert.notEqual(notice.status, 0); + assert.match(notice.stderr, /NOTICE/); + rmSync(join(directory, workspace, "NOTICE")); write(`${workspace}docs/development.md`, "Private development handoff"); const invalid = inspect(); assert.notEqual(invalid.status, 0); diff --git a/scripts/stage-runtime-distribution.mjs b/scripts/stage-runtime-distribution.mjs index 93681a590..28120545b 100644 --- a/scripts/stage-runtime-distribution.mjs +++ b/scripts/stage-runtime-distribution.mjs @@ -30,13 +30,7 @@ visit("@diffexai/diffex-coding-agent-server"); visit("@diffexai/diffex-evolution"); visit("@diffexai/diffex-harness-trace"); mkdirSync(output); -const notices = [ - "LICENSE", - "NOTICE", - "THIRD-PARTY-NOTICES.txt", - "distribution-components.json", - "distribution-files.json", -]; +const notices = ["LICENSE", "THIRD-PARTY-NOTICES.txt", "distribution-components.json", "distribution-files.json"]; for (const pkg of selected.values()) { const prepared = preparePackageDistribution(pkg.directory); const target = join(output, pkg.directory);