diff --git a/.github/workflows/jest-publish.yml b/.github/workflows/jest-publish.yml index 8fd0ca7..2d6dbf3 100644 --- a/.github/workflows/jest-publish.yml +++ b/.github/workflows/jest-publish.yml @@ -17,14 +17,23 @@ name: "@keploy/jest publish" # try to publish that package under this tag's name. jest/README.md, # "Releasing", says the same. # -# On a pull request the same package is published as a dry run, with no -# token, so a package that would ship without its build shows before a tag. +# The package is built and packed once, and that tarball is checked and then +# published, so what ships is what was checked. The license check +# (scripts/license-gate.mjs, as in sdk-publish.yml) installs the tarball as a +# user's `npm install` does and refuses it when anything it installs names a +# license outside its allow-list. test/workflows.test.mjs holds this +# workflow to that, and both of its jobs run those rows first. +# +# On a pull request the same tarball is packed, checked and published as a +# dry run, with no token, so a package that would ship without its build, or +# with a license it may not carry, shows before a tag. on: push: tags: ["jest-v*"] pull_request: paths: - "jest/**" + - "scripts/license-gate.mjs" - ".github/workflows/jest-publish.yml" - ".github/workflows/jest-sdk.yml" @@ -41,6 +50,7 @@ jobs: if: github.event_name == 'push' needs: rows runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read id-token: write # npm --provenance signs the package with this run's identity @@ -57,6 +67,15 @@ jobs: # scope to it (@keploy:registry), in the runner's own npmrc. registry-url: https://registry.npmjs.org scope: "@keploy" + - name: Workflow rows + # test/workflows.test.mjs, from the repository's root, so that every + # tag and pull request this workflow runs on checks that it still + # publishes only the tarball its Licenses step passed. The root's + # install, with no scripts, is only for the rows' YAML parser. + working-directory: . + run: | + npm ci --ignore-scripts + node --test test/workflows.test.mjs - name: The tag names package.json's version run: node scripts/release-version.mjs "$GITHUB_REF_NAME" - name: The npm token is set @@ -71,6 +90,26 @@ jobs: got=$(npm config get @keploy:registry) test "$got" = https://registry.npmjs.org/ || { echo "::error::@keploy:registry is '$got', not https://registry.npmjs.org/; refusing to publish"; exit 1; } - run: npm ci + - name: Build + run: npm run build + - name: Pack + id: pack + run: | + name=$(npm pack --pack-destination "$RUNNER_TEMP" | tail -n 1) + test -f "$RUNNER_TEMP/$name" || { echo "::error::npm pack wrote no $name"; exit 1; } + echo "tarball=$RUNNER_TEMP/$name" >> "$GITHUB_OUTPUT" + - name: The package carries its build + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: | + tar -tzf "$TARBALL" > "$RUNNER_TEMP/files.txt" + for f in dist/index.js dist/index.d.ts package.json README.md; do + grep -qx "package/$f" "$RUNNER_TEMP/files.txt" || { echo "::error::the package would ship without $f"; exit 1; } + done + - name: Licenses + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: node ../scripts/license-gate.mjs "$TARBALL" - name: The token authenticates # Fails here, named, on a token that is expired, revoked or pasted # with a stray character, before anything is uploaded. @@ -78,15 +117,16 @@ jobs: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} run: npm whoami --registry https://registry.npmjs.org/ - name: Publish - # prepublishOnly builds dist/ first. env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: npm publish --access public --provenance + TARBALL: ${{ steps.pack.outputs.tarball }} + run: npm publish "$TARBALL" --access public --provenance dry-run: name: publish dry run if: github.event_name == 'pull_request' runs-on: ubuntu-latest + timeout-minutes: 30 defaults: run: shell: bash # -eo pipefail: a failed command before a pipe fails the step @@ -96,12 +136,37 @@ jobs: - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 + - name: Workflow rows + # test/workflows.test.mjs, from the repository's root, so that every + # tag and pull request this workflow runs on checks that it still + # publishes only the tarball its Licenses step passed. The root's + # install, with no scripts, is only for the rows' YAML parser. + working-directory: . + run: | + npm ci --ignore-scripts + node --test test/workflows.test.mjs - run: npm ci - - name: Publish, dry run - run: npm publish --dry-run --access public 2>&1 | tee "$RUNNER_TEMP/publish.log" + - name: Build + run: npm run build + - name: Pack + id: pack + run: | + name=$(npm pack --pack-destination "$RUNNER_TEMP" | tail -n 1) + test -f "$RUNNER_TEMP/$name" || { echo "::error::npm pack wrote no $name"; exit 1; } + echo "tarball=$RUNNER_TEMP/$name" >> "$GITHUB_OUTPUT" - name: The package carries its build - # A fresh checkout has no dist/; only prepublishOnly puts it there. + env: + TARBALL: ${{ steps.pack.outputs.tarball }} run: | + tar -tzf "$TARBALL" > "$RUNNER_TEMP/files.txt" for f in dist/index.js dist/index.d.ts package.json README.md; do - grep -qE "npm notice [0-9.]+[kMG]?B +$f\$" "$RUNNER_TEMP/publish.log" || { echo "::error::the package would ship without $f"; exit 1; } + grep -qx "package/$f" "$RUNNER_TEMP/files.txt" || { echo "::error::the package would ship without $f"; exit 1; } done + - name: Licenses + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: node ../scripts/license-gate.mjs "$TARBALL" + - name: Publish, dry run + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: npm publish "$TARBALL" --dry-run --access public diff --git a/.github/workflows/jest-sdk.yml b/.github/workflows/jest-sdk.yml index 3080b51..37d2ad4 100644 --- a/.github/workflows/jest-sdk.yml +++ b/.github/workflows/jest-sdk.yml @@ -18,6 +18,8 @@ jobs: rows: name: build and rows (Node ${{ matrix.node }}, Jest ${{ matrix.jest }}) runs-on: ubuntu-latest + # A leg takes a few minutes, npm installs included: one past 30 is a hang. + timeout-minutes: 30 strategy: # One leg failing must not cancel the others: each leg's result is its # own. diff --git a/.github/workflows/playwright-publish.yml b/.github/workflows/playwright-publish.yml index ee8e073..de875ef 100644 --- a/.github/workflows/playwright-publish.yml +++ b/.github/workflows/playwright-publish.yml @@ -7,7 +7,9 @@ name: "@keploy/playwright publish" # playwright-vX.Y.Z on that commit. The rows run on the tagged commit first; # the publish runs only when every leg passes, and refuses a tag that does not # name package.json's version (scripts/release-version.mjs). npm itself -# refuses a version it already has. +# refuses a version it already has. Every action the publish and dry-run +# jobs run is pinned by commit: the publish job holds the npm token and +# signs the package's provenance. # # Push playwright-vX.Y.Z as a plain git tag, never as a GitHub release: a # published GitHub release also starts any workflow of this repository that @@ -15,14 +17,23 @@ name: "@keploy/playwright publish" # try to publish that package under this tag's name. playwright/README.md, # "Releasing", says the same. # -# On a pull request the same package is published as a dry run, with no -# token, so a package that would ship without its build shows before a tag. +# The package is built and packed once, and that tarball is checked and then +# published, so what ships is what was checked. The license check +# (scripts/license-gate.mjs, as in sdk-publish.yml) installs the tarball as a +# user's `npm install` does and refuses it when anything it installs names a +# license outside its allow-list. test/workflows.test.mjs holds this +# workflow to that, and both of its jobs run those rows first. +# +# On a pull request the same tarball is packed, checked and published as a +# dry run, with no token, so a package that would ship without its build, or +# with a license it may not carry, shows before a tag. on: push: tags: ["playwright-v*"] pull_request: paths: - "playwright/**" + - "scripts/license-gate.mjs" - ".github/workflows/playwright-publish.yml" - ".github/workflows/playwright-sdk.yml" @@ -39,6 +50,7 @@ jobs: if: github.event_name == 'push' needs: rows runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read id-token: write # npm --provenance signs the package with this run's identity @@ -47,14 +59,23 @@ jobs: shell: bash # -eo pipefail: a failed command before a pipe fails the step working-directory: playwright steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 # Writes the token line for registry.npmjs.org and pins the @keploy # scope to it (@keploy:registry), in the runner's own npmrc. registry-url: https://registry.npmjs.org scope: "@keploy" + - name: Workflow rows + # test/workflows.test.mjs, from the repository's root, so that every + # tag and pull request this workflow runs on checks that it still + # publishes only the tarball its Licenses step passed. The root's + # install, with no scripts, is only for the rows' YAML parser. + working-directory: . + run: | + npm ci --ignore-scripts + node --test test/workflows.test.mjs - name: The tag names package.json's version run: node scripts/release-version.mjs "$GITHUB_REF_NAME" - name: The npm token is set @@ -69,6 +90,26 @@ jobs: got=$(npm config get @keploy:registry) test "$got" = https://registry.npmjs.org/ || { echo "::error::@keploy:registry is '$got', not https://registry.npmjs.org/; refusing to publish"; exit 1; } - run: npm ci + - name: Build + run: npm run build + - name: Pack + id: pack + run: | + name=$(npm pack --pack-destination "$RUNNER_TEMP" | tail -n 1) + test -f "$RUNNER_TEMP/$name" || { echo "::error::npm pack wrote no $name"; exit 1; } + echo "tarball=$RUNNER_TEMP/$name" >> "$GITHUB_OUTPUT" + - name: The package carries its build + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: | + tar -tzf "$TARBALL" > "$RUNNER_TEMP/files.txt" + for f in dist/index.js dist/index.d.ts package.json README.md; do + grep -qx "package/$f" "$RUNNER_TEMP/files.txt" || { echo "::error::the package would ship without $f"; exit 1; } + done + - name: Licenses + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: node ../scripts/license-gate.mjs "$TARBALL" - name: The token authenticates # Fails here, named, on a token that is expired, revoked or pasted # with a stray character, before anything is uploaded. @@ -76,30 +117,56 @@ jobs: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} run: npm whoami --registry https://registry.npmjs.org/ - name: Publish - # prepublishOnly builds dist/ first. env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: npm publish --access public --provenance + TARBALL: ${{ steps.pack.outputs.tarball }} + run: npm publish "$TARBALL" --access public --provenance dry-run: name: publish dry run if: github.event_name == 'pull_request' runs-on: ubuntu-latest + timeout-minutes: 30 defaults: run: shell: bash # -eo pipefail: a failed command before a pipe fails the step working-directory: playwright steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 + - name: Workflow rows + # test/workflows.test.mjs, from the repository's root, so that every + # tag and pull request this workflow runs on checks that it still + # publishes only the tarball its Licenses step passed. The root's + # install, with no scripts, is only for the rows' YAML parser. + working-directory: . + run: | + npm ci --ignore-scripts + node --test test/workflows.test.mjs - run: npm ci - - name: Publish, dry run - run: npm publish --dry-run --access public 2>&1 | tee "$RUNNER_TEMP/publish.log" + - name: Build + run: npm run build + - name: Pack + id: pack + run: | + name=$(npm pack --pack-destination "$RUNNER_TEMP" | tail -n 1) + test -f "$RUNNER_TEMP/$name" || { echo "::error::npm pack wrote no $name"; exit 1; } + echo "tarball=$RUNNER_TEMP/$name" >> "$GITHUB_OUTPUT" - name: The package carries its build - # A fresh checkout has no dist/; only prepublishOnly puts it there. + env: + TARBALL: ${{ steps.pack.outputs.tarball }} run: | + tar -tzf "$TARBALL" > "$RUNNER_TEMP/files.txt" for f in dist/index.js dist/index.d.ts package.json README.md; do - grep -qE "npm notice [0-9.]+[kMG]?B +$f\$" "$RUNNER_TEMP/publish.log" || { echo "::error::the package would ship without $f"; exit 1; } + grep -qx "package/$f" "$RUNNER_TEMP/files.txt" || { echo "::error::the package would ship without $f"; exit 1; } done + - name: Licenses + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: node ../scripts/license-gate.mjs "$TARBALL" + - name: Publish, dry run + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: npm publish "$TARBALL" --dry-run --access public diff --git a/.github/workflows/playwright-sdk.yml b/.github/workflows/playwright-sdk.yml index 061954d..5dc9b0a 100644 --- a/.github/workflows/playwright-sdk.yml +++ b/.github/workflows/playwright-sdk.yml @@ -17,6 +17,8 @@ jobs: rows: name: build and rows (Node ${{ matrix.node }}) runs-on: ubuntu-latest + # A leg takes a few minutes, npm installs included: one past 30 is a hang. + timeout-minutes: 30 strategy: # One leg failing must not cancel the others: each Node version's # result is its own. diff --git a/.github/workflows/sdk-publish.yml b/.github/workflows/sdk-publish.yml new file mode 100644 index 0000000..6122e7b --- /dev/null +++ b/.github/workflows/sdk-publish.yml @@ -0,0 +1,149 @@ +name: "@keploy/sdk publish" + +# Publishes @keploy/sdk, the package at the repository's root, to npm from a +# tag. +# +# To release X.Y.Z: set "version": "X.Y.Z" in package.json on main (a pull +# request, so the check below runs on it), then push the tag sdk-vX.Y.Z on +# that commit. The publish refuses a tag that does not name package.json's +# version (scripts/release-version.mjs); npm itself refuses a version it +# already has. Every action this workflow runs is pinned by commit: the +# publish job holds the npm token and signs the package's provenance. +# +# The package is packed once, and that tarball is checked and then published, +# so what ships is what was checked. The license check +# (scripts/license-gate.mjs) installs the tarball as a user's `npm install` +# does, for each platform users run, and refuses it when any installed +# package's license field, or any installed file, names a license outside +# its allow-list. test/workflows.test.mjs holds every publish workflow to +# this: the tarball the Licenses step checked is the one published. +# +# Push sdk-vX.Y.Z as a plain git tag, never as a GitHub release: a published +# GitHub release starts any workflow of this repository that runs on +# releases. +# +# On a pull request the same tarball is packed and checked, with no token. +on: + push: + tags: ["sdk-v*"] + pull_request: + paths: + - "package.json" + - "package-lock.json" + - "tsconfig.json" + - "keployCli.ts" + - "v2/**" + - "scripts/**" + - "test/**" + - ".github/workflows/sdk-publish.yml" + # The rows (test/workflows.test.mjs) hold these two to the license + # check as well: a change to either runs them here too. + - ".github/workflows/jest-publish.yml" + - ".github/workflows/playwright-publish.yml" + +permissions: + contents: read + +jobs: + publish: + name: publish to npm + if: github.event_name == 'push' + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + id-token: write # npm --provenance signs the package with this run's identity + defaults: + run: + shell: bash # -eo pipefail: a failed command before a pipe fails the step + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + # Writes the token line for registry.npmjs.org and pins the @keploy + # scope to it (@keploy:registry), in the runner's own npmrc. + registry-url: https://registry.npmjs.org + scope: "@keploy" + - name: The tag names package.json's version + run: node scripts/release-version.mjs "$GITHUB_REF_NAME" + - name: The npm token is set + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + run: test -n "$NPM_TOKEN" || { echo "::error::the NPM_TOKEN secret is not set; refusing to publish"; exit 1; } + - name: The @keploy scope publishes to registry.npmjs.org + # npm sends a scoped package to its scope's registry, whatever + # --registry says. Read the setting npm will use, with every npmrc in + # it (a project one included), and refuse any other. + run: | + got=$(npm config get @keploy:registry) + test "$got" = https://registry.npmjs.org/ || { echo "::error::@keploy:registry is '$got', not https://registry.npmjs.org/; refusing to publish"; exit 1; } + - run: npm ci + - name: Rows + run: node --test test/*.test.mjs + - name: Pack + id: pack + # prepare builds dist/ first. + run: | + name=$(npm pack --pack-destination "$RUNNER_TEMP" | tail -n 1) + test -f "$RUNNER_TEMP/$name" || { echo "::error::npm pack wrote no $name"; exit 1; } + echo "tarball=$RUNNER_TEMP/$name" >> "$GITHUB_OUTPUT" + - name: The package carries its build + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: | + tar -tzf "$TARBALL" > "$RUNNER_TEMP/files.txt" + for f in dist/keployCli.js dist/keployCli.d.ts dist/v2/dedup/register.js dist/v2/dedup/middleware.js package.json README.MD LICENSE; do + grep -qx "package/$f" "$RUNNER_TEMP/files.txt" || { echo "::error::the package would ship without $f"; exit 1; } + done + - name: Licenses + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: node scripts/license-gate.mjs "$TARBALL" + - name: The token authenticates + # Fails here, named, on a token that is expired, revoked or pasted + # with a stray character, before anything is uploaded. + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm whoami --registry https://registry.npmjs.org/ + - name: Publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + TARBALL: ${{ steps.pack.outputs.tarball }} + run: npm publish "$TARBALL" --access public --provenance + + check: + name: pack and license check + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 30 + defaults: + run: + shell: bash # -eo pipefail: a failed command before a pipe fails the step + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + - run: npm ci + - name: Rows + run: node --test test/*.test.mjs + - name: Pack + id: pack + # prepare builds dist/ first. + run: | + name=$(npm pack --pack-destination "$RUNNER_TEMP" | tail -n 1) + test -f "$RUNNER_TEMP/$name" || { echo "::error::npm pack wrote no $name"; exit 1; } + echo "tarball=$RUNNER_TEMP/$name" >> "$GITHUB_OUTPUT" + - name: The package carries its build + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: | + tar -tzf "$TARBALL" > "$RUNNER_TEMP/files.txt" + for f in dist/keployCli.js dist/keployCli.d.ts dist/v2/dedup/register.js dist/v2/dedup/middleware.js package.json README.MD LICENSE; do + grep -qx "package/$f" "$RUNNER_TEMP/files.txt" || { echo "::error::the package would ship without $f"; exit 1; } + done + - name: Licenses + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: node scripts/license-gate.mjs "$TARBALL" diff --git a/package-lock.json b/package-lock.json index 411c3ef..cbfbb2f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "@keploy/sdk", - "version": "2.0.2", + "version": "2.0.11", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -123,6 +123,63 @@ "proxy-from-env": "^1.1.0" } }, + "body-parser": { + "version": "1.20.9", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.9.tgz", + "integrity": "sha512-FISIQxV5u91Y9gvuFq3q/tA/qTqLJn3AhRcEvyozyhJTh5OBFFtv827AdHU7Rbyct4r+Ba1/99oFa1Y3yuFsdw==", + "requires": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.16.0", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + } + } + }, + "bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" + }, + "call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "requires": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + } + }, + "call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "requires": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + } + }, "combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -131,6 +188,11 @@ "delayed-stream": "~1.0.0" } }, + "content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==" + }, "cors": { "version": "2.8.5", "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", @@ -153,6 +215,49 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, + "depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" + }, + "destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" + }, + "dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "requires": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + } + }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" + }, + "es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==" + }, + "es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==" + }, + "es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "requires": { + "es-errors": "^1.3.0" + } + }, "follow-redirects": { "version": "1.15.6", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", @@ -173,6 +278,42 @@ "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" }, + "get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "requires": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + } + }, + "get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "requires": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + } + }, + "gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==" + }, + "has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==" + }, "hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -181,6 +322,31 @@ "function-bind": "^1.1.2" } }, + "http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "requires": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, "is-core-module": { "version": "2.13.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", @@ -194,6 +360,16 @@ "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==" + }, + "media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==" + }, "mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -222,6 +398,19 @@ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" }, + "object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==" + }, + "on-finished": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.2.tgz", + "integrity": "sha512-ATWUv387i+nLD49EiDoOft5scf32Vsdz3ktrQJG9bi7jRFmaXTssvyvj1EmeKVVgvesbqUtvQQvaS2TpTukAZQ==", + "requires": { + "ee-first": "1.1.1" + } + }, "path-parse": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", @@ -232,6 +421,26 @@ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, + "qs": { + "version": "6.16.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.2.tgz", + "integrity": "sha512-VYdnB5S7/0IeXzBacUtwNAaM6d7Y/PgYnNC7bJCI6meqA/OgnsQbZMtVW8Dt2D1PHgbHGtYmhSoEIgvt7xZ2ZA==", + "requires": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + } + }, + "raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "requires": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + } + }, "require-in-the-middle": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-7.3.0.tgz", @@ -252,30 +461,115 @@ "supports-preserve-symlinks-flag": "^1.0.0" } }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "requires": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + } + }, + "side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "requires": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + } + }, + "side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "requires": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + } + }, + "side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "requires": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + } + }, + "statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==" + }, "supports-preserve-symlinks-flag": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==" }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" + }, "tree-kill": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==" }, + "type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + } + }, "typescript": { "version": "5.3.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", - "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==" + "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", + "dev": true }, "undici-types": { "version": "5.26.5", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==" + }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==" + }, + "yaml": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.1.tgz", + "integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==", + "dev": true } } } diff --git a/package.json b/package.json index 8fccf7a..ec0280c 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,14 @@ { "name": "@keploy/sdk", - "version": "2.0.2", + "version": "2.0.11", "description": "This is the client SDK for the [Keploy](https://github.com/keploy/keploy) testing platform. With the TypeScript SDK, you can test both your existing unit test cases in Jest and create new end-to-end test cases for your applications.", "main": "dist/keployCli.js", "types": "dist/keployCli.d.ts", + "files": [ + "dist", + "keployCli.ts", + "v2" + ], "scripts": { "build": "tsc", "prepare": "npm run build", @@ -14,7 +19,7 @@ "url": "git+https://github.com/keploy/typescript-sdk.git" }, "author": "Keploy ", - "license": "MIT", + "license": "Apache-2.0", "bugs": { "url": "https://github.com/keploy/keploy/issues" }, @@ -23,14 +28,16 @@ "@types/express": "^4.17.21", "@types/node": "^20.11.16", "axios": "^1.6.7", + "body-parser": "^1.20.9", "cors": "^2.8.5", "lodash": "^4.17.21", "require-in-the-middle": "^7.3.0", - "tree-kill": "^1.2.2", - "typescript": "^5.3.3" + "tree-kill": "^1.2.2" }, "devDependencies": { "@types/axios": "^0.14.0", - "@types/cors": "^2.8.17" + "@types/cors": "^2.8.17", + "typescript": "^5.3.3", + "yaml": "^2.9.1" } } diff --git a/scripts/license-gate.mjs b/scripts/license-gate.mjs new file mode 100644 index 0000000..5561540 --- /dev/null +++ b/scripts/license-gate.mjs @@ -0,0 +1,459 @@ +// license-gate.mjs TARBALL: the publish workflows' license check. +// +// TARBALL is the file `npm pack` wrote, the one the workflow then publishes. +// The check installs it into an empty project the way a user's `npm install` +// does, with install scripts off (no devDependencies; optional and peer +// dependencies in, whatever an npmrc or an npm_config_ variable says), once +// for each platform in PLATFORMS (npm's --os, --cpu and --libc), so that a dependency +// npm installs only on one of those platforms is checked too. It refuses the package +// when anything those installs put on disk for it names a license outside the +// allow-list below: +// - each installed package's package.json license field (or the old +// `licenses` list) must name allowed SPDX ids only. A missing field, an +// id not on the list, "SEE LICENSE IN ..." and UNLICENSED all fail. +// - every file of every installed package is read in full, whatever its +// name, extension or size, a block at a time, for the names, ids and +// URLs of the licenses in FLAGGED, for license pages (opensource.org, +// choosealicense.com, spdx.org) that name no allowed license, and for +// SPDX-License-Identifier lines that name an id not allowed. Text is +// matched normalised (see normalise): in lower case, with any run of +// spaces, hyphens, underscores and dots as one separator, so "CC BY 4.0", +// "CC-BY-4.0" and "cc_by_4.0" are one spelling. A gzip file is read +// decompressed; a file that starts with a UTF-16 byte order mark is read +// as UTF-16, any other as UTF-8. +// A package's own peerDependencies are the user's install, not ours: the +// user's own jest or playwright tree is already in their project whatever we +// ship. The gate names them, and does not judge them. Peers of our +// dependencies are still judged, and so is anything the package or one of +// its dependencies needs too. Only the peers in USER_PEERS are left to the +// user: any other peerDependency of the package fails, until it is added +// there. +// Accepting a flagged file with a notice is not enough, so there is no +// exception list: a finding fails the publish. +// +// Limits, until a fuller license scanner replaces this script: +// - a license that FLAGGED does not name passes when only a file's text +// names it (the Artistic License, say); a license field naming it fails. +// - the id Unicode-3.0 is matched only in an SPDX-License-Identifier line +// (code that cites "Unicode 3.0" most often names the standard). +// - a name or id is matched only when what splits it is a separator above: +// one split by markup, a comment marker or an escape, or spelt in other +// than ASCII letters, is not. +// - a license text in a language other than English (the Creative Commons +// ports to German, French, Dutch, Austrian or Japanese law, say) passes +// unless it also names a flagged id, English name or URL. +// - only gzip is decompressed, one level deep: zip, xz, bzip2 and brotli +// data, and UTF-16 text with no byte order mark, are read as their bytes. +// - only the platforms in PLATFORMS are installed for: an optional +// dependency only for another (win32/arm64, linux/arm, ia32, the BSDs, +// AIX, say) is not installed, so not checked. +// +// Exit 0 and print what was checked when the package is clean; exit 1 and +// list every finding when not, or when an install fails. +import { spawnSync } from 'node:child_process'; +import { closeSync, createReadStream, existsSync, mkdtempSync, openSync, readdirSync, readFileSync, readSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, dirname, join, relative, resolve } from 'node:path'; +import { pipeline } from 'node:stream'; +import { fileURLToPath } from 'node:url'; +import { createGunzip } from 'node:zlib'; + +export const ALLOWED = new Set([ + 'MIT', 'BSD-2-Clause', 'BSD-3-Clause', '0BSD', 'Apache-2.0', 'ISC', 'Zlib', + 'BSL-1.0', 'CC0-1.0', 'Unlicense', 'PostgreSQL', 'OFL-1.1', +]); + +// The peerDependencies of a package that are left to the user (see the +// header): those @keploy/jest and @keploy/playwright peer on. +export const USER_PEERS = new Set(['jest', 'jest-environment-node', '@playwright/test']); + +// The platforms the tarball is installed for: the runner's, and those of +// other users, as npm's --os, --cpu and (on Linux) --libc name them: glibc +// and musl (Alpine) Linux both, so that an optional dependency for one C +// library only (sharp's LGPL libvips for musl, say) is checked. +export const PLATFORMS = [ + ['linux', 'x64', 'glibc'], ['linux', 'x64', 'musl'], ['linux', 'arm64', 'glibc'], ['linux', 'arm64', 'musl'], + ['darwin', 'arm64'], ['darwin', 'x64'], ['win32', 'x64'], +]; + +// Text as it is matched: lower case, with each run of spaces, hyphens (any +// dash), underscores and dots made one space. +export function normalise(text) { + return text.toLowerCase().replace(/[\s\-\u2010-\u2015_.]+/g, ' '); +} + +// Each flagged license, written once, normalised: its ids, its names and its +// URLs. A space in a pattern is an optional separator ("cc by" also matches +// "ccby"), and a slash in a URL may be escaped ("\/", as JSON writes it). +// An id is matched as a word, with any version after it ("gplv3", "mpl 2 0"). +const LICENSES = [ + ['GPL, LGPL or AGPL', { + ids: ['(?:l|a)?gpl'], + names: ['(?:gnu )?(?:(?:lesser|library|affero) )?general public licen[cs]e'], + urls: ['gnu org/(?:licenses|copyleft)'], + }], + ['MPL', { ids: ['mpl'], names: ['mozilla public licen[cs]e'], urls: ['mozilla org/(?:[a-z]{2}(?: [a-z]{2})?/)?(?:media/)?mpl'] }], + ['Unicode terms', { + // Unicode-3.0 is matched only as an SPDX-License-Identifier (below): + // "Unicode 3.0" in text names a version of the standard. + ids: ['unicode (?:tou|dfs)'], + names: ['unicode,? inc (?:licen[cs]e agreement|terms of use)', 'unicode terms of use', 'unicode licen[cs]e (?:agreement|v 3)'], + urls: ['unicode org/(?:copyright|licen[cs]e|terms of use)'], + }], + ['SSPL', { ids: ['sspl'], names: ['server side public licen[cs]e'] }], + ['BUSL', { ids: ['busl', 'bsl 1 1'], names: ['business source licen[cs]e'] }], + ['EPL', { ids: ['epl'], names: ['eclipse public licen[cs]e'], urls: ['eclipse org/legal'] }], + ['CDDL', { ids: ['cddl'], names: ['common development and distribution licen[cs]e'] }], + ['EUPL', { ids: ['eupl'], names: ['european union public licen[cs]e'] }], + ['OSL', { ids: ['osl v? \\d'], names: ['open software licen[cs]e'] }], + ['CPL or CPAL', { ids: ['cpl v? \\d', 'cpal'], names: ['common public (?:attribution )?licen[cs]e'] }], + ['Creative Commons', { + ids: ['cc by(?: (?:nc|nd|sa)){0,3}'], + // The 3.0 Unported legal code opens "Creative Commons Legal Code / + // Attribution 3.0 Unported" (BY and its NC, ND and SA variants). + names: ['creative commons (?:attribution|by|non commercial|share alike|no derivatives|noderivs|legal code attribution)', 'attribution (?:non commercial|share alike|no derivatives|noderivs)'], + urls: ['creativecommons org/licenses'], + }], + ['ODbL', { ids: ['odbl'], names: ['open database licen[cs]e'] }], + ['Elastic', { ids: ['elastic 2 0'], names: ['elastic licen[cs]e'] }], + ['Commons Clause', { names: ['commons clause'] }], + ['APSL, NPL, RPL, MS-RL or Sleepycat', { + ids: ['apsl', 'npl v? \\d', 'rpl v? \\d', 'ms rl', 'sleepycat'], + names: ['apple public source licen[cs]e', 'netscape public licen[cs]e', 'reciprocal public licen[cs]e', 'microsoft reciprocal licen[cs]e'], + }], +]; + +const sep = (s) => s.replaceAll(' ', ' ?'); +const VERSION = '(?: ?v)?(?: ?\\d{1,4}){0,3}'; +export const FLAGGED = LICENSES.flatMap(([what, { ids = [], names = [], urls = [] }]) => [ + ...ids.map((p) => [what, new RegExp(`(? [what, new RegExp(sep(p), 'g')]), + ...urls.map((p) => [what, new RegExp(sep(p).replaceAll('/', '\\\\?/'), 'g')]), +]); + +// License pages: what follows the path names the license. The text is matched +// in lower case as it is written, so the page ends where the URL does. +const PAGE_URL = /(?:opensource\.org\\?\/licenses?|choosealicense\.com\\?\/licenses|spdx\.org\\?\/licenses)\\?\/([a-z0-9.+_-]{1,64})/g; +const ALLOWED_PAGES = new Set([...[...ALLOWED].map(normalise), 'mit license', 'bsd license', 'isc license', 'zlib license']); +const pageName = (page) => normalise(page).trim().replace(/ (?:php|html?|txt)$/, ''); + +// An SPDX-License-Identifier line names its ids as a license field does, and +// each must be allowed (ids are matched in any case, as SPDX says). +const SPDX_LINE = /spdx-license-identifier:[ \t]{0,8}([a-z0-9.+:() \t-]{1,80})/g; +const ALLOWED_IDS = new Set([...ALLOWED].map((id) => id.toLowerCase())); +const spdxIds = (expr) => expr.replace(/[()]/g, ' ').split(/[ \t]+/).map((t) => t.replace(/\.+$/, '')).filter((t) => /[a-z0-9]/.test(t) && !/^(?:or|and|with)$/.test(t)); + +// At least the longest text a pattern above matches (a license page URL, at +// up to 95 characters), and the context kept around it, so that text read a +// block at a time is matched as it would be whole. +const LONGEST = 128; +const BEFORE = 8; +const AFTER = 64; +export const KEEP = LONGEST + BEFORE + AFTER; + +// Matches text pushed to it a piece at a time, keeping only the last KEEP +// characters of what came before: memory is bounded by the piece's size. A +// match is taken only where the window holds BEFORE characters before it +// and AFTER after it (or the text's start or end), so that a word cut at a +// window's edge is not read as a shorter one; the next window holds what the +// last one could not take yet. +export class Scanner { + #norm = ''; + #raw = ''; + #normCut = false; + #rawCut = false; + #found = new Map(); + #pages = new Set(); + #ids = new Set(); + + push(text) { + this.#feed(text, false); + } + + end() { + this.#feed('', true); + return this.findings(); + } + + findings() { + return [ + ...[...this.#found].sort(([i], [j]) => i - j).map(([, f]) => f), + ...[...this.#pages].map((u) => `a license that is not allowed (${JSON.stringify(u)})`), + ...[...this.#ids].map((id) => `license ${JSON.stringify(id)} is not allowed (SPDX-License-Identifier)`), + ]; + } + + #feed(text, last) { + const norm = normalise(this.#norm + text); + const raw = this.#raw + text.toLowerCase(); + FLAGGED.forEach(([what, re], i) => { + if (this.#found.has(i)) return; + const m = take(re, norm, this.#normCut, last); + if (m) this.#found.set(i, `${what} (${JSON.stringify(m[0])})`); + }); + for (let m, from = 0; (m = take(PAGE_URL, raw, this.#rawCut, last, from)); from = m.index + m[0].length) { + if (!ALLOWED_PAGES.has(pageName(m[1]))) this.#pages.add(m[0]); + } + for (let m, from = 0; (m = take(SPDX_LINE, raw, this.#rawCut, last, from)); from = m.index + m[0].length) { + for (const id of spdxIds(m[1])) if (!ALLOWED_IDS.has(id)) this.#ids.add(id); + } + this.#normCut ||= norm.length > KEEP; + this.#rawCut ||= raw.length > KEEP; + this.#norm = norm.slice(-KEEP); + this.#raw = raw.slice(-KEEP); + } +} + +// The first match of re in window from from on that the window can judge: +// one that starts BEFORE characters in (when text was cut off before the +// window) and ends AFTER characters before the window's end (unless the text +// ends there), or null. +function take(re, window, cut, last, from = 0) { + re.lastIndex = Math.max(from, cut ? BEFORE : 0); + const m = re.exec(window); + if (!m || (!last && m.index + m[0].length > window.length - AFTER)) return null; + return m; +} + +// The findings for one file's text, as "what (the text that matched)". +export function textFindings(text) { + const s = new Scanner(); + s.push(text); + return s.end(); +} + +// What a file's bytes start with: gzip's magic, or a UTF-16 byte order mark. +function encoding(head) { + if (head[0] === 0xff && head[1] === 0xfe) return 'utf-16le'; + if (head[0] === 0xfe && head[1] === 0xff) return 'utf-16be'; + return 'utf-8'; +} + +// The findings for the file at path, read in blocks of block bytes. +export async function fileFindings(path, block = 1 << 20) { + const head = Buffer.alloc(2); + const fd = openSync(path, 'r'); + let n; + try { + n = readSync(fd, head, 0, 2, 0); + } finally { + closeSync(fd); + } + let bytes = createReadStream(path, { highWaterMark: block }); + if (n === 2 && head[0] === 0x1f && head[1] === 0x8b) bytes = pipeline(bytes, createGunzip({ chunkSize: block }), () => {}); + const s = new Scanner(); + let decoder = null; + let start = Buffer.alloc(0); + try { + for await (const chunk of bytes) { + let piece = chunk; + if (!decoder) { + start = Buffer.concat([start, chunk]); + if (start.length < 2) continue; + decoder = new TextDecoder(encoding(start)); + piece = start; + } + s.push(decoder.decode(piece, { stream: true })); + } + // A file under 2 bytes names no license. + if (decoder) s.push(decoder.decode()); + } catch (err) { + return [`not readable (${err.message})`]; + } + return s.end(); +} + +// The SPDX ids of a license expression: every id must be allowed, whatever +// joins them (OR, AND, WITH), since a scanner flags each id it reads. +export function expressionIds(expr) { + return expr.replace(/[()]/g, ' ').split(/\s+/).filter((t) => t && !/^(?:OR|AND|WITH)$/.test(t)); +} + +// The findings for one installed package's license field. +export function licenseFindings(pkg) { + let values; + if (typeof pkg.license === 'string') values = [pkg.license]; + else if (pkg.license && typeof pkg.license.type === 'string') values = [pkg.license.type]; + else if (Array.isArray(pkg.licenses)) values = pkg.licenses.map((l) => (typeof l === 'string' ? l : l && l.type)); + else return ['no license field']; + const found = []; + for (const v of values) { + if (typeof v !== 'string' || v.trim() === '') { + found.push('an empty license field'); + continue; + } + for (const id of expressionIds(v)) { + if (!ALLOWED.has(id)) found.push(`license ${JSON.stringify(id)} is not allowed (field ${JSON.stringify(v)})`); + } + } + return found; +} + +// A package folder's package.json, or the error reading it. +function readPackage(dir) { + try { + return { pkg: JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')) }; + } catch (err) { + return { error: err.message }; + } +} + +// The names a package needs, from the fields of its package.json named. +function needs(pkg, fields) { + return fields.flatMap((f) => (pkg && pkg[f] && typeof pkg[f] === 'object' ? Object.keys(pkg[f]) : [])); +} + +// The findings for the tree installed in project for the package named root, +// and what was checked. Every package folder is found (node_modules/NAME and +// node_modules/@SCOPE/NAME, at any depth) and every file under node_modules +// is read, whichever package it belongs to, but for npm's own record of the +// tree (node_modules/.package-lock.json: the package.json files it copies are +// read where they are) and the packages only root's peerDependencies in +// USER_PEERS need (see the header), which are returned as user (and those peers as peers). +// A link is not read: npm +// makes links only in .bin folders, to files read where they are, so any +// other link, or anything that is not a file or a folder, is a finding. +export async function check(project, root) { + const findings = []; + const packages = new Map(); + const files = []; + const walk = (dir, modules, scope, owner) => { + for (const e of readdirSync(dir, { withFileTypes: true })) { + const p = join(dir, e.name); + if (e.isDirectory()) { + if (modules && e.name.startsWith('@')) { + walk(p, false, true, owner); + continue; + } + // A folder with no package.json is not a package (a package's test + // fixtures can sit under a node_modules folder of their own); its + // files are read all the same. + let o = owner; + if ((scope || (modules && !e.name.startsWith('.'))) && existsSync(join(p, 'package.json'))) { + packages.set(p, readPackage(p)); + o = p; + } + walk(p, e.name === 'node_modules', false, o); + } else if (e.isFile()) { + if (!(modules && e.name === '.package-lock.json')) files.push([p, owner]); + } else if (!(e.isSymbolicLink() && basename(dir) === '.bin')) { + findings.push(`${relative(project, p)}: not a file or a folder, so not read`); + } + } + }; + const modules = join(project, 'node_modules'); + if (existsSync(modules)) walk(modules, true, false, null); + + // Where Node finds the package name from the package folder at dir. + const find = (dir, name) => { + for (let d = dir; d.startsWith(project); d = dirname(d)) { + const p = join(d, 'node_modules', name); + if (packages.has(p)) return p; + } + return null; + }; + // The package folders reached from starts by the fields, and by peers but + // for those of the folders in noPeers. + const reach = (starts, noPeers) => { + const seen = new Set(); + const todo = [...starts]; + while (todo.length) { + const d = todo.pop(); + if (seen.has(d)) continue; + seen.add(d); + const fields = noPeers.has(d) ? ['dependencies', 'optionalDependencies'] : ['dependencies', 'optionalDependencies', 'peerDependencies']; + for (const name of needs(packages.get(d).pkg, fields)) { + const p = find(d, name); + if (p) todo.push(p); + } + } + return seen; + }; + const top = join(modules, root); + let user = new Set(); + let peers = []; + if (!packages.has(top)) findings.push(`${root}: not installed`); + else { + const ours = reach([top], new Set([top])); + const own = needs(packages.get(top).pkg, ['peerDependencies']); + for (const n of own) if (!USER_PEERS.has(n)) findings.push(`${root}: peerDependency ${JSON.stringify(n)} is not one left to the user (USER_PEERS), so it fails until it is added there`); + peers = own.filter((n) => USER_PEERS.has(n)).map((n) => find(top, n)).filter(Boolean); + user = new Set([...reach(peers, new Set())].filter((d) => !ours.has(d))); + } + + let checked = 0; + for (const [dir, { pkg, error }] of packages) { + if (user.has(dir)) continue; + checked++; + const at = relative(project, dir); + if (error) findings.push(`${at}: no readable package.json (${error})`); + else findings.push(...licenseFindings(pkg).map((f) => `${at} (${pkg.name}@${pkg.version}): ${f}`)); + } + let read = 0; + for (const [p, owner] of files) { + if (user.has(owner)) continue; + read++; + for (const f of await fileFindings(p)) findings.push(`${relative(project, p)}: ${f}`); + } + const name = (d) => { + const { pkg } = packages.get(d); + return pkg ? `${pkg.name}@${pkg.version}` : relative(project, d); + }; + return { findings, packages: checked, files: read, user: [...user].map(name).sort(), peers: peers.filter((d) => user.has(d)).map(name).sort() }; +} + +// Installs tarball into the empty project, as a user's `npm install` does on +// platform ([os, cpu, libc]) (npm never installs a dependency's devDependencies; +// no install script runs; the flags put optional and peer dependencies in +// over any npmrc or npm_config_ variable, the C library too), and returns the package's name: the one +// dependency npm saved to the project's package.json. Throws, saying why, on +// a failure. +export function install(tarball, project, platform = PLATFORMS[0]) { + const [os, cpu, libc] = platform; + writeFileSync(join(project, 'package.json'), JSON.stringify({ name: 'license-gate', version: '0.0.0', private: true })); + const args = ['install', '--ignore-scripts', '--include=optional', '--include=peer', '--legacy-peer-deps=false', '--no-audit', '--no-fund', '--no-package-lock', `--os=${os}`, `--cpu=${cpu}`, ...(libc ? [`--libc=${libc}`] : []), resolve(tarball)]; + const r = spawnSync('npm', args, { cwd: project, encoding: 'utf8' }); + if (r.status !== 0) throw new Error(`npm install ${tarball} for ${platform.join('/')} failed (${r.error ? r.error.message : `exit ${r.status}`}): ${(r.stderr || '').trim()}`); + const names = needs(JSON.parse(readFileSync(join(project, 'package.json'), 'utf8')), ['dependencies']); + if (names.length !== 1) throw new Error(`installing ${tarball} for ${platform.join('/')} saved ${names.length} packages to the project, not 1`); + return names[0]; +} + +export async function main(tarball) { + if (!existsSync(tarball ?? '')) { + console.error(`license gate: no package to check (${tarball || 'no TARBALL given'})`); + return 1; + } + // Each finding once, with the platforms it was found on. + const findings = new Map(); + const user = new Set(); + const peers = new Set(); + const counts = []; + for (const platform of PLATFORMS) { + const project = mkdtempSync(join(tmpdir(), 'license-gate-')); + try { + const root = install(tarball, project, platform); + const r = await check(project, root); + for (const f of r.findings) findings.set(f, [...(findings.get(f) || []), platform.join('/')]); + for (const u of r.user) user.add(u); + for (const p of r.peers) peers.add(p); + counts.push(`${platform.join('/')}: ${r.packages} packages, ${r.files} files`); + } catch (err) { + console.error(`license gate: ${err.message}`); + return 1; + } finally { + rmSync(project, { recursive: true, force: true }); + } + } + const skipped = user.size ? `; not checked, the user's own: ${user.size} packages its peerDependencies install (${[...peers].sort().join(', ')})` : ''; + if (findings.size) { + console.error(`license gate: ${tarball} would ship ${findings.size} license finding(s):`); + for (const [f, on] of findings) console.error(` ${f}${on.length === PLATFORMS.length ? '' : ` (on ${on.join(', ')})`}`); + console.error(`checked ${counts.join('; ')}${skipped}`); + return 1; + } + console.log(`license gate: ${tarball}: ${counts.join('; ')}; every license allowed${skipped}`); + return 0; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) process.exit(await main(process.argv[2])); diff --git a/scripts/release-version.mjs b/scripts/release-version.mjs new file mode 100644 index 0000000..78708be --- /dev/null +++ b/scripts/release-version.mjs @@ -0,0 +1,30 @@ +// release-version.mjs TAG: the publish workflow's version check. +// +// TAG is the pushed git tag. It must name package.json's version exactly +// (sdk-v), so the version on npm is always the one a tagged +// commit carries. The version must be a release (x.y.z): npm 10 would make a +// prerelease the package's `latest`, so this workflow refuses one. +// +// Exit 0 with no output when the tag may publish; exit 1 and say why when not. +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +export function refusal(tag, version) { + if (!/^\d+\.\d+\.\d+$/.test(version)) { + return `package.json's version ${version} is not a release (x.y.z); this workflow publishes releases only`; + } + const want = `sdk-v${version}`; + if (tag !== want) { + return `the tag ${tag || '(none)'} does not name package.json's version ${version}: the tag for it is ${want}`; + } + return ''; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')); + const why = refusal(process.argv[2] ?? '', pkg.version); + if (why) { + console.error(`refusing to publish: ${why}`); + process.exit(1); + } +} diff --git a/test/license-gate-cli.test.mjs b/test/license-gate-cli.test.mjs new file mode 100644 index 0000000..70f44db --- /dev/null +++ b/test/license-gate-cli.test.mjs @@ -0,0 +1,178 @@ +// Rows for scripts/license-gate.mjs run as the publish workflows run it: on a +// tarball `npm pack` wrote. Each row packs small packages of its own (their +// dependencies are tarballs too, named by path, so npm needs no registry), +// then runs the script on one. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const script = fileURLToPath(new URL('../scripts/license-gate.mjs', import.meta.url)); + +const work = mkdtempSync(join(tmpdir(), 'license-gate-cli-')); +test.after(() => rmSync(work, { recursive: true, force: true })); + +// Packs a package named name from files, and returns the tarball's path. +function pack(name, files) { + const dir = join(work, name); + mkdirSync(dir, { recursive: true }); + for (const [path, body] of Object.entries(files)) writeFileSync(join(dir, path), typeof body === 'string' ? body : JSON.stringify(body)); + const r = spawnSync('npm', ['pack', '--pack-destination', work], { cwd: dir, encoding: 'utf8' }); + assert.equal(r.status, 0, r.stderr); + return join(work, r.stdout.trim().split('\n').pop()); +} + +// Runs the script with its own TMPDIR, so a row can see what it leaves there. +function run(args, env = {}, cwd = undefined) { + const tmp = mkdtempSync(join(work, 'tmp-')); + const r = spawnSync(process.execPath, [script, ...args], { cwd, encoding: 'utf8', env: { ...process.env, TMPDIR: tmp, ...env } }); + // npm keeps caches of its own there (node-compile-cache): only the script's + // projects count. + return { ...r, left: readdirSync(tmp).filter((n) => n.startsWith('license-gate-')) }; +} + +const quote = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +const counts = (n) => ['linux/x64/glibc', 'linux/x64/musl', 'linux/arm64/glibc', 'linux/arm64/musl', 'darwin/arm64', 'darwin/x64', 'win32/x64'].map((p) => `${p}: ${n} packages, \\d+ files`).join('; '); + +test('a clean package passes on every platform, and the projects it was installed into are gone', () => { + const tgz = pack('clean', { 'package.json': { name: 'clean', version: '1.0.0', license: 'Apache-2.0' }, 'index.js': 'module.exports = 1;' }); + const r = run([tgz]); + assert.equal(r.status, 0, r.stderr); + assert.match(r.stdout, new RegExp(`^license gate: ${quote(tgz)}: ${counts(1)}; every license allowed\n$`)); + assert.deepEqual(r.left, []); +}); + +test('a tarball named by a relative path is found', () => { + const tgz = pack('relative', { 'package.json': { name: 'relative', version: '1.0.0', license: 'MIT' } }); + const r = run([basename(tgz)], {}, work); + assert.equal(r.status, 0, r.stderr); +}); + +test('a package whose license field is not allowed fails, named', () => { + const tgz = pack('red', { 'package.json': { name: 'red', version: '1.0.0', license: 'GPL-3.0' } }); + const r = run([tgz]); + assert.equal(r.status, 1); + assert.equal(r.stdout, ''); + assert.match(r.stderr, /would ship \d+ license finding\(s\):\n/); + assert.match(r.stderr, /\n {2}node_modules\/red \(red@1\.0\.0\): license "GPL-3\.0" is not allowed \(field "GPL-3\.0"\)\n/); + assert.match(r.stderr, new RegExp(`\nchecked ${counts(1)}\n$`)); + assert.deepEqual(r.left, []); +}); + +test('a package carrying a flagged file fails, with the file named', () => { + const tgz = pack('notice', { + 'package.json': { name: 'notice', version: '1.0.0', license: 'MIT' }, + 'ThirdPartyNoticeText.txt': 'UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE\nSee Terms of Use at http://www.unicode.org/copyright.html', + }); + const r = run([tgz]); + assert.equal(r.status, 1); + assert.match(r.stderr, /\n {2}node_modules\/notice\/ThirdPartyNoticeText\.txt: Unicode terms \("unicode, inc license agreement"\)\n/); + assert.match(r.stderr, /\n {2}node_modules\/notice\/ThirdPartyNoticeText\.txt: Unicode terms \("unicode org\/copyright"\)\n/); +}); + +test('optional dependencies, those of other platforms, and the peers of dependencies are installed and checked', () => { + const dep = (name, extra) => pack(name, { 'package.json': { name, version: '1.0.0', ...extra } }); + const opt = dep('redopt', { license: 'GPL-3.0' }); + const darwin = dep('reddarwin', { license: 'LGPL-3.0', os: ['darwin'], cpu: ['arm64'] }); + const win = dep('redwin', { license: 'MPL-2.0', os: ['win32'] }); + const peer = dep('redpeer', { license: 'EPL-2.0' }); + const mid = dep('mid', { license: 'MIT', peerDependencies: { redpeer: `file:${peer}` } }); + // The user's own peer needs redpeer too: it is checked all the same. + const own = dep('jest', { license: 'SSPL-1.0', dependencies: { redpeer: `file:${peer}` } }); + const tgz = pack('top', { + 'package.json': { + name: 'top', version: '1.0.0', license: 'MIT', + dependencies: { mid: `file:${mid}` }, + optionalDependencies: { redopt: `file:${opt}`, reddarwin: `file:${darwin}`, redwin: `file:${win}` }, + // The package's own peer (one of USER_PEERS) is the user's: named, + // not checked. + peerDependencies: { jest: `file:${own}` }, + }, + }); + const r = run([tgz]); + assert.equal(r.status, 1, r.stderr); + const lines = r.stderr.split('\n').filter((l) => / is not allowed /.test(l)).sort(); + assert.deepEqual(lines, [ + ' node_modules/reddarwin (reddarwin@1.0.0): license "LGPL-3.0" is not allowed (field "LGPL-3.0") (on darwin/arm64)', + ' node_modules/redopt (redopt@1.0.0): license "GPL-3.0" is not allowed (field "GPL-3.0")', + ' node_modules/redpeer (redpeer@1.0.0): license "EPL-2.0" is not allowed (field "EPL-2.0")', + ' node_modules/redwin (redwin@1.0.0): license "MPL-2.0" is not allowed (field "MPL-2.0") (on win32/x64)', + ]); + assert.match(r.stderr, /\nchecked linux\/x64\/glibc: 4 packages, \d+ files; linux\/x64\/musl: 4 packages, \d+ files; linux\/arm64\/glibc: 4 packages, \d+ files; linux\/arm64\/musl: 4 packages, \d+ files; darwin\/arm64: 5 packages, \d+ files; darwin\/x64: 4 packages, \d+ files; win32\/x64: 5 packages, \d+ files; not checked, the user's own: 1 packages its peerDependencies install \(jest@1\.0\.0\)\n$/); + assert.deepEqual(r.left, []); +}); + +test('an optional dependency for one C library only is installed and checked, whatever npm_config_libc says', () => { + const dep = (name, extra) => pack(name, { 'package.json': { name, version: '1.0.0', ...extra } }); + const musl = dep('redmusl', { license: 'LGPL-3.0-or-later', os: ['linux'], libc: ['musl'] }); + const glibc = dep('redglibc', { license: 'GPL-2.0', os: ['linux'], libc: ['glibc'] }); + const tgz = pack('libctop', { 'package.json': { name: 'libctop', version: '1.0.0', license: 'MIT', optionalDependencies: { redmusl: `file:${musl}`, redglibc: `file:${glibc}` } } }); + for (const env of [{}, { npm_config_libc: 'musl' }, { npm_config_libc: 'glibc' }]) { + const r = run([tgz], env); + assert.equal(r.status, 1, r.stderr); + assert.deepEqual(r.stderr.split('\n').filter((l) => / is not allowed /.test(l)).sort(), [ + ' node_modules/redglibc (redglibc@1.0.0): license "GPL-2.0" is not allowed (field "GPL-2.0") (on linux/x64/glibc, linux/arm64/glibc)', + ' node_modules/redmusl (redmusl@1.0.0): license "LGPL-3.0-or-later" is not allowed (field "LGPL-3.0-or-later") (on linux/x64/musl, linux/arm64/musl)', + ], JSON.stringify(env)); + } +}); + +test('an npm config that leaves optional or peer dependencies out changes nothing', () => { + const dep = (name, extra) => pack(name, { 'package.json': { name, version: '1.0.0', ...extra } }); + const opt = dep('envopt', { license: 'GPL-3.0' }); + const peer = dep('envpeer', { license: 'EPL-2.0' }); + const mid = dep('envmid', { license: 'MIT', peerDependencies: { envpeer: `file:${peer}` } }); + const tgz = pack('envtop', { 'package.json': { name: 'envtop', version: '1.0.0', license: 'MIT', dependencies: { envmid: `file:${mid}` }, optionalDependencies: { envopt: `file:${opt}` } } }); + for (const omit of ['optional', 'peer']) { + const r = run([tgz], { npm_config_omit: omit, npm_config_legacy_peer_deps: 'true' }); + assert.equal(r.status, 1, r.stderr); + assert.deepEqual(r.stderr.split('\n').filter((l) => / is not allowed /.test(l)).sort(), [ + ' node_modules/envopt (envopt@1.0.0): license "GPL-3.0" is not allowed (field "GPL-3.0")', + ' node_modules/envpeer (envpeer@1.0.0): license "EPL-2.0" is not allowed (field "EPL-2.0")', + ], `npm_config_omit=${omit}`); + } +}); + +test('a peerDependency of the package other than those left to the user fails, and is checked', () => { + const other = pack('otherpeer', { 'package.json': { name: 'otherpeer', version: '1.0.0', license: 'MPL-2.0' } }); + const tgz = pack('peertop', { 'package.json': { name: 'peertop', version: '1.0.0', license: 'MIT', peerDependencies: { otherpeer: `file:${other}` } } }); + const r = run([tgz]); + assert.equal(r.status, 1, r.stderr); + assert.deepEqual(r.stderr.split('\n').filter((l) => /^ /.test(l)), [ + ' peertop: peerDependency "otherpeer" is not one left to the user (USER_PEERS), so it fails until it is added there', + ' node_modules/otherpeer (otherpeer@1.0.0): license "MPL-2.0" is not allowed (field "MPL-2.0")', + ' node_modules/otherpeer/package.json: MPL ("mpl 2 0")', + ]); +}); + +test('a file that is not a package fails at the install', () => { + const bad = join(work, 'not-a-package.tgz'); + writeFileSync(bad, 'not a tarball'); + const r = run([bad]); + assert.equal(r.status, 1); + assert.match(r.stderr, /^license gate: npm install .*not-a-package\.tgz for linux\/x64\/glibc failed \(exit \d+\): /); + assert.deepEqual(r.left, []); +}); + +test('no tarball, or one that is not there, fails before any install', () => { + const none = run([]); + assert.equal(none.status, 1); + assert.equal(none.stderr, 'license gate: no package to check (no TARBALL given)\n'); + const missing = run([join(work, 'missing.tgz')]); + assert.equal(missing.status, 1); + assert.equal(missing.stderr, `license gate: no package to check (${join(work, 'missing.tgz')})\n`); + assert.deepEqual(missing.left, []); +}); + +test('an install that puts nothing on disk fails: nothing checked is not clean', () => { + // npm_config_dry_run, set in a job's environment, makes `npm install` exit + // 0 having installed and saved nothing. + const tgz = pack('dry', { 'package.json': { name: 'dry', version: '1.0.0', license: 'MIT' } }); + const r = run([tgz], { npm_config_dry_run: 'true' }); + assert.equal(r.status, 1); + assert.equal(r.stderr, `license gate: installing ${tgz} for linux/x64/glibc saved 0 packages to the project, not 1\n`); + assert.deepEqual(r.left, []); +}); diff --git a/test/license-gate.test.mjs b/test/license-gate.test.mjs new file mode 100644 index 0000000..d8b9661 --- /dev/null +++ b/test/license-gate.test.mjs @@ -0,0 +1,377 @@ +// Rows for the publish workflows' license check (scripts/license-gate.mjs): +// its classifier, its reading of files, and its walk of an installed tree. +// The rows that run the script on packed tarballs are in +// license-gate-cli.test.mjs. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { gzipSync } from 'node:zlib'; +import { ALLOWED, FLAGGED, KEEP, PLATFORMS, Scanner, USER_PEERS, check, expressionIds, fileFindings, licenseFindings, normalise, textFindings } from '../scripts/license-gate.mjs'; + +test('every allowed SPDX id passes, alone and in an expression', () => { + const ids = ['MIT', 'BSD-2-Clause', 'BSD-3-Clause', '0BSD', 'Apache-2.0', 'ISC', 'Zlib', 'BSL-1.0', 'CC0-1.0', 'Unlicense', 'PostgreSQL', 'OFL-1.1']; + assert.deepEqual([...ALLOWED].sort(), [...ids].sort()); + for (const id of ids) assert.deepEqual(licenseFindings({ license: id }), [], id); + assert.deepEqual(licenseFindings({ license: '(MIT OR Apache-2.0)' }), []); + assert.deepEqual(licenseFindings({ license: 'MIT AND ISC' }), []); + assert.deepEqual(licenseFindings({ license: { type: 'ISC' } }), []); + assert.deepEqual(licenseFindings({ licenses: [{ type: 'BSD-3-Clause' }, 'MIT'] }), []); +}); + +test('a license field naming anything else fails, each id on its own', () => { + assert.deepEqual(licenseFindings({ license: 'LGPL-2.1' }), ['license "LGPL-2.1" is not allowed (field "LGPL-2.1")']); + assert.deepEqual(licenseFindings({ license: '(MIT OR GPL-3.0)' }), ['license "GPL-3.0" is not allowed (field "(MIT OR GPL-3.0)")']); + assert.deepEqual(licenseFindings({ license: 'MPL-2.0 AND Unicode-TOU' }), [ + 'license "MPL-2.0" is not allowed (field "MPL-2.0 AND Unicode-TOU")', + 'license "Unicode-TOU" is not allowed (field "MPL-2.0 AND Unicode-TOU")', + ]); + // An exception id is not on the list either. + assert.deepEqual(licenseFindings({ license: 'Apache-2.0 WITH LLVM-exception' }), ['license "LLVM-exception" is not allowed (field "Apache-2.0 WITH LLVM-exception")']); + // An id that holds an operator's letters is still an id. + assert.deepEqual(licenseFindings({ license: 'LicenseRef-ORACLE' }), ['license "LicenseRef-ORACLE" is not allowed (field "LicenseRef-ORACLE")']); + assert.deepEqual(licenseFindings({ license: 'UNLICENSED' }), ['license "UNLICENSED" is not allowed (field "UNLICENSED")']); + assert.match(licenseFindings({ license: 'SEE LICENSE IN LICENSE.md' }).join('\n'), /license "SEE" is not allowed/); + assert.deepEqual(licenseFindings({ license: { type: 'EPL-2.0' } }), ['license "EPL-2.0" is not allowed (field "EPL-2.0")']); + assert.deepEqual(licenseFindings({ licenses: [{ type: 'MIT' }, { type: 'CDDL-1.0' }] }), ['license "CDDL-1.0" is not allowed (field "CDDL-1.0")']); + assert.deepEqual(licenseFindings({ licenses: [{ type: 'GPL-2.0' }] }), ['license "GPL-2.0" is not allowed (field "GPL-2.0")']); + assert.deepEqual(licenseFindings({ license: 'WTFPL' }), ['license "WTFPL" is not allowed (field "WTFPL")']); +}); + +test('a missing or empty license field fails', () => { + assert.deepEqual(licenseFindings({ name: 'x' }), ['no license field']); + assert.deepEqual(licenseFindings({ license: { url: 'https://example.com' } }), ['no license field']); + assert.deepEqual(licenseFindings({ license: '' }), ['an empty license field']); + assert.deepEqual(licenseFindings({ license: ' ' }), ['an empty license field']); + assert.deepEqual(licenseFindings({ licenses: [{ url: 'https://example.com' }] }), ['an empty license field']); +}); + +test('the ids of an expression drop only parentheses and operators', () => { + assert.deepEqual(expressionIds('((MIT OR Apache-2.0) AND BSD-3-Clause)'), ['MIT', 'Apache-2.0', 'BSD-3-Clause']); + assert.deepEqual(expressionIds('GPL-2.0 WITH Classpath-exception-2.0'), ['GPL-2.0', 'Classpath-exception-2.0']); +}); + +test('text is matched in lower case, with any run of separators as one space', () => { + assert.equal(normalise('CC BY 4.0'), 'cc by 4 0'); + assert.equal(normalise('CC-BY-4.0'), 'cc by 4 0'); + assert.equal(normalise('cc_by_4.0'), 'cc by 4 0'); + assert.equal(normalise('CC \t- _.\n BY–SA'), 'cc by sa'); + assert.equal(normalise('a/b,c'), 'a/b,c'); +}); + +// Each sample and every finding it gives, listed whole: a pattern taken out +// of the check, or loosened or narrowed, changes some row's list. +const flagged = [ + ['Licensed under LGPL-2.1 terms', ['GPL, LGPL or AGPL ("lgpl 2 1")']], + ['AGPLv3 only', ['GPL, LGPL or AGPL ("agplv3")']], + ['GPL-3.0-or-later here', ['GPL, LGPL or AGPL ("gpl 3 0")']], + ['see https://choosealicense.com/licenses/lgpl-3.0/', ['GPL, LGPL or AGPL ("lgpl 3 0")', 'a license that is not allowed ("choosealicense.com/licenses/lgpl-3.0")']], + ['License: https://opensource.org/license/gpl-3-0', ['GPL, LGPL or AGPL ("gpl 3 0")', 'a license that is not allowed ("opensource.org/license/gpl-3-0")']], + ['the GNU Lesser General Public License applies', ['GPL, LGPL or AGPL ("gnu lesser general public license")']], + ['see the gnu affero general public licence', ['GPL, LGPL or AGPL ("gnu affero general public licence")']], + ['GNU Library General Public License', ['GPL, LGPL or AGPL ("gnu library general public license")']], + ['under the General_Public_License', ['GPL, LGPL or AGPL ("general public license")']], + ['', ['GPL, LGPL or AGPL ("gnu org/licenses")']], + ['https://www.gnu.org/copyleft/', ['GPL, LGPL or AGPL ("gnu org/copyleft")']], + ['"https:\\/\\/www.GNU.org\\/licenses\\/"', ['GPL, LGPL or AGPL ("gnu org\\\\/licenses")']], + ['SPDX-License-Identifier: MPL-2.0', ['MPL ("mpl 2 0")', 'license "mpl-2.0" is not allowed (SPDX-License-Identifier)']], + ['Portions are licensed under MPL 2.0.', ['MPL ("mpl 2 0")']], + ['MPL2.0', ['MPL ("mpl2 0")']], + ['subject to the Mozilla Public License, v. 2.0', ['MPL ("mozilla public license")']], + ['Mozilla Public Licence', ['MPL ("mozilla public licence")']], + ['https://www.mozilla.org/media/MPL/2.0/index.txt', ['MPL ("mpl")', 'MPL ("mozilla org/media/mpl")']], + ['http://mozilla.org/MPL/2.0/', ['MPL ("mpl")', 'MPL ("mozilla org/mpl")']], + ['https://www.mozilla.org/en-US/MPL/', ['MPL ("mpl")', 'MPL ("mozilla org/en us/mpl")']], + ['License: Unicode-TOU', ['Unicode terms ("unicode tou")']], + ['Unicode-DFS-2016', ['Unicode terms ("unicode dfs 2016")']], + ['/* SPDX-License-Identifier: Unicode-3.0 */', ['license "unicode-3.0" is not allowed (SPDX-License-Identifier)']], + ['// spdx-license-identifier:\t(MIT OR Artistic-2.0).', ['license "artistic-2.0" is not allowed (SPDX-License-Identifier)']], + ['# SPDX-License-Identifier: BSD-3-Clause\nSPDX-License-Identifier: WTFPL AND ISC', ['license "wtfpl" is not allowed (SPDX-License-Identifier)']], + ['SPDX-License-Identifier: Foo-1.0', ['license "foo-1.0" is not allowed (SPDX-License-Identifier)']], + // Up to 8 spaces or tabs after the colon are not counted in the + // expression's 80 characters. + [`SPDX-License-Identifier:${' '.repeat(76)}Foo-1.0`, ['license "foo-1.0" is not allowed (SPDX-License-Identifier)']], + ['UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE', ['Unicode terms ("unicode, inc license agreement")']], + ['Unicode, Inc. Terms of Use', ['Unicode terms ("unicode, inc terms of use")']], + ['Data Files and Software: Unicode Terms of Use apply', ['Unicode terms ("unicode terms of use")']], + ['UNICODE LICENSE V3', ['Unicode terms ("unicode license v3")']], + ['the Unicode License Agreement', ['Unicode terms ("unicode license agreement")']], + ['http://www.unicode.org/copyright.html', ['Unicode terms ("unicode org/copyright")']], + ['https://www.unicode.org/license.txt', ['Unicode terms ("unicode org/license")']], + ['http://www.unicode.org/terms_of_use.html', ['Unicode terms ("unicode org/terms of use")']], + ['SSPL-1.0', ['SSPL ("sspl 1 0")']], + ['the Server Side Public License', ['SSPL ("server side public license")']], + ['BUSL-1.1', ['BUSL ("busl 1 1")']], + ['MariaDB BSL 1.1', ['BUSL ("bsl 1 1")']], + ['Business Source License 1.1', ['BUSL ("business source license")']], + ['EPL-2.0', ['EPL ("epl 2 0")']], + ['Eclipse Public License', ['EPL ("eclipse public license")']], + ['https://www.eclipse.org/legal/epl-v20.html', ['EPL ("epl v20")', 'EPL ("eclipse org/legal")']], + ['CDDL-1.0', ['CDDL ("cddl 1 0")']], + ['Common Development and Distribution License', ['CDDL ("common development and distribution license")']], + ['EUPL-1.2', ['EUPL ("eupl 1 2")']], + ['European Union Public Licence', ['EUPL ("european union public licence")']], + ['OSL-3.0', ['OSL ("osl 3 0")']], + ['Open Software License', ['OSL ("open software license")']], + ['CPL-1.0', ['CPL or CPAL ("cpl 1 0")']], + ['CPAL-1.0', ['CPL or CPAL ("cpal 1 0")']], + ['Common Public Attribution License', ['CPL or CPAL ("common public attribution license")']], + ['CC-BY-4.0', ['Creative Commons ("cc by 4 0")']], + ['This data is licensed CC BY 4.0.', ['Creative Commons ("cc by 4 0")']], + ['CC BY-SA 4.0', ['Creative Commons ("cc by sa 4 0")']], + ['cc_by_nc_nd_4.0', ['Creative Commons ("cc by nc nd 4 0")']], + ['Creative Commons Attribution 4.0 International', ['Creative Commons ("creative commons attribution")']], + ['Creative Commons BY-SA', ['Creative Commons ("creative commons by")']], + ['Attribution-NonCommercial-ShareAlike', ['Creative Commons ("attribution noncommercial")']], + ['Attribution-ShareAlike 4.0 International', ['Creative Commons ("attribution sharealike")']], + ['https://creativecommons.org/licenses/by/4.0/', ['Creative Commons ("creativecommons org/licenses")']], + // The first lines of the 3.0 Unported legal code, as SPDX has it: they say + // neither "Creative Commons Attribution" nor "CC BY". + ['Creative Commons Legal Code\n\nAttribution 3.0 Unported\n\n CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE\n LEGAL SERVICES.', ['Creative Commons ("creative commons legal code attribution")']], + ['Creative Commons Legal Code\n\nAttribution-ShareAlike 3.0 Unported', ['Creative Commons ("creative commons legal code attribution")', 'Creative Commons ("attribution sharealike")']], + ['ODbL-1.0', ['ODbL ("odbl 1 0")']], + ['Open Database License', ['ODbL ("open database license")']], + ['Elastic-2.0', ['Elastic ("elastic 2 0")']], + ['the elastic license', ['Elastic ("elastic license")']], + ['"Commons Clause" License Condition v1.0', ['Commons Clause ("commons clause")']], + ['APSL-2.0', ['APSL, NPL, RPL, MS-RL or Sleepycat ("apsl 2 0")']], + ['NPL-1.1', ['APSL, NPL, RPL, MS-RL or Sleepycat ("npl 1 1")']], + ['RPL-1.5', ['APSL, NPL, RPL, MS-RL or Sleepycat ("rpl 1 5")']], + ['MS-RL', ['APSL, NPL, RPL, MS-RL or Sleepycat ("ms rl")']], + ['Sleepycat', ['APSL, NPL, RPL, MS-RL or Sleepycat ("sleepycat")']], + ['Apple Public Source License', ['APSL, NPL, RPL, MS-RL or Sleepycat ("apple public source license")']], + ['Netscape Public License', ['APSL, NPL, RPL, MS-RL or Sleepycat ("netscape public license")']], + ['Reciprocal Public License', ['APSL, NPL, RPL, MS-RL or Sleepycat ("reciprocal public license")']], + ['Microsoft Reciprocal License', ['APSL, NPL, RPL, MS-RL or Sleepycat ("microsoft reciprocal license")']], + ['https://opensource.org/licenses/Artistic-2.0', ['a license that is not allowed ("opensource.org/licenses/artistic-2.0")']], + ['https://spdx.org/licenses/Artistic-2.0.html', ['a license that is not allowed ("spdx.org/licenses/artistic-2.0.html")']], + ['see https://opensource.org/licenses/MIT and https://opensource.org/license/artistic-2-0', ['a license that is not allowed ("opensource.org/license/artistic-2-0")']], + ['"https:\\/\\/opensource.org\\/licenses\\/Artistic-2.0"', ['a license that is not allowed ("opensource.org\\\\/licenses\\\\/artistic-2.0")']], +]; + +test('a file naming a flagged license is found, by name, id or URL, in any spelling', () => { + for (const [text, want] of flagged) assert.deepEqual(textFindings(text), want, text); + // Every pattern of the check is some sample's. + const labels = new Set(flagged.flatMap(([, w]) => w.map((f) => f.replace(/ \(.*$/, '')))); + for (const [what] of FLAGGED) assert.ok(labels.has(what), what); +}); + +const clean = [ + 'MIT License\n\nPermission is hereby granted, free of charge, to any person obtaining a copy', + 'Licensed under the Apache License, Version 2.0 (the "License"); http://www.apache.org/licenses/LICENSE-2.0', + 'https://developer.mozilla.org/en-US/docs/Web/JavaScript and https://www.gnu.org/software/libc/', + 'https://creativecommons.org/publicdomain/zero/1.0/ and http://www.unicode.org/reports/tr44/ and CC0-1.0', + 'const GPLAN = 1; // GPLx is not an id, nor is cplus or nplx', + 'a sample template example: implement the epilogue; Simpl.js, templ and hepl', + 'BSL-1.0 is the Boost license; see https://opensource.org/licenses', + '// SPDX-License-Identifier: (MIT OR Apache-2.0) */ and and SPDX-License-Identifier: 0BSD.', + '// Check for surrogate pairs (reference Unicode 3.0 section "3.7 Surrogates").', + 'elastic band; Unicode 15 tables; the commons; creative work; attribution required', + // CC0's legal code opens as the attribution licenses' do. + 'Creative Commons Legal Code\n\nCC0 1.0 Universal\n\n CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE', +]; + +test('a file naming allowed licenses only is clean', () => { + for (const t of clean) assert.deepEqual(textFindings(t), [], t); + const pages = ['MIT', 'mit-license.php', 'BSD-2-Clause', 'BSD-3-Clause', 'bsd-license.php', '0BSD', 'Apache-2.0', 'apache-2-0', 'ISC', 'isc-license.txt', 'isc-license-txt', 'Zlib', 'zlib-license.php', 'BSL-1.0', 'bsl-1-0', 'CC0-1.0', 'Unlicense', 'PostgreSQL', 'OFL-1.1', 'MIT.html', 'mit-license.html.', 'MIT.htm']; + for (const host of ['https://opensource.org/licenses/', 'https://opensource.org/license/', 'https://choosealicense.com/licenses/', 'https://spdx.org/licenses/', 'HTTPS://OPENSOURCE.ORG/LICENSES/']) { + for (const p of pages) assert.deepEqual(textFindings(`see ${host}${p} for the terms`), [], host + p); + } +}); + +// The findings for text pushed to a Scanner in pieces of size n. +function inPieces(text, n) { + const s = new Scanner(); + for (let i = 0; i < text.length; i += n) s.push(text.slice(i, i + n)); + return s.end(); +} + +test('text read a piece at a time is matched as it would be whole', () => { + for (const [text] of [...flagged, ...clean.map((t) => [t])]) { + for (const n of [1, 2, 7]) assert.deepEqual(inPieces(`${'x '.repeat(KEEP)}${text}`, n), textFindings(text), `${n}: ${text}`); + } + // A match far into a long text, cut across pieces, is found. + const long = `${'x'.repeat(5000)} Mozilla Public License ${'y'.repeat(5000)}`; + assert.deepEqual(inPieces(long, 100), ['MPL ("mozilla public license")']); + assert.deepEqual(inPieces(long, 5011), ['MPL ("mozilla public license")']); + // A match at the very start, before a long run of separators, is found. + assert.deepEqual(inPieces(`GPL${' '.repeat(5000)}x`, 1000), ['GPL, LGPL or AGPL ("gpl")']); + // A word cut at the end of a piece is not read as a shorter one. + const s = new Scanner(); + s.push(`${'x'.repeat(5000)} gpl`); + s.push('an and https://opensource.org/licenses/mi'); + s.push('t-license.php'); + assert.deepEqual(s.end(), []); + // A long license URL cut across pieces is found whole. + const page = `choosealicense.com/licenses/${'a'.repeat(64)}`; + for (const after of [0, 50, 63]) { + const u = new Scanner(); + u.push(`${'x'.repeat(5000)} https://${page}${' '.repeat(after)}`); + u.push(' and more'); + assert.deepEqual(u.end(), [`a license that is not allowed (${JSON.stringify(page)})`], `${after}`); + } + // Nor is a word whose start was cut off before the window. + const t = new Scanner(); + t.push(`${'b'.repeat(1000)}agpl ${'c'.repeat(KEEP - 4)}`); + t.push('d'); + assert.deepEqual(t.end(), []); +}); + +const work = mkdtempSync(join(tmpdir(), 'license-gate-row-')); +test.after(() => rmSync(work, { recursive: true, force: true })); +let nfile = 0; +const file = (body) => { + const p = join(work, `f${nfile++}`); + writeFileSync(p, body); + return p; +}; + +test('every file is read whole, whatever its name or size, a block at a time', async () => { + const big = `${'x'.repeat(3 << 20)}\n/* Mozilla Public License 2.0 */\n`; + assert.deepEqual(await fileFindings(file(big)), ['MPL ("mozilla public license")']); + assert.deepEqual(await fileFindings(file(big), 4096), ['MPL ("mozilla public license")']); + // A UTF-8 character split across blocks is read whole. + assert.deepEqual(await fileFindings(file('CC BY 4.0'), 3), ['Creative Commons ("cc by 4 0")']); + assert.deepEqual(await fileFindings(file('')), []); + assert.deepEqual(await fileFindings(file('G')), []); +}); + +test('a gzip file is read decompressed, and UTF-16 text by its byte order mark', async () => { + const gpl = 'GNU GENERAL PUBLIC LICENSE\nVersion 3, 29 June 2007\n'; + const want = ['GPL, LGPL or AGPL ("gnu general public license")']; + assert.deepEqual(await fileFindings(file(gzipSync(Buffer.from(gpl)))), want); + assert.deepEqual(await fileFindings(file(gzipSync(Buffer.from(gpl.repeat(1 << 15)))), 512), want); + const le = Buffer.from(`${gpl}`, 'utf16le'); + assert.deepEqual(await fileFindings(file(le)), want); + assert.deepEqual(await fileFindings(file(le), 1), want); + assert.deepEqual(await fileFindings(file(Buffer.from(le).swap16())), want); + assert.deepEqual(await fileFindings(file(gzipSync(le))), want); + const bad = await fileFindings(file(Buffer.from([0x1f, 0x8b, 1, 2, 3, 4]))); + assert.equal(bad.length, 1); + assert.match(bad[0], /^not readable \(.+\)$/); +}); + +// A tree as npm installs it, written under a new directory. +function tree(files) { + const root = mkdtempSync(join(work, 'tree-')); + for (const [path, body] of Object.entries(files)) { + mkdirSync(dirname(join(root, path)), { recursive: true }); + writeFileSync(join(root, path), typeof body === 'string' ? body : JSON.stringify(body)); + } + return root; +} + +// What JSON.parse says of text that is not JSON, in this Node. +function parseError(text) { + try { + JSON.parse(text); + } catch (err) { + return err.message; + } + return ''; +} + +test('the walk reads every package folder and file but the user\'s own peers, scoped and nested', async () => { + const root = tree({ + 'package.json': { name: 'license-gate', license: 'GPL-3.0' }, // the project itself is not shipped + 'node_modules/.package-lock.json': { packages: { 'node_modules/q': { license: 'MPL-2.0' } } }, // npm's record, not read + 'node_modules/.cache/x/data.json': '{}', // a dot folder is read, and is not a package + 'node_modules/.cache/package.json': { name: 'cache' }, + 'node_modules/stray.txt': 'Mozilla Public License', // a file outside any package is read too + 'node_modules/top/package.json': { name: 'top', version: '1.0.0', license: 'MIT', dependencies: { a: '1', '@s/b': '2' }, optionalDependencies: { d: '4', ghost: '9' }, peerDependencies: { jest: '1', a: '1' } }, + 'node_modules/a/package.json': { name: 'a', version: '1.0.0', license: 'MIT' }, + // Every file is read, whatever its extension or size. + 'node_modules/a/index.js': 'module.exports = "ODbL-1.0";', + 'node_modules/a/lib/x.cjs': '// SPDX-License-Identifier: GPL-2.0-only', + 'node_modules/a/lib/x.mjs': 'export const l = "EUPL-1.2";', + 'node_modules/a/lib/x.d.ts': '// Eclipse Public License', + 'node_modules/a/lib/x.js.map': '{"sources":["CDDL-1.0"]}', + 'node_modules/a/README.md': 'Licensed under the Elastic License', + 'node_modules/a/dist/big.min.js': `${'x'.repeat(300 << 10)} Sleepycat`, + 'node_modules/a/.package-lock.json': 'Mozilla Public License', // a package's own file of that name is read + 'node_modules/a/@x/y/z.js': '// a folder named @x inside a package is not a scope', + 'node_modules/a/@x/y/package.json': { name: 'y' }, + 'node_modules/@s/stray': 'SSPL', + 'node_modules/@s/b/package.json': { name: '@s/b', version: '2.0.0', license: 'Apache-2.0', dependencies: { c: '3', '@t/e2': '5' } }, + 'node_modules/@s/b/lib/x.js': '// fine', + // A package.json inside a package (as module-type markers are) is a file + // of it, not a package. + 'node_modules/@s/b/lib/package.json': { type: 'commonjs' }, + 'node_modules/@s/b/node_modules/c/package.json': { name: 'c', version: '3.0.0', license: 'LGPL-3.0', peerDependencies: { r: '1' } }, + 'node_modules/@s/b/node_modules/c/NOTICE': 'Mozilla Public License', + 'node_modules/@s/b/node_modules/@t/e2/package.json': { name: '@t/e2', version: '5.0.0', license: 'BUSL-1.1' }, + 'node_modules/d/package.json': { name: 'd', version: '4.0.0', license: 'ISC' }, + 'node_modules/d/ThirdPartyNoticeText.txt': 'UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE', + 'node_modules/e/index.js': 'no package.json here, so not a package', + 'node_modules/a/test/fixture/node_modules/util/index.js': '// a fixture folder, not a package', + 'node_modules/f/package.json': 'not json{', + // The peer of a dependency is the package's to ship: checked, though the + // user's peer jest needs it too. + 'node_modules/r/package.json': { name: 'r', version: '1.0.0', license: 'EPL-2.0' }, + // top's own peer jest (one of USER_PEERS), and what only it needs, are + // the user's: not checked. What top needs too (a) is checked all the + // same, and a, a peer of top's not in USER_PEERS, is a finding. + 'node_modules/jest/package.json': { name: 'jest', version: '1.0.0', license: 'GPL-3.0', dependencies: { q: '1', a: '1', r: '1' } }, + 'node_modules/jest/COPYING': 'GNU General Public License', + 'node_modules/jest/node_modules/q/package.json': { name: 'q', version: '1.0.0', license: 'MPL-2.0' }, + }); + // npm's bin links are not read (the files they name are, where they are); + // any other link is a finding. + mkdirSync(join(root, 'node_modules/.bin')); + symlinkSync('../a/index.js', join(root, 'node_modules/.bin/tool')); + symlinkSync('../a/index.js', join(root, 'node_modules/d/linked')); + const { findings, packages, files, user, peers } = await check(root, 'top'); + assert.deepEqual(user, ['jest@1.0.0', 'q@1.0.0']); + assert.deepEqual(peers, ['jest@1.0.0']); + assert.equal(packages, 8); + assert.equal(files, 28); + assert.deepEqual(findings.sort(), [ + 'node_modules/stray.txt: MPL ("mozilla public license")', + 'top: peerDependency "a" is not one left to the user (USER_PEERS), so it fails until it is added there', + 'node_modules/a/.package-lock.json: MPL ("mozilla public license")', + 'node_modules/a/index.js: ODbL ("odbl 1 0")', + 'node_modules/a/lib/x.cjs: GPL, LGPL or AGPL ("gpl 2 0")', + 'node_modules/a/lib/x.cjs: license "gpl-2.0-only" is not allowed (SPDX-License-Identifier)', + 'node_modules/a/lib/x.mjs: EUPL ("eupl 1 2")', + 'node_modules/a/lib/x.d.ts: EPL ("eclipse public license")', + 'node_modules/a/lib/x.js.map: CDDL ("cddl 1 0")', + 'node_modules/a/README.md: Elastic ("elastic license")', + 'node_modules/a/dist/big.min.js: APSL, NPL, RPL, MS-RL or Sleepycat ("sleepycat")', + 'node_modules/@s/stray: SSPL ("sspl")', + 'node_modules/@s/b/node_modules/c (c@3.0.0): license "LGPL-3.0" is not allowed (field "LGPL-3.0")', + 'node_modules/@s/b/node_modules/c/NOTICE: MPL ("mozilla public license")', + 'node_modules/@s/b/node_modules/c/package.json: GPL, LGPL or AGPL ("lgpl 3 0")', + 'node_modules/@s/b/node_modules/@t/e2 (@t/e2@5.0.0): license "BUSL-1.1" is not allowed (field "BUSL-1.1")', + 'node_modules/@s/b/node_modules/@t/e2/package.json: BUSL ("busl 1 1")', + 'node_modules/d/ThirdPartyNoticeText.txt: Unicode terms ("unicode, inc license agreement")', + 'node_modules/d/linked: not a file or a folder, so not read', + `node_modules/f: no readable package.json (${parseError('not json{')})`, + 'node_modules/r (r@1.0.0): license "EPL-2.0" is not allowed (field "EPL-2.0")', + 'node_modules/r/package.json: EPL ("epl 2 0")', + ].sort()); +}); + +test('the peers left to the user are those @keploy/jest and @keploy/playwright peer on, and no others', () => { + const peers = (dir) => Object.keys(JSON.parse(readFileSync(new URL(`../${dir}/package.json`, import.meta.url), 'utf8')).peerDependencies); + assert.deepEqual([...USER_PEERS].sort(), [...peers('jest'), ...peers('playwright')].sort()); + assert.deepEqual([...USER_PEERS].sort(), ['@playwright/test', 'jest', 'jest-environment-node']); +}); + +test('a clean tree has no findings; a package that is not there is one', async () => { + const root = tree({ + // A peer left to the user that is not installed is no finding. + 'node_modules/a/package.json': { name: 'a', version: '1.0.0', license: 'MIT', peerDependencies: { jest: '1' } }, + 'node_modules/a/LICENSE': 'MIT License\n\nPermission is hereby granted', + }); + const empty = tree({ 'package.json': { name: 'license-gate' } }); + assert.deepEqual(await check(root, 'a'), { findings: [], packages: 1, files: 2, user: [], peers: [] }); + assert.deepEqual(await check(empty, 'a'), { findings: ['a: not installed'], packages: 0, files: 0, user: [], peers: [] }); +}); + +test('the tarball is installed for the runner and for other users\' platforms', () => { + assert.deepEqual(PLATFORMS, [ + ['linux', 'x64', 'glibc'], ['linux', 'x64', 'musl'], ['linux', 'arm64', 'glibc'], ['linux', 'arm64', 'musl'], + ['darwin', 'arm64'], ['darwin', 'x64'], ['win32', 'x64'], + ]); +}); diff --git a/test/package.test.mjs b/test/package.test.mjs new file mode 100644 index 0000000..d99502a --- /dev/null +++ b/test/package.test.mjs @@ -0,0 +1,268 @@ +// Rows for @keploy/sdk as a user gets it: what its built code requires, and +// its entry points loaded from a clean install of its packed tarball. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { builtinModules, createRequire } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = fileURLToPath(new URL('..', import.meta.url)); +const pkg = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8')); +const entries = ['@keploy/sdk', '@keploy/sdk/dist/v2/dedup/middleware', '@keploy/sdk/dist/v2/dedup/register']; + +// The .js files under dir, at any depth. +function scripts(dir) { + return readdirSync(dir, { withFileTypes: true }).flatMap((e) => (e.isDirectory() ? scripts(join(dir, e.name)) : e.name.endsWith('.js') ? [join(dir, e.name)] : [])); +} + +test('every package the built code requires is a dependency', () => { + assert.ok(existsSync(join(root, 'dist/keployCli.js')), 'dist/ is built (npm ci runs prepare)'); + const required = new Set(); + for (const f of scripts(join(root, 'dist'))) { + for (const [, spec] of readFileSync(f, 'utf8').matchAll(/\brequire\(["']([^"']+)["']\)/g)) { + if (spec.startsWith('.') || spec.startsWith('node:') || builtinModules.includes(spec)) continue; + required.add(spec.startsWith('@') ? spec.split('/').slice(0, 2).join('/') : spec.split('/')[0]); + } + } + assert.deepEqual([...required].filter((n) => !(n in pkg.dependencies)), []); + // The one the dedup entry points need, which 2.0.10 did not declare; and + // no YAML package: the middleware writes its entries with our own code. + assert.ok(required.has('body-parser')); + for (const n of ['yaml', 'js-yaml']) assert.ok(!required.has(n) && !(n in pkg.dependencies), n); +}); + +// A clean install of the packed package, as a user's app gets it. +const work = mkdtempSync(join(tmpdir(), 'sdk-install-')); +test.after(() => rmSync(work, { recursive: true, force: true })); +const app = join(work, 'app'); +const readers = join(work, 'readers'); +const reader = (v) => createRequire(join(readers, 'package.json'))(`js-yaml-${v}`); +test.before(() => { + const packed = spawnSync('npm', ['pack', '--ignore-scripts', '--pack-destination', work], { cwd: root, encoding: 'utf8' }); + assert.equal(packed.status, 0, packed.stderr); + const tgz = join(work, packed.stdout.trim().split('\n').pop()); + mkdirSync(app); + writeFileSync(join(app, 'package.json'), JSON.stringify({ name: 'app', version: '0.0.0', private: true })); + const inst = spawnSync('npm', ['install', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', tgz], { cwd: app, encoding: 'utf8' }); + assert.equal(inst.status, 0, inst.stderr); + // The YAML readers a user's app may have had: js-yaml 3 (nyc's) and 4. + mkdirSync(readers); + writeFileSync(join(readers, 'package.json'), JSON.stringify({ name: 'readers', version: '0.0.0', private: true })); + const r = spawnSync('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--no-package-lock', 'js-yaml-3@npm:js-yaml@3.15.2', 'js-yaml-4@npm:js-yaml@4.3.2'], { cwd: readers, encoding: 'utf8' }); + assert.equal(r.status, 0, r.stderr); +}); + +test('the entry points load from a clean install of the packed package', () => { + for (const e of entries) { + const r = spawnSync(process.execPath, ['-e', `require(${JSON.stringify(e)})`], { cwd: app, encoding: 'utf8' }); + assert.equal(r.status, 0, `${e}: ${r.stderr}`); + } + for (const n of ['yaml', 'js-yaml', 'esprima']) assert.ok(!existsSync(join(app, 'node_modules', n)), `${n} is installed`); +}); + +// dedupData.yaml as 2.0.10's middleware wrote it, with js-yaml 3.15.2 and +// with js-yaml 4.3.2 (whichever the app had), before a request and after +// one more (test-set-0/test-3, lines 1 and 4 of index.js run). +const BEFORE = (winKey) => `- id: test-set-0/test-1 + executedLinesByFile: + /home/runner/app/src/index.js: + - 1 + - 2 + - 5 + - 12 + /home/runner/app/src/routes/user routes.js: [] +- id: undefined/test-2 + executedLinesByFile: + ${winKey}: + - 3 + /app/src/ünïcode/日本.js: + - 7 + - 8 +`; +const JS_YAML_3 = BEFORE(String.raw`'C:\Users\dev\app\src\index.js'`); +const JS_YAML_4 = BEFORE(String.raw`C:\Users\dev\app\src\index.js`); +const ADDED = `- id: test-set-0/test-3 + executedLinesByFile: + /home/runner/app/src/index.js: + - 1 + - 4 +`; + +// The coverage of a run with lines 1 and 4 of each file in files run. +const COVERAGE = (files) => ` + global.__coverage__ = Object.fromEntries(${JSON.stringify(files)}.map((f) => [f, { statementMap: { 0: { start: { line: 1 } }, 1: { start: { line: 4 } }, 2: { start: { line: 9 } } }, s: { 0: 1, 1: 2, 2: 0 } }])); +`; + +// The middleware's request: test-set-0/test-3, with lines 1 and 4 of each +// file in files run. +const REQUEST = (files) => COVERAGE(files) + ` + const res = new (require('events').EventEmitter)(); + const h = { 'KEPLOY-TEST-ID': 'test-3', 'KEPLOY-TEST-SET-ID': 'test-set-0' }; + require('@keploy/sdk/dist/v2/dedup/middleware')()({ get: (n) => h[n] }, res, () => {}); +`; + +// Runs the installed middleware on one request in a folder holding seed as +// dedupData.yaml, and returns the file it leaves. +function request(seed, files = ['/home/runner/app/src/index.js']) { + const cwd = mkdtempSync(join(app, 'run-')); + writeFileSync(join(cwd, 'dedupData.yaml'), seed); + const r = spawnSync(process.execPath, ['-e', REQUEST(files) + "res.emit('finish');"], { cwd, encoding: 'utf8' }); + assert.equal(r.status, 0, r.stderr); + return readFileSync(join(cwd, 'dedupData.yaml'), 'utf8'); +} + +test('the middleware appends to dedupData.yaml as 2.0.10 wrote it, and js-yaml 3 and 4 read it back the same', () => { + const entry = (id, executedLinesByFile) => ({ id, executedLinesByFile }); + const want = [ + entry('test-set-0/test-1', { '/home/runner/app/src/index.js': [1, 2, 5, 12], '/home/runner/app/src/routes/user routes.js': [] }), + entry('undefined/test-2', { [String.raw`C:\Users\dev\app\src\index.js`]: [3], '/app/src/ünïcode/日本.js': [7, 8] }), + entry('test-set-0/test-3', { '/home/runner/app/src/index.js': [1, 4] }), + ]; + for (const seed of [JS_YAML_3, JS_YAML_4]) { + const file = request(seed); + assert.equal(file, seed + ADDED); + for (const v of [3, 4]) assert.deepStrictEqual(reader(v).load(file), want, `js-yaml ${v}`); + } + assert.equal(request(''), ADDED); + // A name a YAML 1.1 reader takes for a boolean or a date is quoted, as + // js-yaml quoted it. + const lines = '\n - 1\n - 4\n'; + assert.equal(request('', ['no', '2026-11-04']), `- id: test-set-0/test-3\n executedLinesByFile:\n 'no':${lines} '2026-11-04':${lines}`); +}); + +test('the middleware leaves what the file holds as it is, and loads none of it', () => { + const tags = "- id: a/b\n executedLinesByFile: !!js/function 'function () { return 42 }'\n- !!js/regexp /x/\n- !!js/undefined ''\n"; + assert.equal(request(tags), tags + ADDED); +}); + +test('on a file of 1000 entries, the setup only checks that it exists, and a request only appends its own entry', () => { + const cwd = mkdtempSync(join(app, 'big-')); + // 1000 tests, each with 60 lines run in each of 40 files: 26 MiB (27,558,890 bytes). + const body = Array.from({ length: 40 }, (_, f) => ` /home/runner/app/src/file-${f}.js:\n` + Array.from({ length: 60 }, (_, l) => ` - ${l + 1}\n`).join('')).join(''); + const seed = Array.from({ length: 1000 }, (_, t) => `- id: test-set-0/test-${t}\n executedLinesByFile:\n${body}`).join(''); + writeFileSync(join(cwd, 'dedupData.yaml'), seed); + // Every call into fs from the setup on, until the process exits (not those + // one of them makes), logged to calls.jsonl as it is made, with its first + // argument and the size of its second: to fs's functions, and to those of + // fs.promises, which require('fs/promises') is too. They are wrapped before + // the middleware loads, so that one it binds then is wrapped too; only the + // load is not logged. + const r = spawnSync(process.execPath, ['-e', ` + const fs = require('fs'); + const log = fs.appendFileSync; + let depth = 0; + for (const [api, o] of [['', fs], ['promises.', fs.promises]]) { + for (const k of Object.keys(o)) { + if (typeof o[k] !== 'function' || /^[A-Z]/.test(k)) continue; + const f = o[k]; + o[k] = function (...a) { + if (depth++ === 0) log('calls.jsonl', JSON.stringify([api + k, a[0], typeof a[1] === 'string' ? Buffer.byteLength(a[1]) : a[1] && a[1].length]) + '\\n'); + try { return f.apply(this, a); } finally { depth--; } + }; + } + } + depth++; + const middleware = require('@keploy/sdk/dist/v2/dedup/middleware'); + depth--; + ` + COVERAGE(['/home/runner/app/src/index.js']) + ` + const res = new (require('events').EventEmitter)(); + const h = { 'KEPLOY-TEST-ID': 'test-3', 'KEPLOY-TEST-SET-ID': 'test-set-0' }; + middleware()({ get: (n) => h[n] }, res, () => {}); + res.emit('finish'); + `], { cwd, encoding: 'utf8' }); + assert.equal(r.status, 0, r.stderr); + const logged = join(cwd, 'calls.jsonl'); + const calls = existsSync(logged) ? readFileSync(logged, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)) : []; + assert.deepEqual(calls, [['existsSync', 'dedupData.yaml', null], ['appendFileSync', 'dedupData.yaml', Buffer.byteLength(ADDED)]]); + assert.ok(readFileSync(join(cwd, 'dedupData.yaml'), 'utf8') === seed + ADDED, 'the file is the seed, then the new entry'); +}); + +test('requests that finish in the tick that set the middleware up keep their entries', () => { + // No dedupData.yaml yet. The app sets the middleware up and, in that same + // tick, 40 requests finish. The tick sleeps 200 ms first, as start-up work + // would take, so that what the setup started in the thread pool has run: + // 2.0.10's check found the file missing there and, once the tick ended, + // wrote it empty, after the 40 had written their entries. + const cwd = mkdtempSync(join(app, 'setup-')); + const r = spawnSync(process.execPath, ['-e', COVERAGE(['/home/runner/app/src/index.js']) + ` + const handle = require('@keploy/sdk/dist/v2/dedup/middleware')(); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 200); + for (let i = 0; i < 40; i++) { + const res = new (require('events').EventEmitter)(); + const h = { 'KEPLOY-TEST-ID': 'test-' + i, 'KEPLOY-TEST-SET-ID': 'test-set-0' }; + handle({ get: (n) => h[n] }, res, () => {}); + res.emit('finish'); + } + `], { cwd, encoding: 'utf8' }); + assert.equal(r.status, 0, r.stderr); + const file = readFileSync(join(cwd, 'dedupData.yaml'), 'utf8'); + const ids = Array.from({ length: 40 }, (_, i) => `test-set-0/test-${i}`); + for (const v of [3, 4]) assert.deepEqual((reader(v).load(file) || []).map((e) => e.id), ids, `js-yaml ${v}`); + // The first ran lines 1 and 4; the others, nothing new. + assert.equal(file, ids.map((id, i) => `- id: ${id}\n executedLinesByFile:\n /home/runner/app/src/index.js:${i ? ' []\n' : '\n - 1\n - 4\n'}`).join('')); + // With no request, the setup leaves the file there, empty. + const idle = mkdtempSync(join(app, 'setup-')); + const s = spawnSync(process.execPath, ['-e', "require('@keploy/sdk/dist/v2/dedup/middleware')()"], { cwd: idle, encoding: 'utf8' }); + assert.equal(s.status, 0, s.stderr); + assert.ok(existsSync(join(idle, 'dedupData.yaml')), 'the setup creates the file'); + assert.equal(readFileSync(join(idle, 'dedupData.yaml'), 'utf8'), ''); +}); + +test('the setup writes nothing to a dedupData.yaml that exists', () => { + // As 2.0.10's: an app whose dedupData.yaml it may not write (here a folder + // of that name, which no user may write, root included) starts, and a + // request with no test id leaves it alone. + const cwd = mkdtempSync(join(app, 'setup-')); + mkdirSync(join(cwd, 'dedupData.yaml')); + const r = spawnSync(process.execPath, ['-e', ` + const res = new (require('events').EventEmitter)(); + require('@keploy/sdk/dist/v2/dedup/middleware')()({ get: () => undefined }, res, () => {}); + res.emit('finish'); + `], { cwd, encoding: 'utf8' }); + assert.equal(r.status, 0, r.stderr); + assert.match(r.stderr, /No test ID found in the request headers/); +}); + +// Names and ids a coverage tool and a request can give, YAML's own words, +// indicators and escapes among them, and lines. +const NAMES = ['/home/runner/app/src/index.js', String.raw`C:\Users\dev\app\src\index.js`, '/a b/c.js', '/app/ünïcode/日本.js', '/x/😀.js', + 'a#b', 'a #b', 'a: b', 'a:b', 'a:', 'a ', ' a', ' ', ':', '-a', '- a', '?a', ':a', ',a', '[a]', '{a}', '#a', '&a', '*a', '!a', '|a', '=a', '>a', "'a", '"a', '%a', '@a', '`a', + "it's", 'a"b', 'a\\b', 'no', 'No', 'NO', 'yes', 'y', 'n', 'on', 'off', 'OFF', 'true', 'True', 'false', 'null', 'Null', 'NULL', '~', '', '<<', '0o12', '+0o7', '1_000', '+.5', '._1', '._97', '._e0', '.5', '08', + '0x1F', '0b101', '1:20', '_:_', '1:20.5', '2026-11-04', '2026-11-04T10:00:00Z', '.inf', '-.inf', '.NaN', '.nan', '1e5', '123', '-1', '...', '---', + 'a\tb', 'a\nb', 'a\rb', '\x00', '\x7f', '\x85', 'a\xa0b', '\u2028', 'a\u2029b', 'a\ufeffb', '\ufeffa', 'a\ud800b', '\ud83d', 'a\x1fb', 'a~b', 'a\xa1b', 'a\ud7ffb', 'a\ue000b', + 'a\ufffdb', 'a\ufffeb', 'a\x80b', 'a\x01b', 'a\t"\\b', '\x1b\x07\x08\x0b\x0c', + 'x'.repeat(1023) + "'", 'x'.repeat(1024), 'x'.repeat(1025), "'".repeat(600), '/p/' + 'q'.repeat(2000)]; +const LINES = [1, 2, 0, 12, 1.5, NaN, Infinity, -Infinity, -0, 1e-7, 1e21, 2 ** 53, -3]; + +test('each entry is the text js-yaml 4 wrote but where the limits say, and js-yaml 3 and 4 read back its values', () => { + const { entryYaml } = createRequire(join(app, 'package.json'))('@keploy/sdk/dist/v2/dedup/entry'); + const short = (s) => JSON.stringify(s).slice(0, 24); + const cases = [ + ...NAMES.flatMap((s) => [[`key ${short(s)}`, 't/1', { [s]: [1, 4] }], [`empty ${short(s)}`, 't/1', { [s]: [] }], [`id ${short(s)}`, s, { '/a.js': [1] }]]), + ...LINES.map((n) => [`line ${n}`, 't/1', { '/a.js': [n] }]), + ['no files', 't/1', {}], ['not numbers', 't/1', { '/a.js': ['5', null, undefined, true] }], + ['three files', 'undefined/test-2', { '/a.js': [1, 2], 'C:\\x.js': [], '/b.js': [3] }], + ...[76, 77].flatMap((n) => [[`id of ${n}`, 'i'.repeat(n), {}], [`id of ${n} after a space`, ' ' + 'i'.repeat(n), {}]]), + ]; + const differ = []; + for (const [label, id, lines] of cases) { + const text = entryYaml(id, lines); + const theirs = reader(4).dump([{ id, executedLinesByFile: lines }]); + for (const v of [3, 4]) { + let got; + assert.doesNotThrow(() => { got = reader(v).load(text); }, `${label}: js-yaml ${v} reads it`); + assert.deepStrictEqual(got, reader(4).load(theirs), `${label}: js-yaml ${v}`); + } + if (text !== theirs) differ.push(label); + } + // The limits: names that start like a number or with `._` (js-yaml 3 reads + // js-yaml 4's plain `._1` as 0.1 and `._e0` as NaN), ids js-yaml 4 writes + // as a block, and lines that are not numbers. + const quoted = ['1_000', '+.5', '._1', '._97', '._e0'].flatMap((s) => [`key "${s}"`, `empty "${s}"`, `id "${s}"`]); + const x = `id "${'x'.repeat(23)}`; + assert.deepEqual(differ, [...quoted, 'id "a\\nb"', + x, x, x, `id "${"'".repeat(23)}`, 'id "/p/qqqqqqqqqqqqqqqqqqqq', 'not numbers', 'id of 77']); +}); diff --git a/test/release.test.mjs b/test/release.test.mjs new file mode 100644 index 0000000..8a3dc9b --- /dev/null +++ b/test/release.test.mjs @@ -0,0 +1,41 @@ +// Rows for the publish workflow's version check (scripts/release-version.mjs). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { refusal } from '../scripts/release-version.mjs'; + +const script = fileURLToPath(new URL('../scripts/release-version.mjs', import.meta.url)); +const { version } = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')); +const run = (...args) => spawnSync(process.execPath, [script, ...args], { encoding: 'utf8' }); + +test('a tag naming the version publishes', () => { + assert.equal(refusal('sdk-v1.2.3', '1.2.3'), ''); +}); + +test('a tag naming another version is refused, and names the right tag', () => { + assert.match(refusal('sdk-v1.2.4', '1.2.3'), /does not name package\.json's version 1\.2\.3: the tag for it is sdk-v1\.2\.3$/); + assert.match(refusal('sdk-v1.2.3.1', '1.2.3'), /does not name/); + assert.match(refusal('v1.2.3', '1.2.3'), /does not name/); + assert.match(refusal('', '1.2.3'), /the tag \(none\) does not name/); +}); + +test('a version that is not x.y.z is refused, even with its own tag', () => { + assert.match(refusal('sdk-v1.2.3-rc.1', '1.2.3-rc.1'), /is not a release \(x\.y\.z\)/); + assert.match(refusal('sdk-v1.2', '1.2'), /is not a release/); + // Four numbers end in x.y.z and start with it: the check matches the whole version. + assert.match(refusal('sdk-v1.2.3.4', '1.2.3.4'), /is not a release/); +}); + +test('the script reads package.json and exits 1 with the reason on a wrong tag', () => { + const ok = run(`sdk-v${version}`); + assert.equal(ok.status, 0, ok.stderr); + assert.equal(ok.stdout + ok.stderr, ''); + const bad = run('sdk-v0.0.0-not-this'); + assert.equal(bad.status, 1); + assert.match(bad.stderr, new RegExp(`^refusing to publish: the tag sdk-v0\\.0\\.0-not-this does not name package\\.json's version ${version.replace(/\./g, '\\.')}`)); + const none = run(); + assert.equal(none.status, 1); + assert.match(none.stderr, /the tag \(none\)/); +}); diff --git a/test/workflows.test.mjs b/test/workflows.test.mjs new file mode 100644 index 0000000..98540de --- /dev/null +++ b/test/workflows.test.mjs @@ -0,0 +1,174 @@ +// Rows over the workflows that publish to npm: each publishes the tarball its +// license check passed, and nothing in it skips that check or these rows; +// every pull request that changes one of them, these rows or the check runs +// these rows; and every job, those of a workflow one calls included, has a +// timeout, so none can hold a runner for GitHub's 6 hours. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readdirSync, readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; + +const YAML = createRequire(import.meta.url)('yaml'); +const dir = new URL('../.github/workflows/', import.meta.url); +const text = (name) => readFileSync(new URL(name, dir), 'utf8'); +const load = (name) => YAML.parse(text(name)); +const TARBALL = '${{ steps.pack.outputs.tarball }}'; + +// The steps of job, in order: each one's name (or command), run, env, id and +// the keys that can skip it or let it fail. +function steps(job) { + return job.steps.map((s) => ({ + name: s.name || s.run || s.uses, run: s.run || '', env: s.env || {}, id: s.id, + if: s.if, continueOnError: s['continue-on-error'], cwd: s['working-directory'], + })); +} + +// Whether GitHub's paths filter pattern p matches path: ** matches any +// characters, * any but a slash. +function matches(p, path) { + const re = p.split(/(\*\*|\*)/).map((s) => (s === '**' ? '.*' : s === '*' ? '[^/]*' : s.replace(/[.+?^${}()|[\]\\]/g, '\\$&'))).join(''); + return new RegExp(`^${re}$`).test(path); +} + +// What runs these rows from the repository's root in each job: the root's +// own rows (this file among them), or, in a job that works in a package's +// folder, the root's install (no scripts: only the YAML parser is needed) and +// this file. +const ALL_ROWS = { name: 'Rows', run: 'node --test test/*.test.mjs', cwd: undefined }; +const WORKFLOW_ROWS = { name: 'Workflow rows', run: 'npm ci --ignore-scripts\nnode --test test/workflows.test.mjs\n', cwd: '.' }; + +// The keys a workflow, and a job that packs, checks or publishes, may have: +// none of them sets the environment of every step (env), or runs the steps +// somewhere else (container). A NODE_OPTIONS that preloads process.exit(0), +// on the workflow or on a job, would pass every node in the job, the license +// check and these rows among them. +const WORKFLOW_KEYS = ['name', 'on', 'permissions', 'jobs']; +const JOB_KEYS = ['name', 'if', 'needs', 'runs-on', 'timeout-minutes', 'permissions', 'defaults', 'steps']; +// In such a job: the actions a step may run, each pinned by commit, and their +// inputs (an action may set any variable for the steps after it); and the +// variables a step may set for itself. +const ACTIONS = ['actions/checkout@11d5960a326750d5838078e36cf38b85af677262', 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020']; +const ACTION_INPUTS = ['node-version', 'registry-url', 'scope']; +// The registry and scope setup-node writes into npm's config, where present: +// the license check installs from that registry. +const ACTION_VALUES = { 'registry-url': 'https://registry.npmjs.org', scope: '@keploy' }; +const STEP_ENV = ['TARBALL', 'NPM_TOKEN', 'NODE_AUTH_TOKEN']; + +const workflows = [ + // [file, tag, folder, publishing job, pull request job, the rows step, the gate's path from the folder] + ['sdk-publish.yml', 'sdk-v*', undefined, 'publish', 'check', ALL_ROWS, 'scripts/license-gate.mjs'], + ['playwright-publish.yml', 'playwright-v*', 'playwright', 'publish', 'dry-run', WORKFLOW_ROWS, '../scripts/license-gate.mjs'], + ['jest-publish.yml', 'jest-v*', 'jest', 'publish', 'dry-run', WORKFLOW_ROWS, '../scripts/license-gate.mjs'], +]; + +test('every workflow that publishes to npm is one these rows hold', () => { + const publishing = readdirSync(dir).filter((n) => /\.ya?ml$/.test(n) && /\bnpm publish\b/.test(text(n))).sort(); + assert.deepEqual(publishing, workflows.map(([file]) => file).sort()); +}); + +for (const [file, tag, folder, publish, pr, rows, gate] of workflows) { + test(`${file}: the tarball packed once is license-checked, then published`, () => { + const wf = load(file); + for (const job of [publish, pr]) { + const s = steps(wf.jobs[job]); + const at = (name) => s.findIndex((x) => x.name === name); + const pack = at('Pack'); + assert.ok(pack >= 0, `${job}: a Pack step`); + assert.equal(s[pack].id, 'pack'); + assert.match(s[pack].run, /^name=\$\(npm pack --pack-destination "\$RUNNER_TEMP" \| tail -n 1\)\n/); + const lic = at('Licenses'); + assert.ok(lic > pack, `${job}: a Licenses step after Pack`); + assert.equal(s[lic].run, `node ${gate} "$TARBALL"`); + assert.equal(s[lic].env.TARBALL, TARBALL); + // Nothing else on the step: no shell of its own, and no other variable + // (one that preloads code, say). + const raw = wf.jobs[job].steps[lic]; + assert.deepEqual(Object.keys(raw).sort(), ['env', 'name', 'run'], `${job}: the Licenses step's keys`); + assert.deepEqual(Object.keys(raw.env), ['TARBALL'], `${job}: the Licenses step's env`); + // npm packs or publishes nothing else. + assert.deepEqual(s.filter((x) => /\bnpm (?:pack|publish)\b/.test(x.run)).map((x) => x.name), job === publish ? ['Pack', 'Publish'] : file === 'sdk-publish.yml' ? ['Pack'] : ['Pack', 'Publish, dry run']); + const pub = s.findIndex((x) => /\bnpm publish\b/.test(x.run)); + if (pub < 0) continue; + assert.ok(pub > lic, `${job}: the publish runs after the Licenses step`); + assert.match(s[pub].run, /^npm publish "\$TARBALL" /); + assert.equal(s[pub].env.TARBALL, TARBALL); + } + assert.equal(steps(wf.jobs[publish]).find((x) => x.name === 'Publish').run, 'npm publish "$TARBALL" --access public --provenance'); + }); + + test(`${file}: nothing skips the license check or the rows, or lets them fail`, () => { + const wf = load(file); + assert.deepEqual(Object.keys(wf).filter((k) => !WORKFLOW_KEYS.includes(k)), [], 'the workflow\'s keys'); + for (const [name, j] of Object.entries(wf.jobs)) assert.equal(j.env, undefined, `${name}: env`); + for (const [job, event] of [[publish, 'push'], [pr, 'pull_request']]) { + const j = wf.jobs[job]; + assert.deepEqual(Object.keys(j).filter((k) => !JOB_KEYS.includes(k)), [], `${job}: its keys`); + assert.equal(j['runs-on'], 'ubuntu-latest', `${job}: runs-on`); + // The job runs on its event, and on nothing narrower. + assert.equal(j.if, `github.event_name == '${event}'`, `${job}: if`); + assert.equal(j['continue-on-error'], undefined, `${job}: continue-on-error`); + // Its steps run in bash (-eo pipefail), in its folder, and nothing else. + assert.deepEqual(j.defaults, { run: { shell: 'bash', ...(folder ? { 'working-directory': folder } : {}) } }, `${job}: defaults`); + const s = steps(j); + for (const x of s) { + assert.equal(x.if, undefined, `${job}: ${x.name}: if`); + assert.equal(x.continueOnError, undefined, `${job}: ${x.name}: continue-on-error`); + // No step sets a variable or a path for the steps after it. + assert.doesNotMatch(x.run, /\bGITHUB_(?:ENV|PATH)\b|::(?:set-env|add-path)\b/i, `${job}: ${x.name}: sets a variable or a path for later steps`); + assert.deepEqual(Object.keys(x.env).filter((k) => !STEP_ENV.includes(k)), [], `${job}: ${x.name}: env`); + } + for (const st of j.steps.filter((st) => st.uses)) { + assert.ok(ACTIONS.includes(st.uses), `${job}: ${st.uses}: an action pinned here`); + assert.deepEqual(Object.keys(st.with || {}).filter((k) => !ACTION_INPUTS.includes(k)), [], `${job}: ${st.uses}: its inputs`); + for (const [k, v] of Object.entries(ACTION_VALUES)) if (st.with && k in st.with) assert.equal(st.with[k], v, `${job}: ${st.uses}: ${k}`); + } + const r = s.findIndex((x) => x.name === rows.name); + assert.ok(r >= 0, `${job}: a ${rows.name} step`); + assert.deepEqual([s[r].run, s[r].cwd], [rows.run, rows.cwd], `${job}: ${rows.name} runs from the root`); + // Nothing else on the rows step: no env, no shell of its own. + assert.deepEqual(Object.keys(j.steps[r]).sort(), rows.cwd ? ['name', 'run', 'working-directory'] : ['name', 'run'], `${job}: the ${rows.name} step's keys`); + assert.ok(r < s.findIndex((x) => x.name === 'Licenses'), `${job}: ${rows.name} before Licenses`); + } + // A tag of this package's name starts the publish; any pull request that + // touches what its paths name starts the check. + assert.deepEqual(wf.on.push, { tags: [tag] }); + assert.deepEqual(Object.keys(wf.on.pull_request), ['paths']); + assert.ok(wf.on.pull_request.paths.every((p) => !p.startsWith('!')), 'no path is excluded'); + }); + + test(`${file}: a pull request that changes it, or the check, runs its own check and these rows`, () => { + const own = load(file).on.pull_request.paths; + const root = load('sdk-publish.yml').on.pull_request.paths; + for (const path of [`.github/workflows/${file}`, 'scripts/license-gate.mjs', ...(folder ? [`${folder}/package.json`] : [])]) { + assert.ok(own.some((p) => matches(p, path)), `${file} runs on a change to ${path}`); + } + // sdk-publish.yml's check runs every root row, these among them. + for (const path of [`.github/workflows/${file}`, 'test/workflows.test.mjs', 'scripts/license-gate.mjs', 'package.json', 'package-lock.json']) { + assert.ok(root.some((p) => matches(p, path)), `sdk-publish.yml runs on a change to ${path}`); + } + }); + + test(`${file}: every job, a called workflow's included, has a timeout`, () => { + const jobs = (wf, from) => Object.entries(wf.jobs).flatMap(([name, job]) => { + if (!job.uses) return [[`${from}: ${name}`, job]]; + const called = job.uses.match(/^\.\/\.github\/workflows\/([^/@]+)$/); + assert.ok(called, `${from}: ${name}: calls a workflow of this repository (${job.uses})`); + return jobs(load(called[1]), called[1]); + }); + const all = jobs(load(file), file); + assert.ok(all.length >= 2); + for (const [name, job] of all) { + assert.ok(Number.isInteger(job['timeout-minutes']) && job['timeout-minutes'] <= 60, `${name}: timeout-minutes`); + } + }); +} + +test('the paths filter matches as GitHub does', () => { + assert.ok(matches('scripts/**', 'scripts/license-gate.mjs')); + assert.ok(matches('v2/**', 'v2/dedup/middleware.ts')); + assert.ok(matches('.github/workflows/jest-*.yml', '.github/workflows/jest-sdk.yml')); + assert.ok(!matches('.github/workflows/*.yml', '.github/workflows/sub/x.yml')); + assert.ok(!matches('scripts/**', 'test/scripts/x.mjs')); + assert.ok(!matches('package.json', 'jest/package.json')); + assert.ok(!matches('a.b', 'axb')); +}); diff --git a/v2/dedup/entry.ts b/v2/dedup/entry.ts new file mode 100644 index 0000000..98fb5dc --- /dev/null +++ b/v2/dedup/entry.ts @@ -0,0 +1,79 @@ +// One dedupData.yaml entry as YAML text: the bytes js-yaml 4's dump() gave +// it as an item of the file's list, so that the middleware appends the new +// entry to the file instead of reading, parsing and rewriting the whole file +// on every request. Our own code: no YAML package is needed. +// Limits, each written another way that every YAML 1.1 and 1.2 reader +// (js-yaml 3 and 4 among them) reads back as the same value: +// - a name that starts like a number (a sign, then a digit, or a dot and a +// digit), or with a dot and an underscore (js-yaml 3 reads `._1` as 0.1), +// or is ~, null, <<, .inf, .nan or a YAML 1.1 or 1.2 boolean, in any +// letter case, is single-quoted where js-yaml 4 may leave it plain; +// - an id with a line break, which js-yaml 4 writes as a block, is +// double-quoted, and one over 76 characters, which it folds, stays on one +// line; +// - a line that is not a number (no coverage tool gives one) is JSON. + +const ESCAPES: { [c: number]: string } = { + 0x00: '\\0', 0x07: '\\a', 0x08: '\\b', 0x09: '\\t', 0x0a: '\\n', 0x0b: '\\v', 0x0c: '\\f', 0x0d: '\\r', + 0x1b: '\\e', 0x22: '\\"', 0x5c: '\\\\', 0x85: '\\N', 0xa0: '\\_', 0x2028: '\\L', 0x2029: '\\P', +}; +// js-yaml 4 single-quotes these, for YAML 1.1 readers, at any length. +const BASE60 = /^[-+]?[0-9_]+(?::[0-9_]+)+(?:\.[0-9_]*)?$/; +// Every text a YAML 1.1 or 1.2 reader takes for a null, boolean, number, +// timestamp or merge key matches this, in js-yaml 3 and 4 too. js-yaml 3's +// float takes a dot then digits or underscores, so it reads plain `._1` as +// 0.1 and `._e0` as NaN: a name that starts with `._` is quoted. +const NOT_A_STRING = /^(?:~|null|true|false|y|yes|n|no|on|off|<<|[-+]?\.(?:inf|nan))$|^\._|^[-+]?\.?[0-9]/i; +// The characters a plain scalar may not start with: space and YAML's indicators. +const NOT_FIRST = ' -?:,[]{}#&*!|=>\'"%@`'; + +// A character js-yaml 4 writes as itself. +function printable(c: number): boolean { + return (c >= 0x20 && c <= 0x7e) || (c >= 0xa1 && c <= 0xd7ff && c !== 0x2028 && c !== 0x2029) || + (c >= 0xe000 && c <= 0xfffd && c !== 0xfeff) || c >= 0x10000; +} + +function escape(c: number): string { + if (ESCAPES[c] !== undefined) return ESCAPES[c]; + if (printable(c)) return String.fromCodePoint(c); + // Every character over U+FFFF is printable, so four digits are enough. + return c <= 0xff ? '\\x' + ('0' + c.toString(16).toUpperCase()).slice(-2) : '\\u' + ('000' + c.toString(16).toUpperCase()).slice(-4); +} + +// s as a scalar, on one line. +export function scalar(s: string): string { + if (s === '') return "''"; + if (BASE60.test(s)) return "'" + s + "'"; + const chars = Array.from(s, (ch) => ch.codePointAt(0) as number); + if (!chars.every(printable)) { + return '"' + chars.map(escape).join('') + '"'; + } + const plain = !NOT_FIRST.includes(s[0]) && !/[ :]$/.test(s) && !/: | #/.test(s) && !NOT_A_STRING.test(s); + return plain ? s : "'" + s.replace(/'/g, "''") + "'"; +} + +// n as js-yaml 4 writes a number. +export function number(n: number): string { + if (Number.isNaN(n)) return '.nan'; + if (n === Infinity) return '.inf'; + if (n === -Infinity) return '-.inf'; + if (Object.is(n, -0)) return '-0.0'; + const s = String(n); + return n % 1 !== 0 && /^[-+]?[0-9]+e/.test(s) ? s.replace('e', '.e') : s; +} + +// The entry for one request, ending in a line break. +export function entryYaml(id: string, executedLinesByFile: { [filename: string]: unknown[] }): string { + let out = '- id: ' + scalar(id) + '\n executedLinesByFile:'; + const files = Object.keys(executedLinesByFile); + if (files.length === 0) return out + ' {}\n'; + for (const file of files) { + const key = scalar(file); + const items = executedLinesByFile[file].map((n) => '- ' + (typeof n === 'number' ? number(n) : JSON.stringify(n) ?? 'null')); + // A key over 1024 characters is written explicit (? key, then : value), + // as YAML allows no longer implicit key. + if (key.length > 1024) out += '\n ? ' + key + '\n : ' + (items.length ? items.join('\n ') : '[]'); + else out += '\n ' + key + ':' + (items.length ? '\n ' + items.join('\n ') : ' []'); + } + return out + '\n'; +} diff --git a/v2/dedup/middleware.ts b/v2/dedup/middleware.ts index b13ca9e..dfbb432 100644 --- a/v2/dedup/middleware.ts +++ b/v2/dedup/middleware.ts @@ -2,7 +2,7 @@ import { log } from "console"; import { Request, Response, NextFunction } from "express"; const fs = require('fs'); -const yaml = require('js-yaml'); +import { entryYaml } from "./entry"; const filePath = 'dedupData.yaml'; @@ -10,13 +10,11 @@ const filePath = 'dedupData.yaml'; export default function middleware(): (req: Request, res: Response, next: NextFunction) => void { // console.log("Inside middleware..."); - // @ts-ignore - fs.access(filePath, fs.constants.F_OK, (err) => { - if (err) { - // Create the file if it doesn't exist - fs.writeFileSync(filePath, '', 'utf-8'); - } - }); + // Create the file if it does not exist, now, with an append of nothing, + // which never empties it; a file that exists is not written. Checking for + // it asynchronously and then writing it empty, as before, dropped the + // entries of requests that finished between the check and its callback. + if (!fs.existsSync(filePath)) fs.appendFileSync(filePath, '', 'utf-8'); return (req: Request, res: Response, next: NextFunction) => { res.on("finish", () => { afterMiddleware(req, res); @@ -34,34 +32,10 @@ export function afterMiddleware(req: Request, res: Response) { } let executedLinesByFile = GetCoverage(); - let currentData = { - id: testSet + "/" + id, - executedLinesByFile: executedLinesByFile - }; - - let existingData = []; - - try { - const fileContent = fs.readFileSync(filePath, 'utf-8'); - existingData = yaml.load(fileContent) || []; - } catch (error) { - // Handle the case where the file doesn't exist or is not valid YAML - console.error("Error reading existing file:", error); - } - - if (!Array.isArray(existingData)) { - console.error('Expected an array for existingData, but got:', typeof existingData); - existingData = []; // Reset to an empty array or handle accordingly - } - - // Add or update the entry for the current id - existingData.push(currentData); - - // Convert the array to YAML format - const yamlData = yaml.dump(existingData); - - // Write the updated YAML data back to the file - fs.writeFileSync(filePath, yamlData, 'utf-8'); + // The file is a list that only grows: append this request's entry, as + // js-yaml 4 wrote it, rather than read, parse and rewrite the whole file, + // which cost each request time and memory in proportion to the file. + fs.appendFileSync(filePath, entryYaml(testSet + "/" + id, executedLinesByFile), 'utf-8'); // Log to the console // console.log("Executed lines by file:", executedLinesByFile);