From e35e83252bf4cec71bbd3713103442387ca9fdde Mon Sep 17 00:00:00 2001 From: slayerjain Date: Wed, 4 Nov 2026 15:58:06 +0530 Subject: [PATCH 1/9] fix: ship @keploy/sdk without typescript, and check licenses on publish @keploy/sdk listed typescript under dependencies, so every install of the SDK pulled in typescript, whose ThirdPartyNoticeText.txt carries the Unicode, Inc. license agreement that license scanners flag. Nothing imports typescript at run time; it only builds dist/. It moves to devDependencies, so an install of the SDK no longer brings it. - package.json said "license": "MIT", while LICENSE, the README badge and the jest and playwright packages say Apache-2.0. It now says Apache-2.0. - package.json lists the files the package ships: dist/ and the sources that 2.0.10 shipped. Without that list, a pack of main also carried jest/ and playwright/ (sources, rows, the e2e lane and their lockfiles, which name a CC-BY-4.0 package). - The version is 2.0.11 (npm has 2.0.10; package.json said 2.0.2). - @types/express and @types/node stay in dependencies: dist/v2/dedup/middleware.d.ts imports express's types, which import node's, so a TypeScript user of the middleware needs them. Both trees are MIT. sdk-publish.yml publishes @keploy/sdk from an sdk-vX.Y.Z tag, the way the jest and playwright packages publish: the tag must name package.json's version, the rows run, the package is packed once, the tarball must carry its build, scripts/license-gate.mjs checks it, and that same tarball is published with provenance. On a pull request the same pack and checks run, with no token. scripts/license-gate.mjs installs the tarball into an empty project as a user's npm install does (no devDependencies), then fails when any installed package's license field names an id outside the allow-list (MIT, BSD-2-Clause, BSD-3-Clause, 0BSD, Apache-2.0, ISC, Zlib, BSL-1.0, CC0-1.0, Unlicense, PostgreSQL, OFL-1.1), when a field is missing, and when any installed file names a flagged license by name, SPDX id or URL (GPL, LGPL, AGPL, MPL, Unicode terms, SSPL, BUSL, EPL, CDDL, EUPL, OSL, Creative Commons, Elastic, Commons Clause). It has no exception list. Checked by installing each package into an empty project: - @keploy/sdk@2.0.10 from npm: the check fails on node_modules/typescript/ThirdPartyNoticeText.txt (53 packages). - main, packed: the same, plus CC-BY in the two jest lockfiles it ships. - this commit, packed: 52 packages, every license allowed (51 MIT, the SDK Apache-2.0). The installed tree is 2.0.10's without typescript, and dist/*.js and dist/*.d.ts are byte for byte those of 2.0.10. Signed-off-by: slayerjain --- .github/workflows/sdk-publish.yml | 141 ++++++++++++++++++++ package-lock.json | 5 +- package.json | 15 ++- scripts/license-gate.mjs | 205 ++++++++++++++++++++++++++++++ scripts/release-version.mjs | 30 +++++ test/license-gate-cli.test.mjs | 93 ++++++++++++++ test/license-gate.test.mjs | 194 ++++++++++++++++++++++++++++ test/release.test.mjs | 41 ++++++ 8 files changed, 717 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/sdk-publish.yml create mode 100644 scripts/license-gate.mjs create mode 100644 scripts/release-version.mjs create mode 100644 test/license-gate-cli.test.mjs create mode 100644 test/license-gate.test.mjs create mode 100644 test/release.test.mjs diff --git a/.github/workflows/sdk-publish.yml b/.github/workflows/sdk-publish.yml new file mode 100644 index 0000000..17225cd --- /dev/null +++ b/.github/workflows/sdk-publish.yml @@ -0,0 +1,141 @@ +name: "@keploy/sdk publish" + +# Publishes @keploy/sdk, the package at the repository's root, to npm from a +# tag. +# +# To release X.Y.Z: set "version": "X.Y.Z" in package.json on main (a pull +# request, so the check below runs on it), then push the tag sdk-vX.Y.Z on +# that commit. The publish refuses a tag that does not name package.json's +# version (scripts/release-version.mjs); npm itself refuses a version it +# already has. Every action this workflow runs is pinned by commit: the +# publish job holds the npm token and signs the package's provenance. +# +# The package is packed once, and that tarball is checked and then published, +# so what ships is what was checked. The license check +# (scripts/license-gate.mjs) installs the tarball as a user's `npm install` +# does and refuses it when any installed package's license field, or any +# installed file, names a license outside its allow-list. +# +# Push sdk-vX.Y.Z as a plain git tag, never as a GitHub release: a published +# GitHub release starts any workflow of this repository that runs on +# releases. +# +# On a pull request the same tarball is packed and checked, with no token. +on: + push: + tags: ["sdk-v*"] + pull_request: + paths: + - "package.json" + - "package-lock.json" + - "tsconfig.json" + - "keployCli.ts" + - "v2/**" + - "scripts/**" + - "test/**" + - ".github/workflows/sdk-publish.yml" + +permissions: + contents: read + +jobs: + publish: + name: publish to npm + if: github.event_name == 'push' + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write # npm --provenance signs the package with this run's identity + defaults: + run: + shell: bash # -eo pipefail: a failed command before a pipe fails the step + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + # Writes the token line for registry.npmjs.org and pins the @keploy + # scope to it (@keploy:registry), in the runner's own npmrc. + registry-url: https://registry.npmjs.org + scope: "@keploy" + - name: The tag names package.json's version + run: node scripts/release-version.mjs "$GITHUB_REF_NAME" + - name: The npm token is set + env: + NPM_TOKEN: ${{ secrets.NPM_TOKEN }} + run: test -n "$NPM_TOKEN" || { echo "::error::the NPM_TOKEN secret is not set; refusing to publish"; exit 1; } + - name: The @keploy scope publishes to registry.npmjs.org + # npm sends a scoped package to its scope's registry, whatever + # --registry says. Read the setting npm will use, with every npmrc in + # it (a project one included), and refuse any other. + run: | + got=$(npm config get @keploy:registry) + test "$got" = https://registry.npmjs.org/ || { echo "::error::@keploy:registry is '$got', not https://registry.npmjs.org/; refusing to publish"; exit 1; } + - run: npm ci + - name: Rows + run: node --test test/*.test.mjs + - name: Pack + id: pack + # prepare builds dist/ first. + run: | + name=$(npm pack --pack-destination "$RUNNER_TEMP" | tail -n 1) + test -f "$RUNNER_TEMP/$name" || { echo "::error::npm pack wrote no $name"; exit 1; } + echo "tarball=$RUNNER_TEMP/$name" >> "$GITHUB_OUTPUT" + - name: The package carries its build + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: | + tar -tzf "$TARBALL" > "$RUNNER_TEMP/files.txt" + for f in dist/keployCli.js dist/keployCli.d.ts dist/v2/dedup/register.js dist/v2/dedup/middleware.js package.json README.MD LICENSE; do + grep -qx "package/$f" "$RUNNER_TEMP/files.txt" || { echo "::error::the package would ship without $f"; exit 1; } + done + - name: Licenses + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: node scripts/license-gate.mjs "$TARBALL" + - name: The token authenticates + # Fails here, named, on a token that is expired, revoked or pasted + # with a stray character, before anything is uploaded. + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + run: npm whoami --registry https://registry.npmjs.org/ + - name: Publish + env: + NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} + TARBALL: ${{ steps.pack.outputs.tarball }} + run: npm publish "$TARBALL" --access public --provenance + + check: + name: pack and license check + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + defaults: + run: + shell: bash # -eo pipefail: a failed command before a pipe fails the step + steps: + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version: 22 + - run: npm ci + - name: Rows + run: node --test test/*.test.mjs + - name: Pack + id: pack + # prepare builds dist/ first. + run: | + name=$(npm pack --pack-destination "$RUNNER_TEMP" | tail -n 1) + test -f "$RUNNER_TEMP/$name" || { echo "::error::npm pack wrote no $name"; exit 1; } + echo "tarball=$RUNNER_TEMP/$name" >> "$GITHUB_OUTPUT" + - name: The package carries its build + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: | + tar -tzf "$TARBALL" > "$RUNNER_TEMP/files.txt" + for f in dist/keployCli.js dist/keployCli.d.ts dist/v2/dedup/register.js dist/v2/dedup/middleware.js package.json README.MD LICENSE; do + grep -qx "package/$f" "$RUNNER_TEMP/files.txt" || { echo "::error::the package would ship without $f"; exit 1; } + done + - name: Licenses + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: node scripts/license-gate.mjs "$TARBALL" diff --git a/package-lock.json b/package-lock.json index 411c3ef..47097fe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,6 +1,6 @@ { "name": "@keploy/sdk", - "version": "2.0.2", + "version": "2.0.11", "lockfileVersion": 1, "requires": true, "dependencies": { @@ -265,7 +265,8 @@ "typescript": { "version": "5.3.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", - "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==" + "integrity": "sha512-pXWcraxM0uxAS+tN0AG/BF2TyqmHO014Z070UsJ+pFvYuRSq8KH8DmWpnbXe0pEPDHXZV3FcAbJkijJ5oNEnWw==", + "dev": true }, "undici-types": { "version": "5.26.5", diff --git a/package.json b/package.json index 8fccf7a..3d32e7c 100644 --- a/package.json +++ b/package.json @@ -1,9 +1,14 @@ { "name": "@keploy/sdk", - "version": "2.0.2", + "version": "2.0.11", "description": "This is the client SDK for the [Keploy](https://github.com/keploy/keploy) testing platform. With the TypeScript SDK, you can test both your existing unit test cases in Jest and create new end-to-end test cases for your applications.", "main": "dist/keployCli.js", "types": "dist/keployCli.d.ts", + "files": [ + "dist", + "keployCli.ts", + "v2" + ], "scripts": { "build": "tsc", "prepare": "npm run build", @@ -14,7 +19,7 @@ "url": "git+https://github.com/keploy/typescript-sdk.git" }, "author": "Keploy ", - "license": "MIT", + "license": "Apache-2.0", "bugs": { "url": "https://github.com/keploy/keploy/issues" }, @@ -26,11 +31,11 @@ "cors": "^2.8.5", "lodash": "^4.17.21", "require-in-the-middle": "^7.3.0", - "tree-kill": "^1.2.2", - "typescript": "^5.3.3" + "tree-kill": "^1.2.2" }, "devDependencies": { "@types/axios": "^0.14.0", - "@types/cors": "^2.8.17" + "@types/cors": "^2.8.17", + "typescript": "^5.3.3" } } diff --git a/scripts/license-gate.mjs b/scripts/license-gate.mjs new file mode 100644 index 0000000..086efea --- /dev/null +++ b/scripts/license-gate.mjs @@ -0,0 +1,205 @@ +// license-gate.mjs TARBALL: the publish workflow's license check. +// +// TARBALL is the file `npm pack` wrote, the one the workflow then publishes. +// The check installs it into an empty project the way a user does (`npm +// install`, so no devDependencies), and refuses it when anything that install +// puts on disk names a license outside the allow-list below: +// - each installed package's package.json license field (or the old +// `licenses` list) must name allowed SPDX ids only. A missing field, an +// id not on the list, "SEE LICENSE IN ..." and UNLICENSED all fail. +// - every file of every installed package is read for the names, SPDX ids +// and URLs of the licenses a scanner flags (LGPL, GPL, AGPL, MPL, Unicode +// terms, SSPL, BUSL, EPL, CDDL, EUPL, OSL, CC-BY-SA and other Creative +// Commons licenses, Elastic, Commons Clause), and for opensource.org +// license URLs that name no allowed id. +// Accepting a flagged file with a notice is not enough, so there is no +// exception list: a finding fails the publish. +// +// Exit 0 and print what was checked when the package is clean; exit 1 and +// list every finding when not, or when the install fails. +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join, relative, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +export const ALLOWED = new Set([ + 'MIT', 'BSD-2-Clause', 'BSD-3-Clause', '0BSD', 'Apache-2.0', 'ISC', 'Zlib', + 'BSL-1.0', 'CC0-1.0', 'Unlicense', 'PostgreSQL', 'OFL-1.1', +]); + +// opensource.org/licenses/: the pages that name an allowed license. +const OSI_PAGES = new Map([ + ['mit', 'MIT'], ['mit-license', 'MIT'], ['bsd-2-clause', 'BSD-2-Clause'], + ['bsd-3-clause', 'BSD-3-Clause'], ['bsd-license', 'BSD-3-Clause'], ['0bsd', '0BSD'], + ['apache-2.0', 'Apache-2.0'], ['isc', 'ISC'], ['isc-license', 'ISC'], ['zlib', 'Zlib'], + ['zlib-license', 'Zlib'], ['bsl-1.0', 'BSL-1.0'], ['unlicense', 'Unlicense'], + ['postgresql', 'PostgreSQL'], ['ofl-1.1', 'OFL-1.1'], +]); + +// What a scanner flags in a file, by name, SPDX id or URL. The SPDX ids are +// matched case-sensitively, as they are written; names and URLs in any case. +const FLAGGED = [ + ['GPL, LGPL or AGPL', /\b(?:L|A)?GPL(?:v?[123]|-[123]\.[01])?\b/], + ['GPL, LGPL or AGPL', /GNU (?:Lesser |Library |Affero )?General Public Licen[cs]e/i], + ['GPL, LGPL or AGPL', /gnu\.org\/(?:licenses|copyleft)/i], + ['MPL', /\bMPL-?[12]\.[01]\b/], + ['MPL', /Mozilla Public Licen[cs]e/i], + ['MPL', /mozilla\.org\/(?:media\/)?MPL/i], + ['Unicode terms', /\bUnicode-(?:TOU|DFS-20\d\d|3\.0)\b/], + ['Unicode terms', /Unicode,? Inc\.? (?:License Agreement|Terms of Use)/i], + ['Unicode terms', /Unicode Terms of Use/i], + ['Unicode terms', /unicode\.org\/(?:copyright|license|terms_of_use)/i], + ['SSPL', /\bSSPL\b/], + ['SSPL', /Server Side Public Licen[cs]e/i], + ['BUSL', /\bBUSL\b/], + ['BUSL', /Business Source Licen[cs]e/i], + ['EPL', /\bEPL-[12]\.0\b/], + ['EPL', /Eclipse Public Licen[cs]e/i], + ['EPL', /eclipse\.org\/legal/i], + ['CDDL', /\bCDDL\b/], + ['CDDL', /Common Development and Distribution Licen[cs]e/i], + ['EUPL', /\bEUPL\b/], + ['EUPL', /European Union Public Licen[cs]e/i], + ['OSL', /\bOSL-[123]\.[01]\b/], + ['OSL', /Open Software Licen[cs]e/i], + ['Creative Commons', /\bCC-BY(?:-NC)?(?:-ND)?(?:-SA)?\b/], + ['Creative Commons', /creativecommons\.org\/licenses\//i], + ['Creative Commons', /Attribution-(?:NonCommercial-)?ShareAlike/i], + ['Elastic', /\bElastic-2\.0\b/], + ['Elastic', /Elastic Licen[cs]e/i], + ['Commons Clause', /Commons Clause/i], +]; + +const OSI_URL = /opensource\.org\/licenses\/([A-Za-z0-9.+-]+)/gi; + +// The findings for one file's text, as "what (the text that matched)". +export function textFindings(text) { + const found = []; + for (const [what, re] of FLAGGED) { + const m = re.exec(text); + if (m) found.push(`${what} (${JSON.stringify(m[0])})`); + } + for (const m of text.matchAll(OSI_URL)) { + const page = m[1].toLowerCase().replace(/\.$/, '').replace(/\.(?:php|html?)$/, ''); + if (!OSI_PAGES.has(page)) found.push(`a license that is not allowed (${JSON.stringify(m[0])})`); + } + return found; +} + +// The SPDX ids of a license expression: every id must be allowed, whatever +// joins them (OR, AND, WITH), since a scanner flags each id it reads. +export function expressionIds(expr) { + return expr.replace(/[()]/g, ' ').split(/\s+/).filter((t) => t && !/^(?:OR|AND|WITH)$/.test(t)); +} + +// The findings for one installed package's license field. +export function licenseFindings(pkg) { + let values; + if (typeof pkg.license === 'string') values = [pkg.license]; + else if (pkg.license && typeof pkg.license.type === 'string') values = [pkg.license.type]; + else if (Array.isArray(pkg.licenses)) values = pkg.licenses.map((l) => (typeof l === 'string' ? l : l && l.type)); + else return ['no license field']; + const found = []; + for (const v of values) { + if (typeof v !== 'string' || v.trim() === '') { + found.push('an empty license field'); + continue; + } + for (const id of expressionIds(v)) { + if (!ALLOWED.has(id)) found.push(`license ${JSON.stringify(id)} is not allowed (field ${JSON.stringify(v)})`); + } + } + return found; +} + +// The findings for one installed package's package.json, at its folder dir. +function packageFindings(dir, at) { + let pkg; + try { + pkg = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')); + } catch (err) { + return [`${at}: no readable package.json (${err.message})`]; + } + return licenseFindings(pkg).map((f) => `${at} (${pkg.name}@${pkg.version}): ${f}`); +} + +// The findings for the tree installed in project, and what was checked. +// Every file under its node_modules is read, whichever package it belongs +// to (npm's own files there, such as .package-lock.json, included), and +// every package's package.json: node_modules/NAME and +// node_modules/@SCOPE/NAME, at any depth. A link is not read: npm makes +// links only in .bin folders, to files read where they are, so any other +// link, or anything that is not a file or a folder, is a finding. +export function check(project) { + const findings = []; + let packages = 0; + let files = 0; + const walk = (dir, modules, scope) => { + for (const e of readdirSync(dir, { withFileTypes: true })) { + const p = join(dir, e.name); + const at = relative(project, p); + if (e.isDirectory()) { + if (modules && e.name.startsWith('@')) { + walk(p, false, true); + continue; + } + // A folder with no package.json is not a package (a package's test + // fixtures can sit under a node_modules folder of their own); its + // files are read all the same. + if ((scope || (modules && !e.name.startsWith('.'))) && existsSync(join(p, 'package.json'))) { + packages++; + findings.push(...packageFindings(p, at)); + } + walk(p, e.name === 'node_modules', false); + } else if (e.isFile()) { + files++; + for (const f of textFindings(readFileSync(p, 'latin1'))) findings.push(`${at}: ${f}`); + } else if (!(e.isSymbolicLink() && basename(dir) === '.bin')) { + findings.push(`${at}: not a file or a folder, so not read`); + } + } + }; + const modules = join(project, 'node_modules'); + if (existsSync(modules)) walk(modules, true, false); + return { findings, packages, files }; +} + +// Installs tarball into a new empty project, as a user's `npm install` does. +export function install(tarball, project) { + writeFileSync(join(project, 'package.json'), JSON.stringify({ name: 'license-gate', version: '0.0.0', private: true })); + const r = spawnSync('npm', ['install', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', '--no-package-lock', resolve(tarball)], { cwd: project, encoding: 'utf8' }); + if (r.status !== 0) return `npm install ${tarball} failed (${r.error ? r.error.message : `exit ${r.status}`}): ${(r.stderr || '').trim()}`; + return ''; +} + +export function main(tarball) { + if (!existsSync(tarball ?? '')) { + console.error(`license gate: no package to check (${tarball || 'no TARBALL given'})`); + return 1; + } + const project = mkdtempSync(join(tmpdir(), 'license-gate-')); + try { + const failed = install(tarball, project); + if (failed) { + console.error(`license gate: ${failed}`); + return 1; + } + const { findings, packages, files } = check(project); + if (packages === 0) { + console.error(`license gate: installing ${tarball} put no package on disk`); + return 1; + } + if (findings.length) { + console.error(`license gate: ${tarball} would ship ${findings.length} license finding(s) in ${packages} packages:`); + for (const f of findings) console.error(` ${f}`); + return 1; + } + console.log(`license gate: ${tarball}: ${packages} packages, ${files} files, every license allowed`); + return 0; + } finally { + rmSync(project, { recursive: true, force: true }); + } +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) process.exit(main(process.argv[2])); diff --git a/scripts/release-version.mjs b/scripts/release-version.mjs new file mode 100644 index 0000000..78708be --- /dev/null +++ b/scripts/release-version.mjs @@ -0,0 +1,30 @@ +// release-version.mjs TAG: the publish workflow's version check. +// +// TAG is the pushed git tag. It must name package.json's version exactly +// (sdk-v), so the version on npm is always the one a tagged +// commit carries. The version must be a release (x.y.z): npm 10 would make a +// prerelease the package's `latest`, so this workflow refuses one. +// +// Exit 0 with no output when the tag may publish; exit 1 and say why when not. +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; + +export function refusal(tag, version) { + if (!/^\d+\.\d+\.\d+$/.test(version)) { + return `package.json's version ${version} is not a release (x.y.z); this workflow publishes releases only`; + } + const want = `sdk-v${version}`; + if (tag !== want) { + return `the tag ${tag || '(none)'} does not name package.json's version ${version}: the tag for it is ${want}`; + } + return ''; +} + +if (process.argv[1] === fileURLToPath(import.meta.url)) { + const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')); + const why = refusal(process.argv[2] ?? '', pkg.version); + if (why) { + console.error(`refusing to publish: ${why}`); + process.exit(1); + } +} diff --git a/test/license-gate-cli.test.mjs b/test/license-gate-cli.test.mjs new file mode 100644 index 0000000..697223c --- /dev/null +++ b/test/license-gate-cli.test.mjs @@ -0,0 +1,93 @@ +// Rows for scripts/license-gate.mjs run as the publish workflow runs it: on a +// tarball `npm pack` wrote. Each row packs a small package of its own (no +// dependencies, so npm needs no registry), then runs the script on it. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const script = fileURLToPath(new URL('../scripts/license-gate.mjs', import.meta.url)); + +const work = mkdtempSync(join(tmpdir(), 'license-gate-cli-')); +test.after(() => rmSync(work, { recursive: true, force: true })); + +// Packs a package named name from files, and returns the tarball's path. +function pack(name, files) { + const dir = join(work, name); + mkdirSync(dir, { recursive: true }); + for (const [path, body] of Object.entries(files)) writeFileSync(join(dir, path), typeof body === 'string' ? body : JSON.stringify(body)); + const r = spawnSync('npm', ['pack', '--pack-destination', work], { cwd: dir, encoding: 'utf8' }); + assert.equal(r.status, 0, r.stderr); + return join(work, r.stdout.trim().split('\n').pop()); +} + +// Runs the script with its own TMPDIR, so a row can see what it leaves there. +function run(args, env = {}) { + const tmp = mkdtempSync(join(work, 'tmp-')); + const r = spawnSync(process.execPath, [script, ...args], { encoding: 'utf8', env: { ...process.env, TMPDIR: tmp, ...env } }); + // npm keeps caches of its own there (node-compile-cache): only the script's + // project counts. + return { ...r, left: readdirSync(tmp).filter((n) => n.startsWith('license-gate-')) }; +} + +test('a clean package passes, and the project it was installed into is gone', () => { + const tgz = pack('clean', { 'package.json': { name: 'clean', version: '1.0.0', license: 'Apache-2.0' }, 'index.js': 'module.exports = 1;' }); + const r = run([tgz]); + assert.equal(r.status, 0, r.stderr); + // The files: the package's two and what npm writes beside them. + assert.match(r.stdout, new RegExp(`^license gate: ${tgz.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}: 1 packages, \\d+ files, every license allowed\n$`)); + assert.deepEqual(r.left, []); +}); + +test('a package whose license field is not allowed fails, named', () => { + const tgz = pack('red', { 'package.json': { name: 'red', version: '1.0.0', license: 'GPL-3.0' } }); + const r = run([tgz]); + assert.equal(r.status, 1); + assert.equal(r.stdout, ''); + assert.match(r.stderr, /would ship \d+ license finding\(s\) in 1 packages:\n/); + assert.match(r.stderr, /\n {2}node_modules\/red \(red@1\.0\.0\): license "GPL-3\.0" is not allowed \(field "GPL-3\.0"\)\n/); + assert.deepEqual(r.left, []); +}); + +test('a package carrying a flagged file fails, with the file named', () => { + const tgz = pack('notice', { + 'package.json': { name: 'notice', version: '1.0.0', license: 'MIT' }, + 'ThirdPartyNoticeText.txt': 'UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE\nSee Terms of Use at http://www.unicode.org/copyright.html', + }); + const r = run([tgz]); + assert.equal(r.status, 1); + assert.match(r.stderr, /\n {2}node_modules\/notice\/ThirdPartyNoticeText\.txt: Unicode terms \("UNICODE, INC\. LICENSE AGREEMENT"\)\n/); + assert.match(r.stderr, /\n {2}node_modules\/notice\/ThirdPartyNoticeText\.txt: Unicode terms \("unicode\.org\/copyright"\)\n/); +}); + +test('a file that is not a package fails at the install', () => { + const bad = join(work, 'not-a-package.tgz'); + writeFileSync(bad, 'not a tarball'); + const r = run([bad]); + assert.equal(r.status, 1); + assert.match(r.stderr, /^license gate: npm install .*not-a-package\.tgz failed \(exit \d+\): /); + assert.deepEqual(r.left, []); +}); + +test('no tarball, or one that is not there, fails before any install', () => { + const none = run([]); + assert.equal(none.status, 1); + assert.equal(none.stderr, 'license gate: no package to check (no TARBALL given)\n'); + const missing = run([join(work, 'missing.tgz')]); + assert.equal(missing.status, 1); + assert.equal(missing.stderr, `license gate: no package to check (${join(work, 'missing.tgz')})\n`); + assert.deepEqual(missing.left, []); +}); + +test('an install that puts nothing on disk fails: nothing checked is not clean', () => { + // npm_config_dry_run, set in a job's environment, makes `npm install` exit + // 0 having installed nothing. + const tgz = pack('dry', { 'package.json': { name: 'dry', version: '1.0.0', license: 'MIT' } }); + const r = run([tgz], { npm_config_dry_run: 'true' }); + assert.equal(r.status, 1); + assert.equal(r.stderr, `license gate: installing ${tgz} put no package on disk\n`); + assert.deepEqual(r.left, []); +}); diff --git a/test/license-gate.test.mjs b/test/license-gate.test.mjs new file mode 100644 index 0000000..92b5042 --- /dev/null +++ b/test/license-gate.test.mjs @@ -0,0 +1,194 @@ +// Rows for the publish workflow's license check (scripts/license-gate.mjs): +// its classifier and its walk of an installed tree. The rows that run the +// script on packed tarballs are in license-gate-cli.test.mjs. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { ALLOWED, check, expressionIds, licenseFindings, textFindings } from '../scripts/license-gate.mjs'; + +test('every allowed SPDX id passes, alone and in an expression', () => { + const ids = ['MIT', 'BSD-2-Clause', 'BSD-3-Clause', '0BSD', 'Apache-2.0', 'ISC', 'Zlib', 'BSL-1.0', 'CC0-1.0', 'Unlicense', 'PostgreSQL', 'OFL-1.1']; + assert.deepEqual([...ALLOWED].sort(), [...ids].sort()); + for (const id of ids) assert.deepEqual(licenseFindings({ license: id }), [], id); + assert.deepEqual(licenseFindings({ license: '(MIT OR Apache-2.0)' }), []); + assert.deepEqual(licenseFindings({ license: 'MIT AND ISC' }), []); + assert.deepEqual(licenseFindings({ license: { type: 'ISC' } }), []); + assert.deepEqual(licenseFindings({ licenses: [{ type: 'BSD-3-Clause' }, 'MIT'] }), []); +}); + +test('a license field naming anything else fails, each id on its own', () => { + assert.deepEqual(licenseFindings({ license: 'LGPL-2.1' }), ['license "LGPL-2.1" is not allowed (field "LGPL-2.1")']); + assert.deepEqual(licenseFindings({ license: '(MIT OR GPL-3.0)' }), ['license "GPL-3.0" is not allowed (field "(MIT OR GPL-3.0)")']); + assert.deepEqual(licenseFindings({ license: 'MPL-2.0 AND Unicode-TOU' }), [ + 'license "MPL-2.0" is not allowed (field "MPL-2.0 AND Unicode-TOU")', + 'license "Unicode-TOU" is not allowed (field "MPL-2.0 AND Unicode-TOU")', + ]); + // An exception id is not on the list either. + assert.deepEqual(licenseFindings({ license: 'Apache-2.0 WITH LLVM-exception' }), ['license "LLVM-exception" is not allowed (field "Apache-2.0 WITH LLVM-exception")']); + // An id that holds an operator's letters is still an id. + assert.deepEqual(licenseFindings({ license: 'LicenseRef-ORACLE' }), ['license "LicenseRef-ORACLE" is not allowed (field "LicenseRef-ORACLE")']); + assert.deepEqual(licenseFindings({ license: 'UNLICENSED' }), ['license "UNLICENSED" is not allowed (field "UNLICENSED")']); + assert.match(licenseFindings({ license: 'SEE LICENSE IN LICENSE.md' }).join('\n'), /license "SEE" is not allowed/); + assert.deepEqual(licenseFindings({ license: { type: 'EPL-2.0' } }), ['license "EPL-2.0" is not allowed (field "EPL-2.0")']); + assert.deepEqual(licenseFindings({ licenses: [{ type: 'MIT' }, { type: 'CDDL-1.0' }] }), ['license "CDDL-1.0" is not allowed (field "CDDL-1.0")']); +}); + +test('a missing or empty license field fails', () => { + assert.deepEqual(licenseFindings({ name: 'x' }), ['no license field']); + assert.deepEqual(licenseFindings({ license: { url: 'https://example.com' } }), ['no license field']); + assert.deepEqual(licenseFindings({ license: '' }), ['an empty license field']); + assert.deepEqual(licenseFindings({ license: ' ' }), ['an empty license field']); + assert.deepEqual(licenseFindings({ licenses: [{ url: 'https://example.com' }] }), ['an empty license field']); +}); + +test('the ids of an expression drop only parentheses and operators', () => { + assert.deepEqual(expressionIds('((MIT OR Apache-2.0) AND BSD-3-Clause)'), ['MIT', 'Apache-2.0', 'BSD-3-Clause']); + assert.deepEqual(expressionIds('GPL-2.0 WITH Classpath-exception-2.0'), ['GPL-2.0', 'Classpath-exception-2.0']); +}); + +// Each sample matches one pattern of the check, and the findings are listed +// whole: a pattern taken out of the check changes its row's list. +const flagged = [ + ['Licensed under LGPL-2.1 terms', ['GPL, LGPL or AGPL ("LGPL-2.1")']], + ['AGPLv3 only', ['GPL, LGPL or AGPL ("AGPLv3")']], + ['GPL-3.0 here', ['GPL, LGPL or AGPL ("GPL-3.0")']], + ['the GNU Lesser General Public License applies', ['GPL, LGPL or AGPL ("GNU Lesser General Public License")']], + ['see the gnu affero general public licence', ['GPL, LGPL or AGPL ("gnu affero general public licence")']], + ['', ['GPL, LGPL or AGPL ("gnu.org/licenses")']], + ['https://www.gnu.org/copyleft/', ['GPL, LGPL or AGPL ("gnu.org/copyleft")']], + ['SPDX-License-Identifier: MPL-2.0', ['MPL ("MPL-2.0")']], + ['subject to the Mozilla Public License, v. 2.0', ['MPL ("Mozilla Public License")']], + ['https://www.mozilla.org/media/MPL/2.0/index.txt', ['MPL ("mozilla.org/media/MPL")']], + ['http://mozilla.org/MPL/2.0/', ['MPL ("mozilla.org/MPL")']], + ['License: Unicode-TOU', ['Unicode terms ("Unicode-TOU")']], + ['Unicode-DFS-2016', ['Unicode terms ("Unicode-DFS-2016")']], + ['UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE', ['Unicode terms ("UNICODE, INC. LICENSE AGREEMENT")']], + ['Data Files and Software: Unicode Terms of Use apply', ['Unicode terms ("Unicode Terms of Use")']], + ['http://www.unicode.org/copyright.html', ['Unicode terms ("unicode.org/copyright")']], + ['SSPL', ['SSPL ("SSPL")']], + ['the Server Side Public License', ['SSPL ("Server Side Public License")']], + ['BUSL-1.1', ['BUSL ("BUSL")']], + ['Business Source License 1.1', ['BUSL ("Business Source License")']], + ['EPL-2.0', ['EPL ("EPL-2.0")']], + ['Eclipse Public License', ['EPL ("Eclipse Public License")']], + ['https://www.eclipse.org/legal/epl-v20.html', ['EPL ("eclipse.org/legal")']], + ['CDDL', ['CDDL ("CDDL")']], + ['Common Development and Distribution License', ['CDDL ("Common Development and Distribution License")']], + ['EUPL', ['EUPL ("EUPL")']], + ['European Union Public Licence', ['EUPL ("European Union Public Licence")']], + ['OSL-3.0', ['OSL ("OSL-3.0")']], + ['Open Software License', ['OSL ("Open Software License")']], + ['CC-BY-SA', ['Creative Commons ("CC-BY-SA")']], + ['https://creativecommons.org/licenses/by/4.0/', ['Creative Commons ("creativecommons.org/licenses/")']], + ['Attribution-NonCommercial-ShareAlike', ['Creative Commons ("Attribution-NonCommercial-ShareAlike")']], + ['Elastic-2.0', ['Elastic ("Elastic-2.0")']], + ['the elastic license', ['Elastic ("elastic license")']], + ['"Commons Clause" License Condition v1.0', ['Commons Clause ("Commons Clause")']], + ['https://opensource.org/licenses/GPL-3.0', ['GPL, LGPL or AGPL ("GPL-3.0")', 'a license that is not allowed ("opensource.org/licenses/GPL-3.0")']], + ['https://opensource.org/licenses/Artistic-2.0', ['a license that is not allowed ("opensource.org/licenses/Artistic-2.0")']], +]; + +test('a file naming a flagged license is found, by name, id or URL', () => { + for (const [text, want] of flagged) assert.deepEqual(textFindings(text), want, text); +}); + +test('a file naming allowed licenses only is clean', () => { + const clean = [ + 'MIT License\n\nPermission is hereby granted, free of charge, to any person obtaining a copy', + 'Licensed under the Apache License, Version 2.0 (the "License"); http://www.apache.org/licenses/LICENSE-2.0', + 'https://developer.mozilla.org/en-US/docs/Web/JavaScript and https://www.gnu.org/software/libc/', + 'https://creativecommons.org/publicdomain/zero/1.0/ and http://www.unicode.org/reports/tr44/', + 'const GPLAN = 1; mpl2(); // GPLx is not an id', + ]; + for (const t of clean) assert.deepEqual(textFindings(t), [], t); + const pages = ['MIT', 'mit-license.php', 'BSD-2-Clause', 'BSD-3-Clause', 'bsd-license.php', '0BSD', 'Apache-2.0', 'ISC', 'isc-license.txt', 'Zlib', 'zlib-license.php', 'BSL-1.0', 'Unlicense', 'PostgreSQL', 'OFL-1.1', 'MIT.html', 'mit-license.html.']; + for (const p of pages) { + const want = p === 'isc-license.txt' ? ['a license that is not allowed ("opensource.org/licenses/isc-license.txt")'] : []; + assert.deepEqual(textFindings(`see https://opensource.org/licenses/${p}`), want, p); + } +}); + +// What JSON.parse says of text that is not JSON, in this Node. +function parseError(text) { + try { + JSON.parse(text); + } catch (err) { + return err.message; + } + return ''; +} + +// A tree as npm installs it, written under a new directory. +function tree(files) { + const root = mkdtempSync(join(tmpdir(), 'license-gate-row-')); + for (const [path, body] of Object.entries(files)) { + mkdirSync(dirname(join(root, path)), { recursive: true }); + writeFileSync(join(root, path), typeof body === 'string' ? body : JSON.stringify(body)); + } + return root; +} + +test('the walk reads every file under node_modules and every package folder, scoped and nested', (t) => { + const root = tree({ + 'package.json': { name: 'license-gate', license: 'GPL-3.0' }, // the project itself is not shipped + 'node_modules/.package-lock.json': { name: 'license-gate', lockfileVersion: 3 }, + 'node_modules/.cache/x/data.json': '{}', // a dot folder is read, and is not a package + 'node_modules/.cache/package.json': { name: 'cache' }, + 'node_modules/stray.txt': 'Mozilla Public License', // a file outside any package is read too + 'node_modules/a/package.json': { name: 'a', version: '1.0.0', license: 'MIT' }, + 'node_modules/a/index.js': 'module.exports = 1;', + 'node_modules/a/@x/y/z.js': '// a folder named @x inside a package is not a scope', + 'node_modules/a/@x/y/package.json': { name: 'y' }, + 'node_modules/@s/stray': 'SSPL', + 'node_modules/@s/b/package.json': { name: '@s/b', version: '2.0.0', license: 'Apache-2.0' }, + 'node_modules/@s/b/lib/x.js': '// fine', + // A package.json inside a package (as module-type markers are) is a file + // of it, not a package. + 'node_modules/@s/b/lib/package.json': { type: 'commonjs' }, + 'node_modules/@s/b/node_modules/c/package.json': { name: 'c', version: '3.0.0', license: 'LGPL-3.0' }, + 'node_modules/@s/b/node_modules/c/NOTICE': 'Mozilla Public License', + 'node_modules/@s/b/node_modules/@t/e2/package.json': { name: '@t/e2', version: '5.0.0', license: 'BUSL-1.1' }, + 'node_modules/d/package.json': { name: 'd', version: '4.0.0', license: 'ISC' }, + 'node_modules/d/ThirdPartyNoticeText.txt': 'UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE', + 'node_modules/e/index.js': 'no package.json here, so not a package', + 'node_modules/a/test/fixture/node_modules/util/index.js': '// a fixture folder, not a package', + 'node_modules/f/package.json': 'not json{', + }); + t.after(() => rmSync(root, { recursive: true, force: true })); + // npm's bin links are not read (the files they name are, where they are); + // any other link is a finding. + mkdirSync(join(root, 'node_modules/.bin')); + symlinkSync('../a/index.js', join(root, 'node_modules/.bin/tool')); + symlinkSync('../a/index.js', join(root, 'node_modules/d/linked')); + const { findings, packages, files } = check(root); + assert.equal(packages, 6); + assert.equal(files, 20); + assert.deepEqual(findings.sort(), [ + 'node_modules/stray.txt: MPL ("Mozilla Public License")', + 'node_modules/@s/stray: SSPL ("SSPL")', + 'node_modules/@s/b/node_modules/c (c@3.0.0): license "LGPL-3.0" is not allowed (field "LGPL-3.0")', + 'node_modules/@s/b/node_modules/c/NOTICE: MPL ("Mozilla Public License")', + 'node_modules/@s/b/node_modules/c/package.json: GPL, LGPL or AGPL ("LGPL-3.0")', + 'node_modules/@s/b/node_modules/@t/e2 (@t/e2@5.0.0): license "BUSL-1.1" is not allowed (field "BUSL-1.1")', + 'node_modules/@s/b/node_modules/@t/e2/package.json: BUSL ("BUSL")', + 'node_modules/d/ThirdPartyNoticeText.txt: Unicode terms ("UNICODE, INC. LICENSE AGREEMENT")', + 'node_modules/d/linked: not a file or a folder, so not read', + `node_modules/f: no readable package.json (${parseError('not json{')})`, + ].sort()); +}); + +test('a clean tree has no findings, and an empty project has no packages', (t) => { + const root = tree({ + 'node_modules/a/package.json': { name: 'a', version: '1.0.0', license: 'MIT' }, + 'node_modules/a/LICENSE': 'MIT License\n\nPermission is hereby granted', + }); + const empty = tree({ 'package.json': { name: 'license-gate' } }); + t.after(() => { + rmSync(root, { recursive: true, force: true }); + rmSync(empty, { recursive: true, force: true }); + }); + assert.deepEqual(check(root), { findings: [], packages: 1, files: 2 }); + assert.deepEqual(check(empty), { findings: [], packages: 0, files: 0 }); +}); diff --git a/test/release.test.mjs b/test/release.test.mjs new file mode 100644 index 0000000..8a3dc9b --- /dev/null +++ b/test/release.test.mjs @@ -0,0 +1,41 @@ +// Rows for the publish workflow's version check (scripts/release-version.mjs). +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { readFileSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; +import { refusal } from '../scripts/release-version.mjs'; + +const script = fileURLToPath(new URL('../scripts/release-version.mjs', import.meta.url)); +const { version } = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')); +const run = (...args) => spawnSync(process.execPath, [script, ...args], { encoding: 'utf8' }); + +test('a tag naming the version publishes', () => { + assert.equal(refusal('sdk-v1.2.3', '1.2.3'), ''); +}); + +test('a tag naming another version is refused, and names the right tag', () => { + assert.match(refusal('sdk-v1.2.4', '1.2.3'), /does not name package\.json's version 1\.2\.3: the tag for it is sdk-v1\.2\.3$/); + assert.match(refusal('sdk-v1.2.3.1', '1.2.3'), /does not name/); + assert.match(refusal('v1.2.3', '1.2.3'), /does not name/); + assert.match(refusal('', '1.2.3'), /the tag \(none\) does not name/); +}); + +test('a version that is not x.y.z is refused, even with its own tag', () => { + assert.match(refusal('sdk-v1.2.3-rc.1', '1.2.3-rc.1'), /is not a release \(x\.y\.z\)/); + assert.match(refusal('sdk-v1.2', '1.2'), /is not a release/); + // Four numbers end in x.y.z and start with it: the check matches the whole version. + assert.match(refusal('sdk-v1.2.3.4', '1.2.3.4'), /is not a release/); +}); + +test('the script reads package.json and exits 1 with the reason on a wrong tag', () => { + const ok = run(`sdk-v${version}`); + assert.equal(ok.status, 0, ok.stderr); + assert.equal(ok.stdout + ok.stderr, ''); + const bad = run('sdk-v0.0.0-not-this'); + assert.equal(bad.status, 1); + assert.match(bad.stderr, new RegExp(`^refusing to publish: the tag sdk-v0\\.0\\.0-not-this does not name package\\.json's version ${version.replace(/\./g, '\\.')}`)); + const none = run(); + assert.equal(none.status, 1); + assert.match(none.stderr, /the tag \(none\)/); +}); From 6637f8dc4b16eed70e8c1c1adb741534815a56da Mon Sep 17 00:00:00 2001 From: slayerjain Date: Wed, 4 Nov 2026 16:47:32 +0530 Subject: [PATCH 2/9] fix: match licenses in any spelling; gate jest and playwright publishes The license check missed spellings a scanner flags ("CC BY 4.0", "MPL 2.0", an lgpl-3.0 page on choosealicense.com, opensource.org's /license/ URLs), and nothing pinned that it reads every file whole, or installs optional and peer dependencies. It now: - matches text normalised: lower case, any run of spaces, hyphens, underscores and dots as one separator, each license written once as its ids, names and URLs. License pages on opensource.org (/licenses/ and /license/), choosealicense.com and spdx.org must name an allowed license, and so must every SPDX-License-Identifier line. - reads every installed file whole, a block at a time with a bounded window, whatever its name or size; gzip data decompressed, UTF-16 by its byte order mark. - installs the tarball for linux x64 and arm64, darwin arm64 and x64 and win32 x64 (npm's --os and --cpu), so a dependency of another platform is checked too. - leaves out what only the package's own peerDependencies install: the user's jest or @playwright/test, which the package does not ship. The peers of its dependencies are checked. - states its limits in its header: a license it does not name passes when only file text names it; musl-only optional dependencies are not installed, since npm checks the runner's real C library. playwright-publish.yml and jest-publish.yml now build, pack once, check that tarball and publish it, as sdk-publish.yml does. Every publish job has a 30 minute timeout, and test/workflows.test.mjs holds the three workflows to that. The dedup entry points require js-yaml and body-parser, which were not declared, so they failed with MODULE_NOT_FOUND unless the app had them. Both are dependencies now: js-yaml ^3.15.2 (js-yaml 4 needs argparse 2, licensed Python-2.0, outside the allow-list) and body-parser ^1.20.9. test/package.test.mjs requires each entry point from a clean install. Signed-off-by: slayerjain --- .github/workflows/jest-publish.yml | 63 +++- .github/workflows/playwright-publish.yml | 63 +++- .github/workflows/sdk-publish.yml | 8 +- package-lock.json | 314 ++++++++++++++++ package.json | 2 + scripts/license-gate.mjs | 460 +++++++++++++++++------ test/license-gate-cli.test.mjs | 61 ++- test/license-gate.test.mjs | 312 ++++++++++----- test/package.test.mjs | 50 +++ test/workflows.test.mjs | 56 +++ 10 files changed, 1161 insertions(+), 228 deletions(-) create mode 100644 test/package.test.mjs create mode 100644 test/workflows.test.mjs diff --git a/.github/workflows/jest-publish.yml b/.github/workflows/jest-publish.yml index 8fd0ca7..5007ff8 100644 --- a/.github/workflows/jest-publish.yml +++ b/.github/workflows/jest-publish.yml @@ -17,14 +17,23 @@ name: "@keploy/jest publish" # try to publish that package under this tag's name. jest/README.md, # "Releasing", says the same. # -# On a pull request the same package is published as a dry run, with no -# token, so a package that would ship without its build shows before a tag. +# The package is built and packed once, and that tarball is checked and then +# published, so what ships is what was checked. The license check +# (scripts/license-gate.mjs, as in sdk-publish.yml) installs the tarball as a +# user's `npm install` does and refuses it when anything it installs names a +# license outside its allow-list. test/workflows.test.mjs holds this +# workflow to that. +# +# On a pull request the same tarball is packed, checked and published as a +# dry run, with no token, so a package that would ship without its build, or +# with a license it may not carry, shows before a tag. on: push: tags: ["jest-v*"] pull_request: paths: - "jest/**" + - "scripts/license-gate.mjs" - ".github/workflows/jest-publish.yml" - ".github/workflows/jest-sdk.yml" @@ -41,6 +50,7 @@ jobs: if: github.event_name == 'push' needs: rows runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read id-token: write # npm --provenance signs the package with this run's identity @@ -71,6 +81,26 @@ jobs: got=$(npm config get @keploy:registry) test "$got" = https://registry.npmjs.org/ || { echo "::error::@keploy:registry is '$got', not https://registry.npmjs.org/; refusing to publish"; exit 1; } - run: npm ci + - name: Build + run: npm run build + - name: Pack + id: pack + run: | + name=$(npm pack --pack-destination "$RUNNER_TEMP" | tail -n 1) + test -f "$RUNNER_TEMP/$name" || { echo "::error::npm pack wrote no $name"; exit 1; } + echo "tarball=$RUNNER_TEMP/$name" >> "$GITHUB_OUTPUT" + - name: The package carries its build + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: | + tar -tzf "$TARBALL" > "$RUNNER_TEMP/files.txt" + for f in dist/index.js dist/index.d.ts package.json README.md; do + grep -qx "package/$f" "$RUNNER_TEMP/files.txt" || { echo "::error::the package would ship without $f"; exit 1; } + done + - name: Licenses + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: node ../scripts/license-gate.mjs "$TARBALL" - name: The token authenticates # Fails here, named, on a token that is expired, revoked or pasted # with a stray character, before anything is uploaded. @@ -78,15 +108,16 @@ jobs: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} run: npm whoami --registry https://registry.npmjs.org/ - name: Publish - # prepublishOnly builds dist/ first. env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: npm publish --access public --provenance + TARBALL: ${{ steps.pack.outputs.tarball }} + run: npm publish "$TARBALL" --access public --provenance dry-run: name: publish dry run if: github.event_name == 'pull_request' runs-on: ubuntu-latest + timeout-minutes: 30 defaults: run: shell: bash # -eo pipefail: a failed command before a pipe fails the step @@ -97,11 +128,27 @@ jobs: with: node-version: 22 - run: npm ci - - name: Publish, dry run - run: npm publish --dry-run --access public 2>&1 | tee "$RUNNER_TEMP/publish.log" + - name: Build + run: npm run build + - name: Pack + id: pack + run: | + name=$(npm pack --pack-destination "$RUNNER_TEMP" | tail -n 1) + test -f "$RUNNER_TEMP/$name" || { echo "::error::npm pack wrote no $name"; exit 1; } + echo "tarball=$RUNNER_TEMP/$name" >> "$GITHUB_OUTPUT" - name: The package carries its build - # A fresh checkout has no dist/; only prepublishOnly puts it there. + env: + TARBALL: ${{ steps.pack.outputs.tarball }} run: | + tar -tzf "$TARBALL" > "$RUNNER_TEMP/files.txt" for f in dist/index.js dist/index.d.ts package.json README.md; do - grep -qE "npm notice [0-9.]+[kMG]?B +$f\$" "$RUNNER_TEMP/publish.log" || { echo "::error::the package would ship without $f"; exit 1; } + grep -qx "package/$f" "$RUNNER_TEMP/files.txt" || { echo "::error::the package would ship without $f"; exit 1; } done + - name: Licenses + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: node ../scripts/license-gate.mjs "$TARBALL" + - name: Publish, dry run + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: npm publish "$TARBALL" --dry-run --access public diff --git a/.github/workflows/playwright-publish.yml b/.github/workflows/playwright-publish.yml index ee8e073..efd0165 100644 --- a/.github/workflows/playwright-publish.yml +++ b/.github/workflows/playwright-publish.yml @@ -15,14 +15,23 @@ name: "@keploy/playwright publish" # try to publish that package under this tag's name. playwright/README.md, # "Releasing", says the same. # -# On a pull request the same package is published as a dry run, with no -# token, so a package that would ship without its build shows before a tag. +# The package is built and packed once, and that tarball is checked and then +# published, so what ships is what was checked. The license check +# (scripts/license-gate.mjs, as in sdk-publish.yml) installs the tarball as a +# user's `npm install` does and refuses it when anything it installs names a +# license outside its allow-list. test/workflows.test.mjs holds this +# workflow to that. +# +# On a pull request the same tarball is packed, checked and published as a +# dry run, with no token, so a package that would ship without its build, or +# with a license it may not carry, shows before a tag. on: push: tags: ["playwright-v*"] pull_request: paths: - "playwright/**" + - "scripts/license-gate.mjs" - ".github/workflows/playwright-publish.yml" - ".github/workflows/playwright-sdk.yml" @@ -39,6 +48,7 @@ jobs: if: github.event_name == 'push' needs: rows runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read id-token: write # npm --provenance signs the package with this run's identity @@ -69,6 +79,26 @@ jobs: got=$(npm config get @keploy:registry) test "$got" = https://registry.npmjs.org/ || { echo "::error::@keploy:registry is '$got', not https://registry.npmjs.org/; refusing to publish"; exit 1; } - run: npm ci + - name: Build + run: npm run build + - name: Pack + id: pack + run: | + name=$(npm pack --pack-destination "$RUNNER_TEMP" | tail -n 1) + test -f "$RUNNER_TEMP/$name" || { echo "::error::npm pack wrote no $name"; exit 1; } + echo "tarball=$RUNNER_TEMP/$name" >> "$GITHUB_OUTPUT" + - name: The package carries its build + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: | + tar -tzf "$TARBALL" > "$RUNNER_TEMP/files.txt" + for f in dist/index.js dist/index.d.ts package.json README.md; do + grep -qx "package/$f" "$RUNNER_TEMP/files.txt" || { echo "::error::the package would ship without $f"; exit 1; } + done + - name: Licenses + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: node ../scripts/license-gate.mjs "$TARBALL" - name: The token authenticates # Fails here, named, on a token that is expired, revoked or pasted # with a stray character, before anything is uploaded. @@ -76,15 +106,16 @@ jobs: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} run: npm whoami --registry https://registry.npmjs.org/ - name: Publish - # prepublishOnly builds dist/ first. env: NODE_AUTH_TOKEN: ${{ secrets.NPM_TOKEN }} - run: npm publish --access public --provenance + TARBALL: ${{ steps.pack.outputs.tarball }} + run: npm publish "$TARBALL" --access public --provenance dry-run: name: publish dry run if: github.event_name == 'pull_request' runs-on: ubuntu-latest + timeout-minutes: 30 defaults: run: shell: bash # -eo pipefail: a failed command before a pipe fails the step @@ -95,11 +126,27 @@ jobs: with: node-version: 22 - run: npm ci - - name: Publish, dry run - run: npm publish --dry-run --access public 2>&1 | tee "$RUNNER_TEMP/publish.log" + - name: Build + run: npm run build + - name: Pack + id: pack + run: | + name=$(npm pack --pack-destination "$RUNNER_TEMP" | tail -n 1) + test -f "$RUNNER_TEMP/$name" || { echo "::error::npm pack wrote no $name"; exit 1; } + echo "tarball=$RUNNER_TEMP/$name" >> "$GITHUB_OUTPUT" - name: The package carries its build - # A fresh checkout has no dist/; only prepublishOnly puts it there. + env: + TARBALL: ${{ steps.pack.outputs.tarball }} run: | + tar -tzf "$TARBALL" > "$RUNNER_TEMP/files.txt" for f in dist/index.js dist/index.d.ts package.json README.md; do - grep -qE "npm notice [0-9.]+[kMG]?B +$f\$" "$RUNNER_TEMP/publish.log" || { echo "::error::the package would ship without $f"; exit 1; } + grep -qx "package/$f" "$RUNNER_TEMP/files.txt" || { echo "::error::the package would ship without $f"; exit 1; } done + - name: Licenses + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: node ../scripts/license-gate.mjs "$TARBALL" + - name: Publish, dry run + env: + TARBALL: ${{ steps.pack.outputs.tarball }} + run: npm publish "$TARBALL" --dry-run --access public diff --git a/.github/workflows/sdk-publish.yml b/.github/workflows/sdk-publish.yml index 17225cd..1bb3936 100644 --- a/.github/workflows/sdk-publish.yml +++ b/.github/workflows/sdk-publish.yml @@ -13,8 +13,10 @@ name: "@keploy/sdk publish" # The package is packed once, and that tarball is checked and then published, # so what ships is what was checked. The license check # (scripts/license-gate.mjs) installs the tarball as a user's `npm install` -# does and refuses it when any installed package's license field, or any -# installed file, names a license outside its allow-list. +# does, for each platform users run, and refuses it when any installed +# package's license field, or any installed file, names a license outside +# its allow-list. test/workflows.test.mjs holds every publish workflow to +# this: the tarball the Licenses step checked is the one published. # # Push sdk-vX.Y.Z as a plain git tag, never as a GitHub release: a published # GitHub release starts any workflow of this repository that runs on @@ -43,6 +45,7 @@ jobs: name: publish to npm if: github.event_name == 'push' runs-on: ubuntu-latest + timeout-minutes: 30 permissions: contents: read id-token: write # npm --provenance signs the package with this run's identity @@ -109,6 +112,7 @@ jobs: name: pack and license check if: github.event_name == 'pull_request' runs-on: ubuntu-latest + timeout-minutes: 30 defaults: run: shell: bash # -eo pipefail: a failed command before a pipe fails the step diff --git a/package-lock.json b/package-lock.json index 47097fe..6ea972e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -108,6 +108,14 @@ "@types/send": "*" } }, + "argparse": { + "version": "1.0.10", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", + "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", + "requires": { + "sprintf-js": "~1.0.2" + } + }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -123,6 +131,63 @@ "proxy-from-env": "^1.1.0" } }, + "body-parser": { + "version": "1.20.9", + "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.20.9.tgz", + "integrity": "sha512-FISIQxV5u91Y9gvuFq3q/tA/qTqLJn3AhRcEvyozyhJTh5OBFFtv827AdHU7Rbyct4r+Ba1/99oFa1Y3yuFsdw==", + "requires": { + "bytes": "~3.1.2", + "content-type": "~1.0.5", + "debug": "2.6.9", + "depd": "2.0.0", + "destroy": "~1.2.0", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "on-finished": "~2.4.1", + "qs": "~6.16.0", + "raw-body": "~2.5.3", + "type-is": "~1.6.18", + "unpipe": "~1.0.0" + }, + "dependencies": { + "debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "requires": { + "ms": "2.0.0" + } + }, + "ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + } + } + }, + "bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" + }, + "call-bind-apply-helpers": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind-apply-helpers/-/call-bind-apply-helpers-1.0.2.tgz", + "integrity": "sha512-Sp1ablJ0ivDkSzjcaJdxEunN5/XvksFJ2sMBFfq6x0ryhQV/2b/KwFe21cMpmHtPOSij8K99/wSfoEuTObmuMQ==", + "requires": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2" + } + }, + "call-bound": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/call-bound/-/call-bound-1.0.4.tgz", + "integrity": "sha512-+ys997U96po4Kx/ABpBCqhA9EuxJaQWDQg7295H4hBphv3IZg0boBKuwYpt4YXp6MZ5AmZQnU/tyMTlRpaSejg==", + "requires": { + "call-bind-apply-helpers": "^1.0.2", + "get-intrinsic": "^1.3.0" + } + }, "combined-stream": { "version": "1.0.8", "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz", @@ -131,6 +196,11 @@ "delayed-stream": "~1.0.0" } }, + "content-type": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.5.tgz", + "integrity": "sha512-nTjqfcBFEipKdXCv4YDQWCfmcLZKm81ldF0pAopTvyrFGVbcR6P/VAAd5G7N+0tTr8QqiU0tFadD6FK4NtJwOA==" + }, "cors": { "version": "2.8.5", "resolved": "https://registry.npmjs.org/cors/-/cors-2.8.5.tgz", @@ -153,6 +223,54 @@ "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz", "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==" }, + "depd": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz", + "integrity": "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw==" + }, + "destroy": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/destroy/-/destroy-1.2.0.tgz", + "integrity": "sha512-2sJGJTaXIIaR1w4iJSNoN0hnMY7Gpc/n8D4qSCJw8QqFWXf7cuAgnEHxBpweaVcPevC2l3KpjYCx3NypQQgaJg==" + }, + "dunder-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz", + "integrity": "sha512-KIN/nDJBQRcXw0MLVhZE9iQHmG68qAVIBg9CqmUYjmQIhgij9U5MFvrqkUL5FbtyyzZuOeOt0zdeRe4UY7ct+A==", + "requires": { + "call-bind-apply-helpers": "^1.0.1", + "es-errors": "^1.3.0", + "gopd": "^1.2.0" + } + }, + "ee-first": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", + "integrity": "sha512-WMwm9LhRUo+WUaRN+vRuETqG89IgZphVSNkdFgeb6sS/E4OrDIN7t48CAewSHXc6C8lefD8KKfr5vY61brQlow==" + }, + "es-define-property": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", + "integrity": "sha512-e3nRfgfUZ4rNGL232gUgX06QNyyez04KdjFrF+LTRoOXmrOgFKDg4BCdsjW8EnT69eqdYGmRpJwiPVYNrCaW3g==" + }, + "es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==" + }, + "es-object-atoms": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/es-object-atoms/-/es-object-atoms-1.1.2.tgz", + "integrity": "sha512-HWcBoN6NileqtSydK2FqHbS/LoDd2pqrnQHLyJzBj4kOp/ky2MWMN694xOfkK8/SnUsW2DH7EfyVlydKCsm1Zw==", + "requires": { + "es-errors": "^1.3.0" + } + }, + "esprima": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", + "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" + }, "follow-redirects": { "version": "1.15.6", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", @@ -173,6 +291,42 @@ "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==" }, + "get-intrinsic": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.3.0.tgz", + "integrity": "sha512-9fSjSaos/fRIVIp+xSJlE6lfwhES7LNtKaCBIamHsjr2na1BiABJPo0mOjjz8GJDURarmCPGqaiVg5mfjb98CQ==", + "requires": { + "call-bind-apply-helpers": "^1.0.2", + "es-define-property": "^1.0.1", + "es-errors": "^1.3.0", + "es-object-atoms": "^1.1.1", + "function-bind": "^1.1.2", + "get-proto": "^1.0.1", + "gopd": "^1.2.0", + "has-symbols": "^1.1.0", + "hasown": "^2.0.2", + "math-intrinsics": "^1.1.0" + } + }, + "get-proto": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/get-proto/-/get-proto-1.0.1.tgz", + "integrity": "sha512-sTSfBjoXBp89JvIKIefqw7U2CCebsc74kiY6awiGogKtoSGbgjYE/G/+l9sF3MWFPNc9IcoOC4ODfKHfxFmp0g==", + "requires": { + "dunder-proto": "^1.0.1", + "es-object-atoms": "^1.0.0" + } + }, + "gopd": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.2.0.tgz", + "integrity": "sha512-ZUKRh6/kUFoAiTAtTYPZJ3hw9wNxx+BIBOijnlG9PnrJsCcSjs1wyyD6vJpaYtgnzDrKYRSqf3OO6Rfa93xsRg==" + }, + "has-symbols": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz", + "integrity": "sha512-1cDNdwJ2Jaohmb3sg4OmKaMBwuC48sYni5HUw2DvsC8LjGTLK9h+eb1X6RyuOHe4hT0ULCW68iomhjUoKUqlPQ==" + }, "hasown": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", @@ -181,6 +335,31 @@ "function-bind": "^1.1.2" } }, + "http-errors": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz", + "integrity": "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ==", + "requires": { + "depd": "~2.0.0", + "inherits": "~2.0.4", + "setprototypeof": "~1.2.0", + "statuses": "~2.0.2", + "toidentifier": "~1.0.1" + } + }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, + "inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, "is-core-module": { "version": "2.13.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.13.1.tgz", @@ -189,11 +368,30 @@ "hasown": "^2.0.0" } }, + "js-yaml": { + "version": "3.15.2", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.2.tgz", + "integrity": "sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==", + "requires": { + "argparse": "^1.0.7", + "esprima": "^4.0.0" + } + }, "lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" }, + "math-intrinsics": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz", + "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==" + }, + "media-typer": { + "version": "0.3.0", + "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", + "integrity": "sha512-dq+qelQ9akHpcOl/gUVRTxVIOkAJ1wR3QAvb4RsVjS8oVoFjDGTc679wJYmUmknUF5HwMLOgb5O+a3KxfWapPQ==" + }, "mime-db": { "version": "1.52.0", "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz", @@ -222,6 +420,19 @@ "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", "integrity": "sha512-rJgTQnkUnH1sFw8yT6VSU3zD3sWmu6sZhIseY8VX+GRu3P6F7Fu+JNDoXfklElbLJSnc3FUQHVe4cU5hj+BcUg==" }, + "object-inspect": { + "version": "1.13.4", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.13.4.tgz", + "integrity": "sha512-W67iLl4J2EXEGTbfeHCffrjDfitvLANg0UlX3wFUUSTx92KXRFegMHUVgSqE+wvhAbi4WqjGg9czysTV2Epbew==" + }, + "on-finished": { + "version": "2.4.2", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.2.tgz", + "integrity": "sha512-ATWUv387i+nLD49EiDoOft5scf32Vsdz3ktrQJG9bi7jRFmaXTssvyvj1EmeKVVgvesbqUtvQQvaS2TpTukAZQ==", + "requires": { + "ee-first": "1.1.1" + } + }, "path-parse": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", @@ -232,6 +443,26 @@ "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-1.1.0.tgz", "integrity": "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==" }, + "qs": { + "version": "6.16.2", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.16.2.tgz", + "integrity": "sha512-VYdnB5S7/0IeXzBacUtwNAaM6d7Y/PgYnNC7bJCI6meqA/OgnsQbZMtVW8Dt2D1PHgbHGtYmhSoEIgvt7xZ2ZA==", + "requires": { + "es-define-property": "^1.0.1", + "side-channel": "^1.1.1" + } + }, + "raw-body": { + "version": "2.5.3", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.3.tgz", + "integrity": "sha512-s4VSOf6yN0rvbRZGxs8Om5CWj6seneMwK3oDb4lWDH0UPhWcxwOWw5+qk24bxq87szX1ydrwylIOp2uG1ojUpA==", + "requires": { + "bytes": "~3.1.2", + "http-errors": "~2.0.1", + "iconv-lite": "~0.4.24", + "unpipe": "~1.0.0" + } + }, "require-in-the-middle": { "version": "7.3.0", "resolved": "https://registry.npmjs.org/require-in-the-middle/-/require-in-the-middle-7.3.0.tgz", @@ -252,16 +483,94 @@ "supports-preserve-symlinks-flag": "^1.0.0" } }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, + "setprototypeof": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", + "integrity": "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw==" + }, + "side-channel": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.1.1.tgz", + "integrity": "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ==", + "requires": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4", + "side-channel-list": "^1.0.1", + "side-channel-map": "^1.0.1", + "side-channel-weakmap": "^1.0.2" + } + }, + "side-channel-list": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-list/-/side-channel-list-1.0.1.tgz", + "integrity": "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w==", + "requires": { + "es-errors": "^1.3.0", + "object-inspect": "^1.13.4" + } + }, + "side-channel-map": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/side-channel-map/-/side-channel-map-1.0.1.tgz", + "integrity": "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA==", + "requires": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3" + } + }, + "side-channel-weakmap": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/side-channel-weakmap/-/side-channel-weakmap-1.0.2.tgz", + "integrity": "sha512-WPS/HvHQTYnHisLo9McqBHOJk2FkHO/tlpvldyrnem4aeQp4hai3gythswg6p01oSoTl58rcpiFAjF2br2Ak2A==", + "requires": { + "call-bound": "^1.0.2", + "es-errors": "^1.3.0", + "get-intrinsic": "^1.2.5", + "object-inspect": "^1.13.3", + "side-channel-map": "^1.0.1" + } + }, + "sprintf-js": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", + "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" + }, + "statuses": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", + "integrity": "sha512-DvEy55V3DB7uknRo+4iOGT5fP1slR8wQohVdknigZPMpMstaKJQWhwiYBACJE3Ul2pTnATihhBYnRhZQHGBiRw==" + }, "supports-preserve-symlinks-flag": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/supports-preserve-symlinks-flag/-/supports-preserve-symlinks-flag-1.0.0.tgz", "integrity": "sha512-ot0WnXS9fgdkgIcePe6RHNk1WA8+muPa6cSjeR3V8K27q9BB1rTE3R1p7Hv0z1ZyAc8s6Vvv8DIyWf681MAt0w==" }, + "toidentifier": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", + "integrity": "sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==" + }, "tree-kill": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==" }, + "type-is": { + "version": "1.6.18", + "resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.18.tgz", + "integrity": "sha512-TkRKr9sUTxEH8MdfuCSP7VizJyzRNMjj2J2do2Jr3Kym598JVdEksuzPQCnlFPW4ky9Q+iA+ma9BGm06XQBy8g==", + "requires": { + "media-typer": "0.3.0", + "mime-types": "~2.1.24" + } + }, "typescript": { "version": "5.3.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.3.3.tgz", @@ -273,6 +582,11 @@ "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==" }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha512-pjy2bYhSsufwWlKwPc+l3cN7+wuJlK6uz0YdJEOlQDbl6jo/YlPi4mb8agUkVC8BF7V8NuzeyPNqRksA3hztKQ==" + }, "vary": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", diff --git a/package.json b/package.json index 3d32e7c..c9a5c88 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,9 @@ "@types/express": "^4.17.21", "@types/node": "^20.11.16", "axios": "^1.6.7", + "body-parser": "^1.20.9", "cors": "^2.8.5", + "js-yaml": "^3.15.2", "lodash": "^4.17.21", "require-in-the-middle": "^7.3.0", "tree-kill": "^1.2.2" diff --git a/scripts/license-gate.mjs b/scripts/license-gate.mjs index 086efea..db28ad8 100644 --- a/scripts/license-gate.mjs +++ b/scripts/license-gate.mjs @@ -1,90 +1,250 @@ -// license-gate.mjs TARBALL: the publish workflow's license check. +// license-gate.mjs TARBALL: the publish workflows' license check. // // TARBALL is the file `npm pack` wrote, the one the workflow then publishes. // The check installs it into an empty project the way a user does (`npm -// install`, so no devDependencies), and refuses it when anything that install -// puts on disk names a license outside the allow-list below: +// install`: no devDependencies; optional and peer dependencies in), once for +// each platform in PLATFORMS (npm's --os and --cpu), so that a dependency npm +// installs only on another platform is checked too. It refuses the package +// when anything those installs put on disk for it names a license outside the +// allow-list below: // - each installed package's package.json license field (or the old // `licenses` list) must name allowed SPDX ids only. A missing field, an // id not on the list, "SEE LICENSE IN ..." and UNLICENSED all fail. -// - every file of every installed package is read for the names, SPDX ids -// and URLs of the licenses a scanner flags (LGPL, GPL, AGPL, MPL, Unicode -// terms, SSPL, BUSL, EPL, CDDL, EUPL, OSL, CC-BY-SA and other Creative -// Commons licenses, Elastic, Commons Clause), and for opensource.org -// license URLs that name no allowed id. +// - every file of every installed package is read in full, whatever its +// name, extension or size, a block at a time, for the names, ids and +// URLs of the licenses in FLAGGED, for license pages (opensource.org, +// choosealicense.com, spdx.org) that name no allowed license, and for +// SPDX-License-Identifier lines that name an id not allowed. Text is +// matched normalised (see normalise): in lower case, with any run of +// spaces, hyphens, underscores and dots as one separator, so "CC BY 4.0", +// "CC-BY-4.0" and "cc_by_4.0" are one spelling. A gzip file is read +// decompressed; a file that starts with a UTF-16 byte order mark is read +// as UTF-16, any other as UTF-8. +// What the package's own peerDependencies install is the user's: a package +// that peers on jest is installed beside the user's own jest, and ships none +// of it. Those packages are named in the output and not checked, unless the +// package or one of its dependencies needs them too (peers of a dependency +// are checked: the package is what brings them). // Accepting a flagged file with a notice is not enough, so there is no // exception list: a finding fails the publish. // +// Limits, until licensegate's npm mode replaces this script: +// - a license that FLAGGED does not name passes when only a file's text +// names it (the Artistic License, say); a license field naming it fails. +// - the id Unicode-3.0 is matched only in an SPDX-License-Identifier line +// (js-yaml's code, for one, cites "Unicode 3.0", the standard). +// - a name or id is matched only when what splits it is a separator above: +// one split by markup, a comment marker or an escape, or spelt in other +// than ASCII letters, is not. +// - only gzip is decompressed, one level deep: zip, xz, bzip2 and brotli +// data, and UTF-16 text with no byte order mark, are read as their bytes. +// - npm cannot install for another C library: an optional dependency for +// musl (Alpine) Linux only is not installed, so not checked. +// // Exit 0 and print what was checked when the package is clean; exit 1 and -// list every finding when not, or when the install fails. +// list every finding when not, or when an install fails. import { spawnSync } from 'node:child_process'; -import { existsSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { closeSync, createReadStream, existsSync, mkdtempSync, openSync, readdirSync, readFileSync, readSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { basename, join, relative, resolve } from 'node:path'; +import { basename, dirname, join, relative, resolve } from 'node:path'; +import { pipeline } from 'node:stream'; import { fileURLToPath } from 'node:url'; +import { createGunzip } from 'node:zlib'; export const ALLOWED = new Set([ 'MIT', 'BSD-2-Clause', 'BSD-3-Clause', '0BSD', 'Apache-2.0', 'ISC', 'Zlib', 'BSL-1.0', 'CC0-1.0', 'Unlicense', 'PostgreSQL', 'OFL-1.1', ]); -// opensource.org/licenses/: the pages that name an allowed license. -const OSI_PAGES = new Map([ - ['mit', 'MIT'], ['mit-license', 'MIT'], ['bsd-2-clause', 'BSD-2-Clause'], - ['bsd-3-clause', 'BSD-3-Clause'], ['bsd-license', 'BSD-3-Clause'], ['0bsd', '0BSD'], - ['apache-2.0', 'Apache-2.0'], ['isc', 'ISC'], ['isc-license', 'ISC'], ['zlib', 'Zlib'], - ['zlib-license', 'Zlib'], ['bsl-1.0', 'BSL-1.0'], ['unlicense', 'Unlicense'], - ['postgresql', 'PostgreSQL'], ['ofl-1.1', 'OFL-1.1'], -]); +// The platforms the tarball is installed for: the runner's, and those of +// other users, as npm's --os and --cpu name them. +export const PLATFORMS = [ + ['linux', 'x64'], ['linux', 'arm64'], ['darwin', 'arm64'], ['darwin', 'x64'], ['win32', 'x64'], +]; -// What a scanner flags in a file, by name, SPDX id or URL. The SPDX ids are -// matched case-sensitively, as they are written; names and URLs in any case. -const FLAGGED = [ - ['GPL, LGPL or AGPL', /\b(?:L|A)?GPL(?:v?[123]|-[123]\.[01])?\b/], - ['GPL, LGPL or AGPL', /GNU (?:Lesser |Library |Affero )?General Public Licen[cs]e/i], - ['GPL, LGPL or AGPL', /gnu\.org\/(?:licenses|copyleft)/i], - ['MPL', /\bMPL-?[12]\.[01]\b/], - ['MPL', /Mozilla Public Licen[cs]e/i], - ['MPL', /mozilla\.org\/(?:media\/)?MPL/i], - ['Unicode terms', /\bUnicode-(?:TOU|DFS-20\d\d|3\.0)\b/], - ['Unicode terms', /Unicode,? Inc\.? (?:License Agreement|Terms of Use)/i], - ['Unicode terms', /Unicode Terms of Use/i], - ['Unicode terms', /unicode\.org\/(?:copyright|license|terms_of_use)/i], - ['SSPL', /\bSSPL\b/], - ['SSPL', /Server Side Public Licen[cs]e/i], - ['BUSL', /\bBUSL\b/], - ['BUSL', /Business Source Licen[cs]e/i], - ['EPL', /\bEPL-[12]\.0\b/], - ['EPL', /Eclipse Public Licen[cs]e/i], - ['EPL', /eclipse\.org\/legal/i], - ['CDDL', /\bCDDL\b/], - ['CDDL', /Common Development and Distribution Licen[cs]e/i], - ['EUPL', /\bEUPL\b/], - ['EUPL', /European Union Public Licen[cs]e/i], - ['OSL', /\bOSL-[123]\.[01]\b/], - ['OSL', /Open Software Licen[cs]e/i], - ['Creative Commons', /\bCC-BY(?:-NC)?(?:-ND)?(?:-SA)?\b/], - ['Creative Commons', /creativecommons\.org\/licenses\//i], - ['Creative Commons', /Attribution-(?:NonCommercial-)?ShareAlike/i], - ['Elastic', /\bElastic-2\.0\b/], - ['Elastic', /Elastic Licen[cs]e/i], - ['Commons Clause', /Commons Clause/i], +// Text as it is matched: lower case, with each run of spaces, hyphens (any +// dash), underscores and dots made one space. +export function normalise(text) { + return text.toLowerCase().replace(/[\s\-\u2010-\u2015_.]+/g, ' '); +} + +// Each flagged license, written once, normalised: its ids, its names and its +// URLs. A space in a pattern is an optional separator ("cc by" also matches +// "ccby"), and a slash in a URL may be escaped ("\/", as JSON writes it). +// An id is matched as a word, with any version after it ("gplv3", "mpl 2 0"). +const LICENSES = [ + ['GPL, LGPL or AGPL', { + ids: ['(?:l|a)?gpl'], + names: ['(?:gnu )?(?:(?:lesser|library|affero) )?general public licen[cs]e'], + urls: ['gnu org/(?:licenses|copyleft)'], + }], + ['MPL', { ids: ['mpl'], names: ['mozilla public licen[cs]e'], urls: ['mozilla org/(?:[a-z]{2}(?: [a-z]{2})?/)?(?:media/)?mpl'] }], + ['Unicode terms', { + // Unicode-3.0 is matched only as an SPDX-License-Identifier (below): + // "Unicode 3.0" in text names a version of the standard. + ids: ['unicode (?:tou|dfs)'], + names: ['unicode,? inc (?:licen[cs]e agreement|terms of use)', 'unicode terms of use', 'unicode licen[cs]e (?:agreement|v 3)'], + urls: ['unicode org/(?:copyright|licen[cs]e|terms of use)'], + }], + ['SSPL', { ids: ['sspl'], names: ['server side public licen[cs]e'] }], + ['BUSL', { ids: ['busl', 'bsl 1 1'], names: ['business source licen[cs]e'] }], + ['EPL', { ids: ['epl'], names: ['eclipse public licen[cs]e'], urls: ['eclipse org/legal'] }], + ['CDDL', { ids: ['cddl'], names: ['common development and distribution licen[cs]e'] }], + ['EUPL', { ids: ['eupl'], names: ['european union public licen[cs]e'] }], + ['OSL', { ids: ['osl v? \\d'], names: ['open software licen[cs]e'] }], + ['CPL or CPAL', { ids: ['cpl v? \\d', 'cpal'], names: ['common public (?:attribution )?licen[cs]e'] }], + ['Creative Commons', { + ids: ['cc by(?: (?:nc|nd|sa)){0,3}'], + names: ['creative commons (?:attribution|by|non commercial|share alike|no derivatives|noderivs)', 'attribution (?:non commercial|share alike|no derivatives|noderivs)'], + urls: ['creativecommons org/licenses'], + }], + ['ODbL', { ids: ['odbl'], names: ['open database licen[cs]e'] }], + ['Elastic', { ids: ['elastic 2 0'], names: ['elastic licen[cs]e'] }], + ['Commons Clause', { names: ['commons clause'] }], + ['APSL, NPL, RPL, MS-RL or Sleepycat', { + ids: ['apsl', 'npl v? \\d', 'rpl v? \\d', 'ms rl', 'sleepycat'], + names: ['apple public source licen[cs]e', 'netscape public licen[cs]e', 'reciprocal public licen[cs]e', 'microsoft reciprocal licen[cs]e'], + }], ]; -const OSI_URL = /opensource\.org\/licenses\/([A-Za-z0-9.+-]+)/gi; +const sep = (s) => s.replaceAll(' ', ' ?'); +const VERSION = '(?: ?v)?(?: ?\\d{1,4}){0,3}'; +export const FLAGGED = LICENSES.flatMap(([what, { ids = [], names = [], urls = [] }]) => [ + ...ids.map((p) => [what, new RegExp(`(? [what, new RegExp(sep(p), 'g')]), + ...urls.map((p) => [what, new RegExp(sep(p).replaceAll('/', '\\\\?/'), 'g')]), +]); + +// License pages: what follows the path names the license. The text is matched +// in lower case as it is written, so the page ends where the URL does. +const PAGE_URL = /(?:opensource\.org\\?\/licenses?|choosealicense\.com\\?\/licenses|spdx\.org\\?\/licenses)\\?\/([a-z0-9.+_-]{1,64})/g; +const ALLOWED_PAGES = new Set([...[...ALLOWED].map(normalise), 'mit license', 'bsd license', 'isc license', 'zlib license']); +const pageName = (page) => normalise(page).trim().replace(/ (?:php|html?|txt)$/, ''); + +// An SPDX-License-Identifier line names its ids as a license field does, and +// each must be allowed (ids are matched in any case, as SPDX says). +const SPDX_LINE = /spdx-license-identifier:[ \t]{0,8}([a-z0-9.+:() \t-]{1,80})/g; +const ALLOWED_IDS = new Set([...ALLOWED].map((id) => id.toLowerCase())); +const spdxIds = (expr) => expr.replace(/[()]/g, ' ').split(/[ \t]+/).map((t) => t.replace(/\.+$/, '')).filter((t) => /[a-z0-9]/.test(t) && !/^(?:or|and|with)$/.test(t)); + +// At least the longest text a pattern above matches (a license page URL, at +// up to 95 characters), and the context kept around it, so that text read a +// block at a time is matched as it would be whole. +const LONGEST = 128; +const BEFORE = 8; +const AFTER = 64; +export const KEEP = LONGEST + BEFORE + AFTER; + +// Matches text pushed to it a piece at a time, keeping only the last KEEP +// characters of what came before: memory is bounded by the piece's size. A +// match is taken only where the window holds BEFORE characters before it +// and AFTER after it (or the text's start or end), so that a word cut at a +// window's edge is not read as a shorter one; the next window holds what the +// last one could not take yet. +export class Scanner { + #norm = ''; + #raw = ''; + #normCut = false; + #rawCut = false; + #found = new Map(); + #pages = new Set(); + #ids = new Set(); + + push(text) { + this.#feed(text, false); + } + + end() { + this.#feed('', true); + return this.findings(); + } + + findings() { + return [ + ...[...this.#found].sort(([i], [j]) => i - j).map(([, f]) => f), + ...[...this.#pages].map((u) => `a license that is not allowed (${JSON.stringify(u)})`), + ...[...this.#ids].map((id) => `license ${JSON.stringify(id)} is not allowed (SPDX-License-Identifier)`), + ]; + } + + #feed(text, last) { + const norm = normalise(this.#norm + text); + const raw = this.#raw + text.toLowerCase(); + FLAGGED.forEach(([what, re], i) => { + if (this.#found.has(i)) return; + const m = take(re, norm, this.#normCut, last); + if (m) this.#found.set(i, `${what} (${JSON.stringify(m[0])})`); + }); + for (let m, from = 0; (m = take(PAGE_URL, raw, this.#rawCut, last, from)); from = m.index + m[0].length) { + if (!ALLOWED_PAGES.has(pageName(m[1]))) this.#pages.add(m[0]); + } + for (let m, from = 0; (m = take(SPDX_LINE, raw, this.#rawCut, last, from)); from = m.index + m[0].length) { + for (const id of spdxIds(m[1])) if (!ALLOWED_IDS.has(id)) this.#ids.add(id); + } + this.#normCut ||= norm.length > KEEP; + this.#rawCut ||= raw.length > KEEP; + this.#norm = norm.slice(-KEEP); + this.#raw = raw.slice(-KEEP); + } +} + +// The first match of re in window from from on that the window can judge: +// one that starts BEFORE characters in (when text was cut off before the +// window) and ends AFTER characters before the window's end (unless the text +// ends there), or null. +function take(re, window, cut, last, from = 0) { + re.lastIndex = Math.max(from, cut ? BEFORE : 0); + const m = re.exec(window); + if (!m || (!last && m.index + m[0].length > window.length - AFTER)) return null; + return m; +} // The findings for one file's text, as "what (the text that matched)". export function textFindings(text) { - const found = []; - for (const [what, re] of FLAGGED) { - const m = re.exec(text); - if (m) found.push(`${what} (${JSON.stringify(m[0])})`); + const s = new Scanner(); + s.push(text); + return s.end(); +} + +// What a file's bytes start with: gzip's magic, or a UTF-16 byte order mark. +function encoding(head) { + if (head[0] === 0xff && head[1] === 0xfe) return 'utf-16le'; + if (head[0] === 0xfe && head[1] === 0xff) return 'utf-16be'; + return 'utf-8'; +} + +// The findings for the file at path, read in blocks of block bytes. +export async function fileFindings(path, block = 1 << 20) { + const head = Buffer.alloc(2); + const fd = openSync(path, 'r'); + let n; + try { + n = readSync(fd, head, 0, 2, 0); + } finally { + closeSync(fd); } - for (const m of text.matchAll(OSI_URL)) { - const page = m[1].toLowerCase().replace(/\.$/, '').replace(/\.(?:php|html?)$/, ''); - if (!OSI_PAGES.has(page)) found.push(`a license that is not allowed (${JSON.stringify(m[0])})`); + let bytes = createReadStream(path, { highWaterMark: block }); + if (n === 2 && head[0] === 0x1f && head[1] === 0x8b) bytes = pipeline(bytes, createGunzip({ chunkSize: block }), () => {}); + const s = new Scanner(); + let decoder = null; + let start = Buffer.alloc(0); + try { + for await (const chunk of bytes) { + let piece = chunk; + if (!decoder) { + start = Buffer.concat([start, chunk]); + if (start.length < 2) continue; + decoder = new TextDecoder(encoding(start)); + piece = start; + } + s.push(decoder.decode(piece, { stream: true })); + } + // A file under 2 bytes names no license. + if (decoder) s.push(decoder.decode()); + } catch (err) { + return [`not readable (${err.message})`]; } - return found; + return s.end(); } // The SPDX ids of a license expression: every id must be allowed, whatever @@ -113,93 +273,167 @@ export function licenseFindings(pkg) { return found; } -// The findings for one installed package's package.json, at its folder dir. -function packageFindings(dir, at) { - let pkg; +// A package folder's package.json, or the error reading it. +function readPackage(dir) { try { - pkg = JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')); + return { pkg: JSON.parse(readFileSync(join(dir, 'package.json'), 'utf8')) }; } catch (err) { - return [`${at}: no readable package.json (${err.message})`]; + return { error: err.message }; } - return licenseFindings(pkg).map((f) => `${at} (${pkg.name}@${pkg.version}): ${f}`); } -// The findings for the tree installed in project, and what was checked. -// Every file under its node_modules is read, whichever package it belongs -// to (npm's own files there, such as .package-lock.json, included), and -// every package's package.json: node_modules/NAME and -// node_modules/@SCOPE/NAME, at any depth. A link is not read: npm makes -// links only in .bin folders, to files read where they are, so any other -// link, or anything that is not a file or a folder, is a finding. -export function check(project) { +// The names a package needs, from the fields of its package.json named. +function needs(pkg, fields) { + return fields.flatMap((f) => (pkg && pkg[f] && typeof pkg[f] === 'object' ? Object.keys(pkg[f]) : [])); +} + +// The findings for the tree installed in project for the package named root, +// and what was checked. Every package folder is found (node_modules/NAME and +// node_modules/@SCOPE/NAME, at any depth) and every file under node_modules +// is read, whichever package it belongs to, but for npm's own record of the +// tree (node_modules/.package-lock.json: the package.json files it copies are +// read where they are) and the packages only root's peerDependencies need +// (see the header), which are returned as user (and those peers as peers). +// A link is not read: npm +// makes links only in .bin folders, to files read where they are, so any +// other link, or anything that is not a file or a folder, is a finding. +export async function check(project, root) { const findings = []; - let packages = 0; - let files = 0; - const walk = (dir, modules, scope) => { + const packages = new Map(); + const files = []; + const walk = (dir, modules, scope, owner) => { for (const e of readdirSync(dir, { withFileTypes: true })) { const p = join(dir, e.name); - const at = relative(project, p); if (e.isDirectory()) { if (modules && e.name.startsWith('@')) { - walk(p, false, true); + walk(p, false, true, owner); continue; } // A folder with no package.json is not a package (a package's test // fixtures can sit under a node_modules folder of their own); its // files are read all the same. + let o = owner; if ((scope || (modules && !e.name.startsWith('.'))) && existsSync(join(p, 'package.json'))) { - packages++; - findings.push(...packageFindings(p, at)); + packages.set(p, readPackage(p)); + o = p; } - walk(p, e.name === 'node_modules', false); + walk(p, e.name === 'node_modules', false, o); } else if (e.isFile()) { - files++; - for (const f of textFindings(readFileSync(p, 'latin1'))) findings.push(`${at}: ${f}`); + if (!(modules && e.name === '.package-lock.json')) files.push([p, owner]); } else if (!(e.isSymbolicLink() && basename(dir) === '.bin')) { - findings.push(`${at}: not a file or a folder, so not read`); + findings.push(`${relative(project, p)}: not a file or a folder, so not read`); } } }; const modules = join(project, 'node_modules'); - if (existsSync(modules)) walk(modules, true, false); - return { findings, packages, files }; + if (existsSync(modules)) walk(modules, true, false, null); + + // Where Node finds the package name from the package folder at dir. + const find = (dir, name) => { + for (let d = dir; d.startsWith(project); d = dirname(d)) { + const p = join(d, 'node_modules', name); + if (packages.has(p)) return p; + } + return null; + }; + // The package folders reached from starts by the fields, and by peers but + // for those of the folders in noPeers. + const reach = (starts, noPeers) => { + const seen = new Set(); + const todo = [...starts]; + while (todo.length) { + const d = todo.pop(); + if (seen.has(d)) continue; + seen.add(d); + const fields = noPeers.has(d) ? ['dependencies', 'optionalDependencies'] : ['dependencies', 'optionalDependencies', 'peerDependencies']; + for (const name of needs(packages.get(d).pkg, fields)) { + const p = find(d, name); + if (p) todo.push(p); + } + } + return seen; + }; + const top = join(modules, root); + let user = new Set(); + let peers = []; + if (!packages.has(top)) findings.push(`${root}: not installed`); + else { + const ours = reach([top], new Set([top])); + peers = needs(packages.get(top).pkg, ['peerDependencies']).map((n) => find(top, n)).filter(Boolean); + user = new Set([...reach(peers, new Set())].filter((d) => !ours.has(d))); + } + + let checked = 0; + for (const [dir, { pkg, error }] of packages) { + if (user.has(dir)) continue; + checked++; + const at = relative(project, dir); + if (error) findings.push(`${at}: no readable package.json (${error})`); + else findings.push(...licenseFindings(pkg).map((f) => `${at} (${pkg.name}@${pkg.version}): ${f}`)); + } + let read = 0; + for (const [p, owner] of files) { + if (user.has(owner)) continue; + read++; + for (const f of await fileFindings(p)) findings.push(`${relative(project, p)}: ${f}`); + } + const name = (d) => { + const { pkg } = packages.get(d); + return pkg ? `${pkg.name}@${pkg.version}` : relative(project, d); + }; + return { findings, packages: checked, files: read, user: [...user].map(name).sort(), peers: peers.filter((d) => user.has(d)).map(name).sort() }; } -// Installs tarball into a new empty project, as a user's `npm install` does. -export function install(tarball, project) { +// Installs tarball into the empty project, as a user's `npm install` does on +// platform ([os, cpu]) (npm never installs a dependency's devDependencies; +// no install script runs), and returns the package's name: the one +// dependency npm saved to the project's package.json. Throws, saying why, on +// a failure. +export function install(tarball, project, [os, cpu] = PLATFORMS[0]) { writeFileSync(join(project, 'package.json'), JSON.stringify({ name: 'license-gate', version: '0.0.0', private: true })); - const r = spawnSync('npm', ['install', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', '--no-package-lock', resolve(tarball)], { cwd: project, encoding: 'utf8' }); - if (r.status !== 0) return `npm install ${tarball} failed (${r.error ? r.error.message : `exit ${r.status}`}): ${(r.stderr || '').trim()}`; - return ''; + const args = ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--no-package-lock', `--os=${os}`, `--cpu=${cpu}`, resolve(tarball)]; + const r = spawnSync('npm', args, { cwd: project, encoding: 'utf8' }); + if (r.status !== 0) throw new Error(`npm install ${tarball} for ${os}/${cpu} failed (${r.error ? r.error.message : `exit ${r.status}`}): ${(r.stderr || '').trim()}`); + const names = needs(JSON.parse(readFileSync(join(project, 'package.json'), 'utf8')), ['dependencies']); + if (names.length !== 1) throw new Error(`installing ${tarball} for ${os}/${cpu} saved ${names.length} packages to the project, not 1`); + return names[0]; } -export function main(tarball) { +export async function main(tarball) { if (!existsSync(tarball ?? '')) { console.error(`license gate: no package to check (${tarball || 'no TARBALL given'})`); return 1; } - const project = mkdtempSync(join(tmpdir(), 'license-gate-')); - try { - const failed = install(tarball, project); - if (failed) { - console.error(`license gate: ${failed}`); + // Each finding once, with the platforms it was found on. + const findings = new Map(); + const user = new Set(); + const peers = new Set(); + const counts = []; + for (const platform of PLATFORMS) { + const project = mkdtempSync(join(tmpdir(), 'license-gate-')); + try { + const root = install(tarball, project, platform); + const r = await check(project, root); + for (const f of r.findings) findings.set(f, [...(findings.get(f) || []), platform.join('/')]); + for (const u of r.user) user.add(u); + for (const p of r.peers) peers.add(p); + counts.push(`${platform.join('/')}: ${r.packages} packages, ${r.files} files`); + } catch (err) { + console.error(`license gate: ${err.message}`); return 1; + } finally { + rmSync(project, { recursive: true, force: true }); } - const { findings, packages, files } = check(project); - if (packages === 0) { - console.error(`license gate: installing ${tarball} put no package on disk`); - return 1; - } - if (findings.length) { - console.error(`license gate: ${tarball} would ship ${findings.length} license finding(s) in ${packages} packages:`); - for (const f of findings) console.error(` ${f}`); - return 1; - } - console.log(`license gate: ${tarball}: ${packages} packages, ${files} files, every license allowed`); - return 0; - } finally { - rmSync(project, { recursive: true, force: true }); } + const skipped = user.size ? `; not checked, the user's own: ${user.size} packages its peerDependencies install (${[...peers].sort().join(', ')})` : ''; + if (findings.size) { + console.error(`license gate: ${tarball} would ship ${findings.size} license finding(s):`); + for (const [f, on] of findings) console.error(` ${f}${on.length === PLATFORMS.length ? '' : ` (on ${on.join(', ')})`}`); + console.error(`checked ${counts.join('; ')}${skipped}`); + return 1; + } + console.log(`license gate: ${tarball}: ${counts.join('; ')}; every license allowed${skipped}`); + return 0; } -if (process.argv[1] === fileURLToPath(import.meta.url)) process.exit(main(process.argv[2])); +if (process.argv[1] === fileURLToPath(import.meta.url)) process.exit(await main(process.argv[2])); diff --git a/test/license-gate-cli.test.mjs b/test/license-gate-cli.test.mjs index 697223c..59652a5 100644 --- a/test/license-gate-cli.test.mjs +++ b/test/license-gate-cli.test.mjs @@ -1,6 +1,7 @@ -// Rows for scripts/license-gate.mjs run as the publish workflow runs it: on a -// tarball `npm pack` wrote. Each row packs a small package of its own (no -// dependencies, so npm needs no registry), then runs the script on it. +// Rows for scripts/license-gate.mjs run as the publish workflows run it: on a +// tarball `npm pack` wrote. Each row packs small packages of its own (their +// dependencies are tarballs too, named by path, so npm needs no registry), +// then runs the script on one. import { test } from 'node:test'; import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; @@ -29,16 +30,18 @@ function run(args, env = {}) { const tmp = mkdtempSync(join(work, 'tmp-')); const r = spawnSync(process.execPath, [script, ...args], { encoding: 'utf8', env: { ...process.env, TMPDIR: tmp, ...env } }); // npm keeps caches of its own there (node-compile-cache): only the script's - // project counts. + // projects count. return { ...r, left: readdirSync(tmp).filter((n) => n.startsWith('license-gate-')) }; } -test('a clean package passes, and the project it was installed into is gone', () => { +const quote = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); +const counts = (n) => ['linux/x64', 'linux/arm64', 'darwin/arm64', 'darwin/x64', 'win32/x64'].map((p) => `${p}: ${n} packages, \\d+ files`).join('; '); + +test('a clean package passes on every platform, and the projects it was installed into are gone', () => { const tgz = pack('clean', { 'package.json': { name: 'clean', version: '1.0.0', license: 'Apache-2.0' }, 'index.js': 'module.exports = 1;' }); const r = run([tgz]); assert.equal(r.status, 0, r.stderr); - // The files: the package's two and what npm writes beside them. - assert.match(r.stdout, new RegExp(`^license gate: ${tgz.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}: 1 packages, \\d+ files, every license allowed\n$`)); + assert.match(r.stdout, new RegExp(`^license gate: ${quote(tgz)}: ${counts(1)}; every license allowed\n$`)); assert.deepEqual(r.left, []); }); @@ -47,8 +50,9 @@ test('a package whose license field is not allowed fails, named', () => { const r = run([tgz]); assert.equal(r.status, 1); assert.equal(r.stdout, ''); - assert.match(r.stderr, /would ship \d+ license finding\(s\) in 1 packages:\n/); + assert.match(r.stderr, /would ship \d+ license finding\(s\):\n/); assert.match(r.stderr, /\n {2}node_modules\/red \(red@1\.0\.0\): license "GPL-3\.0" is not allowed \(field "GPL-3\.0"\)\n/); + assert.match(r.stderr, new RegExp(`\nchecked ${counts(1)}\n$`)); assert.deepEqual(r.left, []); }); @@ -59,8 +63,39 @@ test('a package carrying a flagged file fails, with the file named', () => { }); const r = run([tgz]); assert.equal(r.status, 1); - assert.match(r.stderr, /\n {2}node_modules\/notice\/ThirdPartyNoticeText\.txt: Unicode terms \("UNICODE, INC\. LICENSE AGREEMENT"\)\n/); - assert.match(r.stderr, /\n {2}node_modules\/notice\/ThirdPartyNoticeText\.txt: Unicode terms \("unicode\.org\/copyright"\)\n/); + assert.match(r.stderr, /\n {2}node_modules\/notice\/ThirdPartyNoticeText\.txt: Unicode terms \("unicode, inc license agreement"\)\n/); + assert.match(r.stderr, /\n {2}node_modules\/notice\/ThirdPartyNoticeText\.txt: Unicode terms \("unicode org\/copyright"\)\n/); +}); + +test('optional dependencies, those of other platforms, and the peers of dependencies are installed and checked', () => { + const dep = (name, extra) => pack(name, { 'package.json': { name, version: '1.0.0', ...extra } }); + const opt = dep('redopt', { license: 'GPL-3.0' }); + const darwin = dep('reddarwin', { license: 'LGPL-3.0', os: ['darwin'], cpu: ['arm64'] }); + const win = dep('redwin', { license: 'MPL-2.0', os: ['win32'] }); + const peer = dep('redpeer', { license: 'EPL-2.0' }); + const mid = dep('mid', { license: 'MIT', peerDependencies: { redpeer: `file:${peer}` } }); + // The user's own peer needs redpeer too: it is checked all the same. + const own = dep('ownpeer', { license: 'SSPL-1.0', dependencies: { redpeer: `file:${peer}` } }); + const tgz = pack('top', { + 'package.json': { + name: 'top', version: '1.0.0', license: 'MIT', + dependencies: { mid: `file:${mid}` }, + optionalDependencies: { redopt: `file:${opt}`, reddarwin: `file:${darwin}`, redwin: `file:${win}` }, + // The package's own peer is the user's: named, not checked. + peerDependencies: { ownpeer: `file:${own}` }, + }, + }); + const r = run([tgz]); + assert.equal(r.status, 1, r.stderr); + const lines = r.stderr.split('\n').filter((l) => / is not allowed /.test(l)).sort(); + assert.deepEqual(lines, [ + ' node_modules/reddarwin (reddarwin@1.0.0): license "LGPL-3.0" is not allowed (field "LGPL-3.0") (on darwin/arm64)', + ' node_modules/redopt (redopt@1.0.0): license "GPL-3.0" is not allowed (field "GPL-3.0")', + ' node_modules/redpeer (redpeer@1.0.0): license "EPL-2.0" is not allowed (field "EPL-2.0")', + ' node_modules/redwin (redwin@1.0.0): license "MPL-2.0" is not allowed (field "MPL-2.0") (on win32/x64)', + ]); + assert.match(r.stderr, /\nchecked linux\/x64: 4 packages, \d+ files; linux\/arm64: 4 packages, \d+ files; darwin\/arm64: 5 packages, \d+ files; darwin\/x64: 4 packages, \d+ files; win32\/x64: 5 packages, \d+ files; not checked, the user's own: 1 packages its peerDependencies install \(ownpeer@1\.0\.0\)\n$/); + assert.deepEqual(r.left, []); }); test('a file that is not a package fails at the install', () => { @@ -68,7 +103,7 @@ test('a file that is not a package fails at the install', () => { writeFileSync(bad, 'not a tarball'); const r = run([bad]); assert.equal(r.status, 1); - assert.match(r.stderr, /^license gate: npm install .*not-a-package\.tgz failed \(exit \d+\): /); + assert.match(r.stderr, /^license gate: npm install .*not-a-package\.tgz for linux\/x64 failed \(exit \d+\): /); assert.deepEqual(r.left, []); }); @@ -84,10 +119,10 @@ test('no tarball, or one that is not there, fails before any install', () => { test('an install that puts nothing on disk fails: nothing checked is not clean', () => { // npm_config_dry_run, set in a job's environment, makes `npm install` exit - // 0 having installed nothing. + // 0 having installed and saved nothing. const tgz = pack('dry', { 'package.json': { name: 'dry', version: '1.0.0', license: 'MIT' } }); const r = run([tgz], { npm_config_dry_run: 'true' }); assert.equal(r.status, 1); - assert.equal(r.stderr, `license gate: installing ${tgz} put no package on disk\n`); + assert.equal(r.stderr, `license gate: installing ${tgz} for linux/x64 saved 0 packages to the project, not 1\n`); assert.deepEqual(r.left, []); }); diff --git a/test/license-gate.test.mjs b/test/license-gate.test.mjs index 92b5042..0ddcd83 100644 --- a/test/license-gate.test.mjs +++ b/test/license-gate.test.mjs @@ -1,12 +1,14 @@ -// Rows for the publish workflow's license check (scripts/license-gate.mjs): -// its classifier and its walk of an installed tree. The rows that run the -// script on packed tarballs are in license-gate-cli.test.mjs. +// Rows for the publish workflows' license check (scripts/license-gate.mjs): +// its classifier, its reading of files, and its walk of an installed tree. +// The rows that run the script on packed tarballs are in +// license-gate-cli.test.mjs. import { test } from 'node:test'; import assert from 'node:assert/strict'; import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; -import { ALLOWED, check, expressionIds, licenseFindings, textFindings } from '../scripts/license-gate.mjs'; +import { gzipSync } from 'node:zlib'; +import { ALLOWED, FLAGGED, KEEP, PLATFORMS, Scanner, check, expressionIds, fileFindings, licenseFindings, normalise, textFindings } from '../scripts/license-gate.mjs'; test('every allowed SPDX id passes, alone and in an expression', () => { const ids = ['MIT', 'BSD-2-Clause', 'BSD-3-Clause', '0BSD', 'Apache-2.0', 'ISC', 'Zlib', 'BSL-1.0', 'CC0-1.0', 'Unlicense', 'PostgreSQL', 'OFL-1.1']; @@ -33,6 +35,8 @@ test('a license field naming anything else fails, each id on its own', () => { assert.match(licenseFindings({ license: 'SEE LICENSE IN LICENSE.md' }).join('\n'), /license "SEE" is not allowed/); assert.deepEqual(licenseFindings({ license: { type: 'EPL-2.0' } }), ['license "EPL-2.0" is not allowed (field "EPL-2.0")']); assert.deepEqual(licenseFindings({ licenses: [{ type: 'MIT' }, { type: 'CDDL-1.0' }] }), ['license "CDDL-1.0" is not allowed (field "CDDL-1.0")']); + assert.deepEqual(licenseFindings({ licenses: [{ type: 'GPL-2.0' }] }), ['license "GPL-2.0" is not allowed (field "GPL-2.0")']); + assert.deepEqual(licenseFindings({ license: 'WTFPL' }), ['license "WTFPL" is not allowed (field "WTFPL")']); }); test('a missing or empty license field fails', () => { @@ -48,81 +52,197 @@ test('the ids of an expression drop only parentheses and operators', () => { assert.deepEqual(expressionIds('GPL-2.0 WITH Classpath-exception-2.0'), ['GPL-2.0', 'Classpath-exception-2.0']); }); -// Each sample matches one pattern of the check, and the findings are listed -// whole: a pattern taken out of the check changes its row's list. +test('text is matched in lower case, with any run of separators as one space', () => { + assert.equal(normalise('CC BY 4.0'), 'cc by 4 0'); + assert.equal(normalise('CC-BY-4.0'), 'cc by 4 0'); + assert.equal(normalise('cc_by_4.0'), 'cc by 4 0'); + assert.equal(normalise('CC \t- _.\n BY–SA'), 'cc by sa'); + assert.equal(normalise('a/b,c'), 'a/b,c'); +}); + +// Each sample and every finding it gives, listed whole: a pattern taken out +// of the check, or loosened or narrowed, changes some row's list. const flagged = [ - ['Licensed under LGPL-2.1 terms', ['GPL, LGPL or AGPL ("LGPL-2.1")']], - ['AGPLv3 only', ['GPL, LGPL or AGPL ("AGPLv3")']], - ['GPL-3.0 here', ['GPL, LGPL or AGPL ("GPL-3.0")']], - ['the GNU Lesser General Public License applies', ['GPL, LGPL or AGPL ("GNU Lesser General Public License")']], + ['Licensed under LGPL-2.1 terms', ['GPL, LGPL or AGPL ("lgpl 2 1")']], + ['AGPLv3 only', ['GPL, LGPL or AGPL ("agplv3")']], + ['GPL-3.0-or-later here', ['GPL, LGPL or AGPL ("gpl 3 0")']], + ['see https://choosealicense.com/licenses/lgpl-3.0/', ['GPL, LGPL or AGPL ("lgpl 3 0")', 'a license that is not allowed ("choosealicense.com/licenses/lgpl-3.0")']], + ['License: https://opensource.org/license/gpl-3-0', ['GPL, LGPL or AGPL ("gpl 3 0")', 'a license that is not allowed ("opensource.org/license/gpl-3-0")']], + ['the GNU Lesser General Public License applies', ['GPL, LGPL or AGPL ("gnu lesser general public license")']], ['see the gnu affero general public licence', ['GPL, LGPL or AGPL ("gnu affero general public licence")']], - ['', ['GPL, LGPL or AGPL ("gnu.org/licenses")']], - ['https://www.gnu.org/copyleft/', ['GPL, LGPL or AGPL ("gnu.org/copyleft")']], - ['SPDX-License-Identifier: MPL-2.0', ['MPL ("MPL-2.0")']], - ['subject to the Mozilla Public License, v. 2.0', ['MPL ("Mozilla Public License")']], - ['https://www.mozilla.org/media/MPL/2.0/index.txt', ['MPL ("mozilla.org/media/MPL")']], - ['http://mozilla.org/MPL/2.0/', ['MPL ("mozilla.org/MPL")']], - ['License: Unicode-TOU', ['Unicode terms ("Unicode-TOU")']], - ['Unicode-DFS-2016', ['Unicode terms ("Unicode-DFS-2016")']], - ['UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE', ['Unicode terms ("UNICODE, INC. LICENSE AGREEMENT")']], - ['Data Files and Software: Unicode Terms of Use apply', ['Unicode terms ("Unicode Terms of Use")']], - ['http://www.unicode.org/copyright.html', ['Unicode terms ("unicode.org/copyright")']], - ['SSPL', ['SSPL ("SSPL")']], - ['the Server Side Public License', ['SSPL ("Server Side Public License")']], - ['BUSL-1.1', ['BUSL ("BUSL")']], - ['Business Source License 1.1', ['BUSL ("Business Source License")']], - ['EPL-2.0', ['EPL ("EPL-2.0")']], - ['Eclipse Public License', ['EPL ("Eclipse Public License")']], - ['https://www.eclipse.org/legal/epl-v20.html', ['EPL ("eclipse.org/legal")']], - ['CDDL', ['CDDL ("CDDL")']], - ['Common Development and Distribution License', ['CDDL ("Common Development and Distribution License")']], - ['EUPL', ['EUPL ("EUPL")']], - ['European Union Public Licence', ['EUPL ("European Union Public Licence")']], - ['OSL-3.0', ['OSL ("OSL-3.0")']], - ['Open Software License', ['OSL ("Open Software License")']], - ['CC-BY-SA', ['Creative Commons ("CC-BY-SA")']], - ['https://creativecommons.org/licenses/by/4.0/', ['Creative Commons ("creativecommons.org/licenses/")']], - ['Attribution-NonCommercial-ShareAlike', ['Creative Commons ("Attribution-NonCommercial-ShareAlike")']], - ['Elastic-2.0', ['Elastic ("Elastic-2.0")']], + ['GNU Library General Public License', ['GPL, LGPL or AGPL ("gnu library general public license")']], + ['under the General_Public_License', ['GPL, LGPL or AGPL ("general public license")']], + ['', ['GPL, LGPL or AGPL ("gnu org/licenses")']], + ['https://www.gnu.org/copyleft/', ['GPL, LGPL or AGPL ("gnu org/copyleft")']], + ['"https:\\/\\/www.GNU.org\\/licenses\\/"', ['GPL, LGPL or AGPL ("gnu org\\\\/licenses")']], + ['SPDX-License-Identifier: MPL-2.0', ['MPL ("mpl 2 0")', 'license "mpl-2.0" is not allowed (SPDX-License-Identifier)']], + ['Portions are licensed under MPL 2.0.', ['MPL ("mpl 2 0")']], + ['MPL2.0', ['MPL ("mpl2 0")']], + ['subject to the Mozilla Public License, v. 2.0', ['MPL ("mozilla public license")']], + ['Mozilla Public Licence', ['MPL ("mozilla public licence")']], + ['https://www.mozilla.org/media/MPL/2.0/index.txt', ['MPL ("mpl")', 'MPL ("mozilla org/media/mpl")']], + ['http://mozilla.org/MPL/2.0/', ['MPL ("mpl")', 'MPL ("mozilla org/mpl")']], + ['https://www.mozilla.org/en-US/MPL/', ['MPL ("mpl")', 'MPL ("mozilla org/en us/mpl")']], + ['License: Unicode-TOU', ['Unicode terms ("unicode tou")']], + ['Unicode-DFS-2016', ['Unicode terms ("unicode dfs 2016")']], + ['/* SPDX-License-Identifier: Unicode-3.0 */', ['license "unicode-3.0" is not allowed (SPDX-License-Identifier)']], + ['// spdx-license-identifier:\t(MIT OR Artistic-2.0).', ['license "artistic-2.0" is not allowed (SPDX-License-Identifier)']], + ['# SPDX-License-Identifier: BSD-3-Clause\nSPDX-License-Identifier: WTFPL AND ISC', ['license "wtfpl" is not allowed (SPDX-License-Identifier)']], + ['UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE', ['Unicode terms ("unicode, inc license agreement")']], + ['Unicode, Inc. Terms of Use', ['Unicode terms ("unicode, inc terms of use")']], + ['Data Files and Software: Unicode Terms of Use apply', ['Unicode terms ("unicode terms of use")']], + ['UNICODE LICENSE V3', ['Unicode terms ("unicode license v3")']], + ['the Unicode License Agreement', ['Unicode terms ("unicode license agreement")']], + ['http://www.unicode.org/copyright.html', ['Unicode terms ("unicode org/copyright")']], + ['https://www.unicode.org/license.txt', ['Unicode terms ("unicode org/license")']], + ['http://www.unicode.org/terms_of_use.html', ['Unicode terms ("unicode org/terms of use")']], + ['SSPL-1.0', ['SSPL ("sspl 1 0")']], + ['the Server Side Public License', ['SSPL ("server side public license")']], + ['BUSL-1.1', ['BUSL ("busl 1 1")']], + ['MariaDB BSL 1.1', ['BUSL ("bsl 1 1")']], + ['Business Source License 1.1', ['BUSL ("business source license")']], + ['EPL-2.0', ['EPL ("epl 2 0")']], + ['Eclipse Public License', ['EPL ("eclipse public license")']], + ['https://www.eclipse.org/legal/epl-v20.html', ['EPL ("epl v20")', 'EPL ("eclipse org/legal")']], + ['CDDL-1.0', ['CDDL ("cddl 1 0")']], + ['Common Development and Distribution License', ['CDDL ("common development and distribution license")']], + ['EUPL-1.2', ['EUPL ("eupl 1 2")']], + ['European Union Public Licence', ['EUPL ("european union public licence")']], + ['OSL-3.0', ['OSL ("osl 3 0")']], + ['Open Software License', ['OSL ("open software license")']], + ['CPL-1.0', ['CPL or CPAL ("cpl 1 0")']], + ['CPAL-1.0', ['CPL or CPAL ("cpal 1 0")']], + ['Common Public Attribution License', ['CPL or CPAL ("common public attribution license")']], + ['CC-BY-4.0', ['Creative Commons ("cc by 4 0")']], + ['This data is licensed CC BY 4.0.', ['Creative Commons ("cc by 4 0")']], + ['CC BY-SA 4.0', ['Creative Commons ("cc by sa 4 0")']], + ['cc_by_nc_nd_4.0', ['Creative Commons ("cc by nc nd 4 0")']], + ['Creative Commons Attribution 4.0 International', ['Creative Commons ("creative commons attribution")']], + ['Creative Commons BY-SA', ['Creative Commons ("creative commons by")']], + ['Attribution-NonCommercial-ShareAlike', ['Creative Commons ("attribution noncommercial")']], + ['Attribution-ShareAlike 4.0 International', ['Creative Commons ("attribution sharealike")']], + ['https://creativecommons.org/licenses/by/4.0/', ['Creative Commons ("creativecommons org/licenses")']], + ['ODbL-1.0', ['ODbL ("odbl 1 0")']], + ['Open Database License', ['ODbL ("open database license")']], + ['Elastic-2.0', ['Elastic ("elastic 2 0")']], ['the elastic license', ['Elastic ("elastic license")']], - ['"Commons Clause" License Condition v1.0', ['Commons Clause ("Commons Clause")']], - ['https://opensource.org/licenses/GPL-3.0', ['GPL, LGPL or AGPL ("GPL-3.0")', 'a license that is not allowed ("opensource.org/licenses/GPL-3.0")']], - ['https://opensource.org/licenses/Artistic-2.0', ['a license that is not allowed ("opensource.org/licenses/Artistic-2.0")']], + ['"Commons Clause" License Condition v1.0', ['Commons Clause ("commons clause")']], + ['APSL-2.0', ['APSL, NPL, RPL, MS-RL or Sleepycat ("apsl 2 0")']], + ['NPL-1.1', ['APSL, NPL, RPL, MS-RL or Sleepycat ("npl 1 1")']], + ['RPL-1.5', ['APSL, NPL, RPL, MS-RL or Sleepycat ("rpl 1 5")']], + ['MS-RL', ['APSL, NPL, RPL, MS-RL or Sleepycat ("ms rl")']], + ['Sleepycat', ['APSL, NPL, RPL, MS-RL or Sleepycat ("sleepycat")']], + ['Apple Public Source License', ['APSL, NPL, RPL, MS-RL or Sleepycat ("apple public source license")']], + ['Netscape Public License', ['APSL, NPL, RPL, MS-RL or Sleepycat ("netscape public license")']], + ['Reciprocal Public License', ['APSL, NPL, RPL, MS-RL or Sleepycat ("reciprocal public license")']], + ['Microsoft Reciprocal License', ['APSL, NPL, RPL, MS-RL or Sleepycat ("microsoft reciprocal license")']], + ['https://opensource.org/licenses/Artistic-2.0', ['a license that is not allowed ("opensource.org/licenses/artistic-2.0")']], + ['https://spdx.org/licenses/Artistic-2.0.html', ['a license that is not allowed ("spdx.org/licenses/artistic-2.0.html")']], + ['see https://opensource.org/licenses/MIT and https://opensource.org/license/artistic-2-0', ['a license that is not allowed ("opensource.org/license/artistic-2-0")']], + ['"https:\\/\\/opensource.org\\/licenses\\/Artistic-2.0"', ['a license that is not allowed ("opensource.org\\\\/licenses\\\\/artistic-2.0")']], ]; -test('a file naming a flagged license is found, by name, id or URL', () => { +test('a file naming a flagged license is found, by name, id or URL, in any spelling', () => { for (const [text, want] of flagged) assert.deepEqual(textFindings(text), want, text); + // Every pattern of the check is some sample's. + const labels = new Set(flagged.flatMap(([, w]) => w.map((f) => f.replace(/ \(.*$/, '')))); + for (const [what] of FLAGGED) assert.ok(labels.has(what), what); }); +const clean = [ + 'MIT License\n\nPermission is hereby granted, free of charge, to any person obtaining a copy', + 'Licensed under the Apache License, Version 2.0 (the "License"); http://www.apache.org/licenses/LICENSE-2.0', + 'https://developer.mozilla.org/en-US/docs/Web/JavaScript and https://www.gnu.org/software/libc/', + 'https://creativecommons.org/publicdomain/zero/1.0/ and http://www.unicode.org/reports/tr44/ and CC0-1.0', + 'const GPLAN = 1; // GPLx is not an id, nor is cplus or nplx', + 'a sample template example: implement the epilogue; Simpl.js, templ and hepl', + 'BSL-1.0 is the Boost license; see https://opensource.org/licenses', + '// SPDX-License-Identifier: (MIT OR Apache-2.0) */ and and SPDX-License-Identifier: 0BSD.', + '// Check for surrogate pairs (reference Unicode 3.0 section "3.7 Surrogates").', + 'elastic band; Unicode 15 tables; the commons; creative work; attribution required', +]; + test('a file naming allowed licenses only is clean', () => { - const clean = [ - 'MIT License\n\nPermission is hereby granted, free of charge, to any person obtaining a copy', - 'Licensed under the Apache License, Version 2.0 (the "License"); http://www.apache.org/licenses/LICENSE-2.0', - 'https://developer.mozilla.org/en-US/docs/Web/JavaScript and https://www.gnu.org/software/libc/', - 'https://creativecommons.org/publicdomain/zero/1.0/ and http://www.unicode.org/reports/tr44/', - 'const GPLAN = 1; mpl2(); // GPLx is not an id', - ]; for (const t of clean) assert.deepEqual(textFindings(t), [], t); - const pages = ['MIT', 'mit-license.php', 'BSD-2-Clause', 'BSD-3-Clause', 'bsd-license.php', '0BSD', 'Apache-2.0', 'ISC', 'isc-license.txt', 'Zlib', 'zlib-license.php', 'BSL-1.0', 'Unlicense', 'PostgreSQL', 'OFL-1.1', 'MIT.html', 'mit-license.html.']; - for (const p of pages) { - const want = p === 'isc-license.txt' ? ['a license that is not allowed ("opensource.org/licenses/isc-license.txt")'] : []; - assert.deepEqual(textFindings(`see https://opensource.org/licenses/${p}`), want, p); + const pages = ['MIT', 'mit-license.php', 'BSD-2-Clause', 'BSD-3-Clause', 'bsd-license.php', '0BSD', 'Apache-2.0', 'apache-2-0', 'ISC', 'isc-license.txt', 'isc-license-txt', 'Zlib', 'zlib-license.php', 'BSL-1.0', 'bsl-1-0', 'CC0-1.0', 'Unlicense', 'PostgreSQL', 'OFL-1.1', 'MIT.html', 'mit-license.html.', 'MIT.htm']; + for (const host of ['https://opensource.org/licenses/', 'https://opensource.org/license/', 'https://choosealicense.com/licenses/', 'https://spdx.org/licenses/', 'HTTPS://OPENSOURCE.ORG/LICENSES/']) { + for (const p of pages) assert.deepEqual(textFindings(`see ${host}${p} for the terms`), [], host + p); } }); -// What JSON.parse says of text that is not JSON, in this Node. -function parseError(text) { - try { - JSON.parse(text); - } catch (err) { - return err.message; - } - return ''; +// The findings for text pushed to a Scanner in pieces of size n. +function inPieces(text, n) { + const s = new Scanner(); + for (let i = 0; i < text.length; i += n) s.push(text.slice(i, i + n)); + return s.end(); } +test('text read a piece at a time is matched as it would be whole', () => { + for (const [text] of [...flagged, ...clean.map((t) => [t])]) { + for (const n of [1, 2, 7]) assert.deepEqual(inPieces(`${'x '.repeat(KEEP)}${text}`, n), textFindings(text), `${n}: ${text}`); + } + // A match far into a long text, cut across pieces, is found. + const long = `${'x'.repeat(5000)} Mozilla Public License ${'y'.repeat(5000)}`; + assert.deepEqual(inPieces(long, 100), ['MPL ("mozilla public license")']); + assert.deepEqual(inPieces(long, 5011), ['MPL ("mozilla public license")']); + // A match at the very start, before a long run of separators, is found. + assert.deepEqual(inPieces(`GPL${' '.repeat(5000)}x`, 1000), ['GPL, LGPL or AGPL ("gpl")']); + // A word cut at the end of a piece is not read as a shorter one. + const s = new Scanner(); + s.push(`${'x'.repeat(5000)} gpl`); + s.push('an and https://opensource.org/licenses/mi'); + s.push('t-license.php'); + assert.deepEqual(s.end(), []); + // A long license URL cut across pieces is found whole. + const page = `choosealicense.com/licenses/${'a'.repeat(64)}`; + const u = new Scanner(); + u.push(`${'x'.repeat(5000)} https://${page}`); + u.push(' and more'); + assert.deepEqual(u.end(), [`a license that is not allowed (${JSON.stringify(page)})`]); + // Nor is a word whose start was cut off before the window. + const t = new Scanner(); + t.push(`${'b'.repeat(1000)}agpl ${'c'.repeat(KEEP - 4)}`); + t.push('d'); + assert.deepEqual(t.end(), []); +}); + +const work = mkdtempSync(join(tmpdir(), 'license-gate-row-')); +test.after(() => rmSync(work, { recursive: true, force: true })); +let nfile = 0; +const file = (body) => { + const p = join(work, `f${nfile++}`); + writeFileSync(p, body); + return p; +}; + +test('every file is read whole, whatever its name or size, a block at a time', async () => { + const big = `${'x'.repeat(3 << 20)}\n/* Mozilla Public License 2.0 */\n`; + assert.deepEqual(await fileFindings(file(big)), ['MPL ("mozilla public license")']); + assert.deepEqual(await fileFindings(file(big), 4096), ['MPL ("mozilla public license")']); + // A UTF-8 character split across blocks is read whole. + assert.deepEqual(await fileFindings(file('CC BY 4.0'), 3), ['Creative Commons ("cc by 4 0")']); + assert.deepEqual(await fileFindings(file('')), []); + assert.deepEqual(await fileFindings(file('G')), []); +}); + +test('a gzip file is read decompressed, and UTF-16 text by its byte order mark', async () => { + const gpl = 'GNU GENERAL PUBLIC LICENSE\nVersion 3, 29 June 2007\n'; + const want = ['GPL, LGPL or AGPL ("gnu general public license")']; + assert.deepEqual(await fileFindings(file(gzipSync(Buffer.from(gpl)))), want); + assert.deepEqual(await fileFindings(file(gzipSync(Buffer.from(gpl.repeat(1 << 15)))), 512), want); + const le = Buffer.from(`${gpl}`, 'utf16le'); + assert.deepEqual(await fileFindings(file(le)), want); + assert.deepEqual(await fileFindings(file(le), 1), want); + assert.deepEqual(await fileFindings(file(Buffer.from(le).swap16())), want); + assert.deepEqual(await fileFindings(file(gzipSync(le))), want); + const bad = await fileFindings(file(Buffer.from([0x1f, 0x8b, 1, 2, 3, 4]))); + assert.equal(bad.length, 1); + assert.match(bad[0], /^not readable \(.+\)$/); +}); + // A tree as npm installs it, written under a new directory. function tree(files) { - const root = mkdtempSync(join(tmpdir(), 'license-gate-row-')); + const root = mkdtempSync(join(work, 'tree-')); for (const [path, body] of Object.entries(files)) { mkdirSync(dirname(join(root, path)), { recursive: true }); writeFileSync(join(root, path), typeof body === 'string' ? body : JSON.stringify(body)); @@ -130,24 +250,36 @@ function tree(files) { return root; } -test('the walk reads every file under node_modules and every package folder, scoped and nested', (t) => { +// What JSON.parse says of text that is not JSON, in this Node. +function parseError(text) { + try { + JSON.parse(text); + } catch (err) { + return err.message; + } + return ''; +} + +test('the walk reads every package folder and file but the user\'s own peers, scoped and nested', async () => { const root = tree({ 'package.json': { name: 'license-gate', license: 'GPL-3.0' }, // the project itself is not shipped - 'node_modules/.package-lock.json': { name: 'license-gate', lockfileVersion: 3 }, + 'node_modules/.package-lock.json': { packages: { 'node_modules/q': { license: 'MPL-2.0' } } }, // npm's record, not read 'node_modules/.cache/x/data.json': '{}', // a dot folder is read, and is not a package 'node_modules/.cache/package.json': { name: 'cache' }, 'node_modules/stray.txt': 'Mozilla Public License', // a file outside any package is read too + 'node_modules/top/package.json': { name: 'top', version: '1.0.0', license: 'MIT', dependencies: { a: '1', '@s/b': '2' }, optionalDependencies: { d: '4', ghost: '9' }, peerDependencies: { p: '1', a: '1' } }, 'node_modules/a/package.json': { name: 'a', version: '1.0.0', license: 'MIT' }, 'node_modules/a/index.js': 'module.exports = 1;', + 'node_modules/a/.package-lock.json': 'Mozilla Public License', // a package's own file of that name is read 'node_modules/a/@x/y/z.js': '// a folder named @x inside a package is not a scope', 'node_modules/a/@x/y/package.json': { name: 'y' }, 'node_modules/@s/stray': 'SSPL', - 'node_modules/@s/b/package.json': { name: '@s/b', version: '2.0.0', license: 'Apache-2.0' }, + 'node_modules/@s/b/package.json': { name: '@s/b', version: '2.0.0', license: 'Apache-2.0', dependencies: { c: '3', '@t/e2': '5' } }, 'node_modules/@s/b/lib/x.js': '// fine', // A package.json inside a package (as module-type markers are) is a file // of it, not a package. 'node_modules/@s/b/lib/package.json': { type: 'commonjs' }, - 'node_modules/@s/b/node_modules/c/package.json': { name: 'c', version: '3.0.0', license: 'LGPL-3.0' }, + 'node_modules/@s/b/node_modules/c/package.json': { name: 'c', version: '3.0.0', license: 'LGPL-3.0', peerDependencies: { r: '1' } }, 'node_modules/@s/b/node_modules/c/NOTICE': 'Mozilla Public License', 'node_modules/@s/b/node_modules/@t/e2/package.json': { name: '@t/e2', version: '5.0.0', license: 'BUSL-1.1' }, 'node_modules/d/package.json': { name: 'd', version: '4.0.0', license: 'ISC' }, @@ -155,40 +287,52 @@ test('the walk reads every file under node_modules and every package folder, sco 'node_modules/e/index.js': 'no package.json here, so not a package', 'node_modules/a/test/fixture/node_modules/util/index.js': '// a fixture folder, not a package', 'node_modules/f/package.json': 'not json{', + // The peer of a dependency is the package's to ship: checked, though the + // user's peer p needs it too. + 'node_modules/r/package.json': { name: 'r', version: '1.0.0', license: 'EPL-2.0' }, + // top's own peer, and what only it needs, are the user's: not checked. + // What top needs too (a) is checked all the same. + 'node_modules/p/package.json': { name: 'p', version: '1.0.0', license: 'GPL-3.0', dependencies: { q: '1', a: '1', r: '1' } }, + 'node_modules/p/COPYING': 'GNU General Public License', + 'node_modules/p/node_modules/q/package.json': { name: 'q', version: '1.0.0', license: 'MPL-2.0' }, }); - t.after(() => rmSync(root, { recursive: true, force: true })); // npm's bin links are not read (the files they name are, where they are); // any other link is a finding. mkdirSync(join(root, 'node_modules/.bin')); symlinkSync('../a/index.js', join(root, 'node_modules/.bin/tool')); symlinkSync('../a/index.js', join(root, 'node_modules/d/linked')); - const { findings, packages, files } = check(root); - assert.equal(packages, 6); - assert.equal(files, 20); + const { findings, packages, files, user, peers } = await check(root, 'top'); + assert.deepEqual(user, ['p@1.0.0', 'q@1.0.0']); + assert.deepEqual(peers, ['p@1.0.0']); + assert.equal(packages, 8); + assert.equal(files, 22); assert.deepEqual(findings.sort(), [ - 'node_modules/stray.txt: MPL ("Mozilla Public License")', - 'node_modules/@s/stray: SSPL ("SSPL")', + 'node_modules/stray.txt: MPL ("mozilla public license")', + 'node_modules/a/.package-lock.json: MPL ("mozilla public license")', + 'node_modules/@s/stray: SSPL ("sspl")', 'node_modules/@s/b/node_modules/c (c@3.0.0): license "LGPL-3.0" is not allowed (field "LGPL-3.0")', - 'node_modules/@s/b/node_modules/c/NOTICE: MPL ("Mozilla Public License")', - 'node_modules/@s/b/node_modules/c/package.json: GPL, LGPL or AGPL ("LGPL-3.0")', + 'node_modules/@s/b/node_modules/c/NOTICE: MPL ("mozilla public license")', + 'node_modules/@s/b/node_modules/c/package.json: GPL, LGPL or AGPL ("lgpl 3 0")', 'node_modules/@s/b/node_modules/@t/e2 (@t/e2@5.0.0): license "BUSL-1.1" is not allowed (field "BUSL-1.1")', - 'node_modules/@s/b/node_modules/@t/e2/package.json: BUSL ("BUSL")', - 'node_modules/d/ThirdPartyNoticeText.txt: Unicode terms ("UNICODE, INC. LICENSE AGREEMENT")', + 'node_modules/@s/b/node_modules/@t/e2/package.json: BUSL ("busl 1 1")', + 'node_modules/d/ThirdPartyNoticeText.txt: Unicode terms ("unicode, inc license agreement")', 'node_modules/d/linked: not a file or a folder, so not read', `node_modules/f: no readable package.json (${parseError('not json{')})`, + 'node_modules/r (r@1.0.0): license "EPL-2.0" is not allowed (field "EPL-2.0")', + 'node_modules/r/package.json: EPL ("epl 2 0")', ].sort()); }); -test('a clean tree has no findings, and an empty project has no packages', (t) => { +test('a clean tree has no findings; a package that is not there is one', async () => { const root = tree({ - 'node_modules/a/package.json': { name: 'a', version: '1.0.0', license: 'MIT' }, + 'node_modules/a/package.json': { name: 'a', version: '1.0.0', license: 'MIT', peerDependencies: { gone: '1' } }, 'node_modules/a/LICENSE': 'MIT License\n\nPermission is hereby granted', }); const empty = tree({ 'package.json': { name: 'license-gate' } }); - t.after(() => { - rmSync(root, { recursive: true, force: true }); - rmSync(empty, { recursive: true, force: true }); - }); - assert.deepEqual(check(root), { findings: [], packages: 1, files: 2 }); - assert.deepEqual(check(empty), { findings: [], packages: 0, files: 0 }); + assert.deepEqual(await check(root, 'a'), { findings: [], packages: 1, files: 2, user: [], peers: [] }); + assert.deepEqual(await check(empty, 'a'), { findings: ['a: not installed'], packages: 0, files: 0, user: [], peers: [] }); +}); + +test('the tarball is installed for the runner and for other users\' platforms', () => { + assert.deepEqual(PLATFORMS, [['linux', 'x64'], ['linux', 'arm64'], ['darwin', 'arm64'], ['darwin', 'x64'], ['win32', 'x64']]); }); diff --git a/test/package.test.mjs b/test/package.test.mjs new file mode 100644 index 0000000..5e200c7 --- /dev/null +++ b/test/package.test.mjs @@ -0,0 +1,50 @@ +// Rows for @keploy/sdk as a user gets it: what its built code requires, and +// its entry points loaded from a clean install of its packed tarball. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { spawnSync } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; +import { builtinModules } from 'node:module'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const root = fileURLToPath(new URL('..', import.meta.url)); +const pkg = JSON.parse(readFileSync(join(root, 'package.json'), 'utf8')); +const entries = ['@keploy/sdk', '@keploy/sdk/dist/v2/dedup/middleware', '@keploy/sdk/dist/v2/dedup/register']; + +// The .js files under dir, at any depth. +function scripts(dir) { + return readdirSync(dir, { withFileTypes: true }).flatMap((e) => (e.isDirectory() ? scripts(join(dir, e.name)) : e.name.endsWith('.js') ? [join(dir, e.name)] : [])); +} + +test('every package the built code requires is a dependency', () => { + assert.ok(existsSync(join(root, 'dist/keployCli.js')), 'dist/ is built (npm ci runs prepare)'); + const required = new Set(); + for (const f of scripts(join(root, 'dist'))) { + for (const [, spec] of readFileSync(f, 'utf8').matchAll(/\brequire\(["']([^"']+)["']\)/g)) { + if (spec.startsWith('.') || spec.startsWith('node:') || builtinModules.includes(spec)) continue; + required.add(spec.startsWith('@') ? spec.split('/').slice(0, 2).join('/') : spec.split('/')[0]); + } + } + assert.deepEqual([...required].filter((n) => !(n in pkg.dependencies)), []); + // The two the dedup entry points need, which 2.0.10 did not declare. + for (const n of ['body-parser', 'js-yaml']) assert.ok(required.has(n), n); +}); + +test('the entry points load from a clean install of the packed package', (t) => { + const work = mkdtempSync(join(tmpdir(), 'sdk-install-')); + t.after(() => rmSync(work, { recursive: true, force: true })); + const packed = spawnSync('npm', ['pack', '--ignore-scripts', '--pack-destination', work], { cwd: root, encoding: 'utf8' }); + assert.equal(packed.status, 0, packed.stderr); + const tgz = join(work, packed.stdout.trim().split('\n').pop()); + const app = join(work, 'app'); + mkdirSync(app); + writeFileSync(join(app, 'package.json'), JSON.stringify({ name: 'app', version: '0.0.0', private: true })); + const inst = spawnSync('npm', ['install', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', tgz], { cwd: app, encoding: 'utf8' }); + assert.equal(inst.status, 0, inst.stderr); + for (const e of entries) { + const r = spawnSync(process.execPath, ['-e', `require(${JSON.stringify(e)})`], { cwd: app, encoding: 'utf8' }); + assert.equal(r.status, 0, `${e}: ${r.stderr}`); + } +}); diff --git a/test/workflows.test.mjs b/test/workflows.test.mjs new file mode 100644 index 0000000..5f82337 --- /dev/null +++ b/test/workflows.test.mjs @@ -0,0 +1,56 @@ +// Rows over the publish workflows themselves: each publishes the tarball its +// license check passed, and no job can hold a runner for GitHub's 6 hours. +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { readFileSync } from 'node:fs'; +import { createRequire } from 'node:module'; + +const yaml = createRequire(import.meta.url)('js-yaml'); +const load = (name) => yaml.load(readFileSync(new URL(`../.github/workflows/${name}`, import.meta.url), 'utf8')); +const TARBALL = '${{ steps.pack.outputs.tarball }}'; + +// The steps of job, in order: each one's name (or command), run, env and id. +function steps(job) { + return job.steps.map((s) => ({ name: s.name || s.run || s.uses, run: s.run || '', env: s.env || {}, id: s.id })); +} + +const workflows = [ + // [file, publishing job, pull request job, the gate's path from the job's folder] + ['sdk-publish.yml', 'publish', 'check', 'scripts/license-gate.mjs'], + ['playwright-publish.yml', 'publish', 'dry-run', '../scripts/license-gate.mjs'], + ['jest-publish.yml', 'publish', 'dry-run', '../scripts/license-gate.mjs'], +]; + +for (const [file, publish, pr, gate] of workflows) { + test(`${file}: the tarball packed once is license-checked, then published`, () => { + const wf = load(file); + for (const job of [publish, pr]) { + const s = steps(wf.jobs[job]); + const at = (name) => s.findIndex((x) => x.name === name); + const pack = at('Pack'); + assert.ok(pack >= 0, `${job}: a Pack step`); + assert.equal(s[pack].id, 'pack'); + assert.match(s[pack].run, /^name=\$\(npm pack --pack-destination "\$RUNNER_TEMP" \| tail -n 1\)\n/); + const lic = at('Licenses'); + assert.ok(lic > pack, `${job}: a Licenses step after Pack`); + assert.equal(s[lic].run, `node ${gate} "$TARBALL"`); + assert.equal(s[lic].env.TARBALL, TARBALL); + // npm packs or publishes nothing else. + assert.deepEqual(s.filter((x) => /\bnpm (?:pack|publish)\b/.test(x.run)).map((x) => x.name), job === publish ? ['Pack', 'Publish'] : file === 'sdk-publish.yml' ? ['Pack'] : ['Pack', 'Publish, dry run']); + const pub = s.findIndex((x) => /\bnpm publish\b/.test(x.run)); + if (pub < 0) continue; + assert.ok(pub > lic, `${job}: the publish runs after the Licenses step`); + assert.match(s[pub].run, /^npm publish "\$TARBALL" /); + assert.equal(s[pub].env.TARBALL, TARBALL); + } + assert.equal(steps(wf.jobs[publish]).find((x) => x.name === 'Publish').run, 'npm publish "$TARBALL" --access public --provenance'); + }); + + test(`${file}: every job that runs on a runner has a timeout`, () => { + const wf = load(file); + for (const [name, job] of Object.entries(wf.jobs)) { + if (job.uses) continue; // a called workflow's jobs carry their own + assert.ok(Number.isInteger(job['timeout-minutes']) && job['timeout-minutes'] <= 60, `${name}: timeout-minutes`); + } + }); +} From f642f6956262e533e4e8648cfc4d402e9741b97b Mon Sep 17 00:00:00 2001 From: slayerjain Date: Wed, 4 Nov 2026 16:54:58 +0530 Subject: [PATCH 3/9] test: pin the scan window with a long license URL near a block end A license page URL of 92 characters that ends up to 63 characters before the end of a block is found once the next block is read. A window kept any shorter than the gate keeps loses it. Signed-off-by: slayerjain --- test/license-gate.test.mjs | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/license-gate.test.mjs b/test/license-gate.test.mjs index 0ddcd83..98e5d8e 100644 --- a/test/license-gate.test.mjs +++ b/test/license-gate.test.mjs @@ -195,10 +195,12 @@ test('text read a piece at a time is matched as it would be whole', () => { assert.deepEqual(s.end(), []); // A long license URL cut across pieces is found whole. const page = `choosealicense.com/licenses/${'a'.repeat(64)}`; - const u = new Scanner(); - u.push(`${'x'.repeat(5000)} https://${page}`); - u.push(' and more'); - assert.deepEqual(u.end(), [`a license that is not allowed (${JSON.stringify(page)})`]); + for (const after of [0, 50, 63]) { + const u = new Scanner(); + u.push(`${'x'.repeat(5000)} https://${page}${' '.repeat(after)}`); + u.push(' and more'); + assert.deepEqual(u.end(), [`a license that is not allowed (${JSON.stringify(page)})`], `${after}`); + } // Nor is a word whose start was cut off before the window. const t = new Scanner(); t.push(`${'b'.repeat(1000)}agpl ${'c'.repeat(KEEP - 4)}`); From e3399b005aa3ac230c40ae7cf54ed2cf06a34cf0 Mon Sep 17 00:00:00 2001 From: slayerjain Date: Wed, 4 Nov 2026 17:00:15 +0530 Subject: [PATCH 4/9] test: pin that the walk reads code, markdown and big files The walk row gains a flagged file of each kind a scan might skip: .js, .cjs, .mjs, .d.ts, .js.map, README.md and a 300 KiB minified file. Signed-off-by: slayerjain --- test/license-gate.test.mjs | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/test/license-gate.test.mjs b/test/license-gate.test.mjs index 98e5d8e..10b3379 100644 --- a/test/license-gate.test.mjs +++ b/test/license-gate.test.mjs @@ -271,7 +271,14 @@ test('the walk reads every package folder and file but the user\'s own peers, sc 'node_modules/stray.txt': 'Mozilla Public License', // a file outside any package is read too 'node_modules/top/package.json': { name: 'top', version: '1.0.0', license: 'MIT', dependencies: { a: '1', '@s/b': '2' }, optionalDependencies: { d: '4', ghost: '9' }, peerDependencies: { p: '1', a: '1' } }, 'node_modules/a/package.json': { name: 'a', version: '1.0.0', license: 'MIT' }, - 'node_modules/a/index.js': 'module.exports = 1;', + // Every file is read, whatever its extension or size. + 'node_modules/a/index.js': 'module.exports = "ODbL-1.0";', + 'node_modules/a/lib/x.cjs': '// SPDX-License-Identifier: GPL-2.0-only', + 'node_modules/a/lib/x.mjs': 'export const l = "EUPL-1.2";', + 'node_modules/a/lib/x.d.ts': '// Eclipse Public License', + 'node_modules/a/lib/x.js.map': '{"sources":["CDDL-1.0"]}', + 'node_modules/a/README.md': 'Licensed under the Elastic License', + 'node_modules/a/dist/big.min.js': `${'x'.repeat(300 << 10)} Sleepycat`, 'node_modules/a/.package-lock.json': 'Mozilla Public License', // a package's own file of that name is read 'node_modules/a/@x/y/z.js': '// a folder named @x inside a package is not a scope', 'node_modules/a/@x/y/package.json': { name: 'y' }, @@ -307,10 +314,18 @@ test('the walk reads every package folder and file but the user\'s own peers, sc assert.deepEqual(user, ['p@1.0.0', 'q@1.0.0']); assert.deepEqual(peers, ['p@1.0.0']); assert.equal(packages, 8); - assert.equal(files, 22); + assert.equal(files, 28); assert.deepEqual(findings.sort(), [ 'node_modules/stray.txt: MPL ("mozilla public license")', 'node_modules/a/.package-lock.json: MPL ("mozilla public license")', + 'node_modules/a/index.js: ODbL ("odbl 1 0")', + 'node_modules/a/lib/x.cjs: GPL, LGPL or AGPL ("gpl 2 0")', + 'node_modules/a/lib/x.cjs: license "gpl-2.0-only" is not allowed (SPDX-License-Identifier)', + 'node_modules/a/lib/x.mjs: EUPL ("eupl 1 2")', + 'node_modules/a/lib/x.d.ts: EPL ("eclipse public license")', + 'node_modules/a/lib/x.js.map: CDDL ("cddl 1 0")', + 'node_modules/a/README.md: Elastic ("elastic license")', + 'node_modules/a/dist/big.min.js: APSL, NPL, RPL, MS-RL or Sleepycat ("sleepycat")', 'node_modules/@s/stray: SSPL ("sspl")', 'node_modules/@s/b/node_modules/c (c@3.0.0): license "LGPL-3.0" is not allowed (field "LGPL-3.0")', 'node_modules/@s/b/node_modules/c/NOTICE: MPL ("mozilla public license")', From 51dcc93e32db6f63e3760c9336bb17116e1041aa Mon Sep 17 00:00:00 2001 From: slayerjain Date: Wed, 4 Nov 2026 17:07:39 +0530 Subject: [PATCH 5/9] test: pin that a tarball named by a relative path is found Signed-off-by: slayerjain --- test/license-gate-cli.test.mjs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/license-gate-cli.test.mjs b/test/license-gate-cli.test.mjs index 59652a5..70fb22c 100644 --- a/test/license-gate-cli.test.mjs +++ b/test/license-gate-cli.test.mjs @@ -7,7 +7,7 @@ import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; import { mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; -import { join } from 'node:path'; +import { basename, join } from 'node:path'; import { fileURLToPath } from 'node:url'; const script = fileURLToPath(new URL('../scripts/license-gate.mjs', import.meta.url)); @@ -26,9 +26,9 @@ function pack(name, files) { } // Runs the script with its own TMPDIR, so a row can see what it leaves there. -function run(args, env = {}) { +function run(args, env = {}, cwd = undefined) { const tmp = mkdtempSync(join(work, 'tmp-')); - const r = spawnSync(process.execPath, [script, ...args], { encoding: 'utf8', env: { ...process.env, TMPDIR: tmp, ...env } }); + const r = spawnSync(process.execPath, [script, ...args], { cwd, encoding: 'utf8', env: { ...process.env, TMPDIR: tmp, ...env } }); // npm keeps caches of its own there (node-compile-cache): only the script's // projects count. return { ...r, left: readdirSync(tmp).filter((n) => n.startsWith('license-gate-')) }; @@ -45,6 +45,12 @@ test('a clean package passes on every platform, and the projects it was installe assert.deepEqual(r.left, []); }); +test('a tarball named by a relative path is found', () => { + const tgz = pack('relative', { 'package.json': { name: 'relative', version: '1.0.0', license: 'MIT' } }); + const r = run([basename(tgz)], {}, work); + assert.equal(r.status, 0, r.stderr); +}); + test('a package whose license field is not allowed fails, named', () => { const tgz = pack('red', { 'package.json': { name: 'red', version: '1.0.0', license: 'GPL-3.0' } }); const r = run([tgz]); From 46e8ed3e4f2b34fd833133233ab63989f9142a1c Mon Sep 17 00:00:00 2001 From: slayerjain Date: Wed, 4 Nov 2026 20:17:29 +0530 Subject: [PATCH 6/9] fix: run workflow rows in every publish job; read dedup data with yaml The jest and playwright publish and dry-run jobs now run test/workflows.test.mjs from the repository's root before their license check, so no tag or pull request that runs one of them skips the rows that hold it to the check. sdk-publish.yml's check also runs on a change to either workflow. The rows now also fail when a job that packs, checks or publishes, or one of its steps, carries an if beyond the job's event or a continue-on-error; when a job loses its rows step; when a workflow's tag trigger changes, or its pull request trigger gains a filter or stops covering it, the rows or the check; when a job of a workflow one calls has no timeout; and when a workflow that runs npm publish is not one they hold. The rows jobs of jest-sdk.yml and playwright-sdk.yml get a 30-minute timeout. The license check now: - flags the Creative Commons 3.0 Unported legal code by its opening words ("Creative Commons Legal Code / Attribution"); - installs with optional and peer dependencies in, whatever an npmrc or an npm_config_ variable says; - leaves to the user only the peers jest, jest-environment-node and @playwright/test: any other peerDependency of the package fails. The dedup middleware reads and writes dedupData.yaml with yaml (ISC, no dependencies) in place of js-yaml 3, which brought in esprima, last released in 2018. It writes the file byte for byte as js-yaml 4 did, reads the files js-yaml 3 and 4 wrote, and loads no js/ tag as code. Signed-off-by: slayerjain --- .github/workflows/jest-publish.yml | 20 +++- .github/workflows/jest-sdk.yml | 2 + .github/workflows/playwright-publish.yml | 20 +++- .github/workflows/playwright-sdk.yml | 2 + .github/workflows/sdk-publish.yml | 4 + package-lock.json | 32 +------ package.json | 4 +- scripts/license-gate.mjs | 50 ++++++---- test/license-gate-cli.test.mjs | 37 +++++++- test/license-gate.test.mjs | 43 ++++++--- test/package.test.mjs | 73 ++++++++++++++- test/workflows.test.mjs | 111 ++++++++++++++++++++--- v2/dedup/middleware.ts | 9 +- 13 files changed, 321 insertions(+), 86 deletions(-) diff --git a/.github/workflows/jest-publish.yml b/.github/workflows/jest-publish.yml index 5007ff8..2d6dbf3 100644 --- a/.github/workflows/jest-publish.yml +++ b/.github/workflows/jest-publish.yml @@ -22,7 +22,7 @@ name: "@keploy/jest publish" # (scripts/license-gate.mjs, as in sdk-publish.yml) installs the tarball as a # user's `npm install` does and refuses it when anything it installs names a # license outside its allow-list. test/workflows.test.mjs holds this -# workflow to that. +# workflow to that, and both of its jobs run those rows first. # # On a pull request the same tarball is packed, checked and published as a # dry run, with no token, so a package that would ship without its build, or @@ -67,6 +67,15 @@ jobs: # scope to it (@keploy:registry), in the runner's own npmrc. registry-url: https://registry.npmjs.org scope: "@keploy" + - name: Workflow rows + # test/workflows.test.mjs, from the repository's root, so that every + # tag and pull request this workflow runs on checks that it still + # publishes only the tarball its Licenses step passed. The root's + # install, with no scripts, is only for the rows' YAML parser. + working-directory: . + run: | + npm ci --ignore-scripts + node --test test/workflows.test.mjs - name: The tag names package.json's version run: node scripts/release-version.mjs "$GITHUB_REF_NAME" - name: The npm token is set @@ -127,6 +136,15 @@ jobs: - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 + - name: Workflow rows + # test/workflows.test.mjs, from the repository's root, so that every + # tag and pull request this workflow runs on checks that it still + # publishes only the tarball its Licenses step passed. The root's + # install, with no scripts, is only for the rows' YAML parser. + working-directory: . + run: | + npm ci --ignore-scripts + node --test test/workflows.test.mjs - run: npm ci - name: Build run: npm run build diff --git a/.github/workflows/jest-sdk.yml b/.github/workflows/jest-sdk.yml index 3080b51..37d2ad4 100644 --- a/.github/workflows/jest-sdk.yml +++ b/.github/workflows/jest-sdk.yml @@ -18,6 +18,8 @@ jobs: rows: name: build and rows (Node ${{ matrix.node }}, Jest ${{ matrix.jest }}) runs-on: ubuntu-latest + # A leg takes a few minutes, npm installs included: one past 30 is a hang. + timeout-minutes: 30 strategy: # One leg failing must not cancel the others: each leg's result is its # own. diff --git a/.github/workflows/playwright-publish.yml b/.github/workflows/playwright-publish.yml index efd0165..1824dc9 100644 --- a/.github/workflows/playwright-publish.yml +++ b/.github/workflows/playwright-publish.yml @@ -20,7 +20,7 @@ name: "@keploy/playwright publish" # (scripts/license-gate.mjs, as in sdk-publish.yml) installs the tarball as a # user's `npm install` does and refuses it when anything it installs names a # license outside its allow-list. test/workflows.test.mjs holds this -# workflow to that. +# workflow to that, and both of its jobs run those rows first. # # On a pull request the same tarball is packed, checked and published as a # dry run, with no token, so a package that would ship without its build, or @@ -65,6 +65,15 @@ jobs: # scope to it (@keploy:registry), in the runner's own npmrc. registry-url: https://registry.npmjs.org scope: "@keploy" + - name: Workflow rows + # test/workflows.test.mjs, from the repository's root, so that every + # tag and pull request this workflow runs on checks that it still + # publishes only the tarball its Licenses step passed. The root's + # install, with no scripts, is only for the rows' YAML parser. + working-directory: . + run: | + npm ci --ignore-scripts + node --test test/workflows.test.mjs - name: The tag names package.json's version run: node scripts/release-version.mjs "$GITHUB_REF_NAME" - name: The npm token is set @@ -125,6 +134,15 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 22 + - name: Workflow rows + # test/workflows.test.mjs, from the repository's root, so that every + # tag and pull request this workflow runs on checks that it still + # publishes only the tarball its Licenses step passed. The root's + # install, with no scripts, is only for the rows' YAML parser. + working-directory: . + run: | + npm ci --ignore-scripts + node --test test/workflows.test.mjs - run: npm ci - name: Build run: npm run build diff --git a/.github/workflows/playwright-sdk.yml b/.github/workflows/playwright-sdk.yml index 061954d..5dc9b0a 100644 --- a/.github/workflows/playwright-sdk.yml +++ b/.github/workflows/playwright-sdk.yml @@ -17,6 +17,8 @@ jobs: rows: name: build and rows (Node ${{ matrix.node }}) runs-on: ubuntu-latest + # A leg takes a few minutes, npm installs included: one past 30 is a hang. + timeout-minutes: 30 strategy: # One leg failing must not cancel the others: each Node version's # result is its own. diff --git a/.github/workflows/sdk-publish.yml b/.github/workflows/sdk-publish.yml index 1bb3936..6122e7b 100644 --- a/.github/workflows/sdk-publish.yml +++ b/.github/workflows/sdk-publish.yml @@ -36,6 +36,10 @@ on: - "scripts/**" - "test/**" - ".github/workflows/sdk-publish.yml" + # The rows (test/workflows.test.mjs) hold these two to the license + # check as well: a change to either runs them here too. + - ".github/workflows/jest-publish.yml" + - ".github/workflows/playwright-publish.yml" permissions: contents: read diff --git a/package-lock.json b/package-lock.json index 6ea972e..72648d3 100644 --- a/package-lock.json +++ b/package-lock.json @@ -108,14 +108,6 @@ "@types/send": "*" } }, - "argparse": { - "version": "1.0.10", - "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", - "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", - "requires": { - "sprintf-js": "~1.0.2" - } - }, "asynckit": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz", @@ -266,11 +258,6 @@ "es-errors": "^1.3.0" } }, - "esprima": { - "version": "4.0.1", - "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", - "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==" - }, "follow-redirects": { "version": "1.15.6", "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.6.tgz", @@ -368,15 +355,6 @@ "hasown": "^2.0.0" } }, - "js-yaml": { - "version": "3.15.2", - "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.15.2.tgz", - "integrity": "sha512-6EuL879VkRA+1Cz578mKMiKvjPNEuk6+r1JaFzoSWejZmtf7xWbIyw1e3KkxlkzTIt9Taw6JBhEppG7utc1P+w==", - "requires": { - "argparse": "^1.0.7", - "esprima": "^4.0.0" - } - }, "lodash": { "version": "4.17.21", "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", @@ -537,11 +515,6 @@ "side-channel-map": "^1.0.1" } }, - "sprintf-js": { - "version": "1.0.3", - "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", - "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==" - }, "statuses": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz", @@ -591,6 +564,11 @@ "version": "1.1.2", "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz", "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==" + }, + "yaml": { + "version": "2.9.1", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.1.tgz", + "integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==" } } } diff --git a/package.json b/package.json index c9a5c88..ec233e6 100644 --- a/package.json +++ b/package.json @@ -30,10 +30,10 @@ "axios": "^1.6.7", "body-parser": "^1.20.9", "cors": "^2.8.5", - "js-yaml": "^3.15.2", "lodash": "^4.17.21", "require-in-the-middle": "^7.3.0", - "tree-kill": "^1.2.2" + "tree-kill": "^1.2.2", + "yaml": "^2.9.1" }, "devDependencies": { "@types/axios": "^0.14.0", diff --git a/scripts/license-gate.mjs b/scripts/license-gate.mjs index db28ad8..a92e516 100644 --- a/scripts/license-gate.mjs +++ b/scripts/license-gate.mjs @@ -1,10 +1,11 @@ // license-gate.mjs TARBALL: the publish workflows' license check. // // TARBALL is the file `npm pack` wrote, the one the workflow then publishes. -// The check installs it into an empty project the way a user does (`npm -// install`: no devDependencies; optional and peer dependencies in), once for -// each platform in PLATFORMS (npm's --os and --cpu), so that a dependency npm -// installs only on another platform is checked too. It refuses the package +// The check installs it into an empty project the way a user's `npm install` +// does, with install scripts off (no devDependencies; optional and peer +// dependencies in, whatever an npmrc or an npm_config_ variable says), once +// for each platform in PLATFORMS (npm's --os and --cpu), so that a dependency +// npm installs only on one of those platforms is checked too. It refuses the package // when anything those installs put on disk for it names a license outside the // allow-list below: // - each installed package's package.json license field (or the old @@ -20,11 +21,13 @@ // "CC-BY-4.0" and "cc_by_4.0" are one spelling. A gzip file is read // decompressed; a file that starts with a UTF-16 byte order mark is read // as UTF-16, any other as UTF-8. -// What the package's own peerDependencies install is the user's: a package -// that peers on jest is installed beside the user's own jest, and ships none -// of it. Those packages are named in the output and not checked, unless the -// package or one of its dependencies needs them too (peers of a dependency -// are checked: the package is what brings them). +// A package's own peerDependencies are the user's install, not ours: the +// user's own jest or playwright tree is already in their project whatever we +// ship. The gate names them, and does not judge them. Peers of our +// dependencies are still judged, and so is anything the package or one of +// its dependencies needs too. Only the peers in USER_PEERS are left to the +// user: any other peerDependency of the package fails, until it is added +// there. // Accepting a flagged file with a notice is not enough, so there is no // exception list: a finding fails the publish. // @@ -32,14 +35,20 @@ // - a license that FLAGGED does not name passes when only a file's text // names it (the Artistic License, say); a license field naming it fails. // - the id Unicode-3.0 is matched only in an SPDX-License-Identifier line -// (js-yaml's code, for one, cites "Unicode 3.0", the standard). +// (code that cites "Unicode 3.0" most often names the standard). // - a name or id is matched only when what splits it is a separator above: // one split by markup, a comment marker or an escape, or spelt in other // than ASCII letters, is not. +// - a license text in a language other than English (the Creative Commons +// ports to German, French, Dutch, Austrian or Japanese law, say) passes +// unless it also names a flagged id, English name or URL. // - only gzip is decompressed, one level deep: zip, xz, bzip2 and brotli // data, and UTF-16 text with no byte order mark, are read as their bytes. // - npm cannot install for another C library: an optional dependency for // musl (Alpine) Linux only is not installed, so not checked. +// - only the platforms in PLATFORMS are installed for: an optional +// dependency only for another (win32/arm64, linux/arm, ia32, the BSDs, +// AIX, say) is not installed, so not checked. // // Exit 0 and print what was checked when the package is clean; exit 1 and // list every finding when not, or when an install fails. @@ -56,6 +65,10 @@ export const ALLOWED = new Set([ 'BSL-1.0', 'CC0-1.0', 'Unlicense', 'PostgreSQL', 'OFL-1.1', ]); +// The peerDependencies of a package that are left to the user (see the +// header): those @keploy/jest and @keploy/playwright peer on. +export const USER_PEERS = new Set(['jest', 'jest-environment-node', '@playwright/test']); + // The platforms the tarball is installed for: the runner's, and those of // other users, as npm's --os and --cpu name them. export const PLATFORMS = [ @@ -95,7 +108,9 @@ const LICENSES = [ ['CPL or CPAL', { ids: ['cpl v? \\d', 'cpal'], names: ['common public (?:attribution )?licen[cs]e'] }], ['Creative Commons', { ids: ['cc by(?: (?:nc|nd|sa)){0,3}'], - names: ['creative commons (?:attribution|by|non commercial|share alike|no derivatives|noderivs)', 'attribution (?:non commercial|share alike|no derivatives|noderivs)'], + // The 3.0 Unported legal code opens "Creative Commons Legal Code / + // Attribution 3.0 Unported" (BY and its NC, ND and SA variants). + names: ['creative commons (?:attribution|by|non commercial|share alike|no derivatives|noderivs|legal code attribution)', 'attribution (?:non commercial|share alike|no derivatives|noderivs)'], urls: ['creativecommons org/licenses'], }], ['ODbL', { ids: ['odbl'], names: ['open database licen[cs]e'] }], @@ -292,8 +307,8 @@ function needs(pkg, fields) { // node_modules/@SCOPE/NAME, at any depth) and every file under node_modules // is read, whichever package it belongs to, but for npm's own record of the // tree (node_modules/.package-lock.json: the package.json files it copies are -// read where they are) and the packages only root's peerDependencies need -// (see the header), which are returned as user (and those peers as peers). +// read where they are) and the packages only root's peerDependencies in +// USER_PEERS need (see the header), which are returned as user (and those peers as peers). // A link is not read: npm // makes links only in .bin folders, to files read where they are, so any // other link, or anything that is not a file or a folder, is a finding. @@ -359,7 +374,9 @@ export async function check(project, root) { if (!packages.has(top)) findings.push(`${root}: not installed`); else { const ours = reach([top], new Set([top])); - peers = needs(packages.get(top).pkg, ['peerDependencies']).map((n) => find(top, n)).filter(Boolean); + const own = needs(packages.get(top).pkg, ['peerDependencies']); + for (const n of own) if (!USER_PEERS.has(n)) findings.push(`${root}: peerDependency ${JSON.stringify(n)} is not one left to the user (USER_PEERS), so it fails until it is added there`); + peers = own.filter((n) => USER_PEERS.has(n)).map((n) => find(top, n)).filter(Boolean); user = new Set([...reach(peers, new Set())].filter((d) => !ours.has(d))); } @@ -386,12 +403,13 @@ export async function check(project, root) { // Installs tarball into the empty project, as a user's `npm install` does on // platform ([os, cpu]) (npm never installs a dependency's devDependencies; -// no install script runs), and returns the package's name: the one +// no install script runs; the flags put optional and peer dependencies in +// over any npmrc or npm_config_ variable), and returns the package's name: the one // dependency npm saved to the project's package.json. Throws, saying why, on // a failure. export function install(tarball, project, [os, cpu] = PLATFORMS[0]) { writeFileSync(join(project, 'package.json'), JSON.stringify({ name: 'license-gate', version: '0.0.0', private: true })); - const args = ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--no-package-lock', `--os=${os}`, `--cpu=${cpu}`, resolve(tarball)]; + const args = ['install', '--ignore-scripts', '--include=optional', '--include=peer', '--legacy-peer-deps=false', '--no-audit', '--no-fund', '--no-package-lock', `--os=${os}`, `--cpu=${cpu}`, resolve(tarball)]; const r = spawnSync('npm', args, { cwd: project, encoding: 'utf8' }); if (r.status !== 0) throw new Error(`npm install ${tarball} for ${os}/${cpu} failed (${r.error ? r.error.message : `exit ${r.status}`}): ${(r.stderr || '').trim()}`); const names = needs(JSON.parse(readFileSync(join(project, 'package.json'), 'utf8')), ['dependencies']); diff --git a/test/license-gate-cli.test.mjs b/test/license-gate-cli.test.mjs index 70fb22c..80fd245 100644 --- a/test/license-gate-cli.test.mjs +++ b/test/license-gate-cli.test.mjs @@ -81,14 +81,15 @@ test('optional dependencies, those of other platforms, and the peers of dependen const peer = dep('redpeer', { license: 'EPL-2.0' }); const mid = dep('mid', { license: 'MIT', peerDependencies: { redpeer: `file:${peer}` } }); // The user's own peer needs redpeer too: it is checked all the same. - const own = dep('ownpeer', { license: 'SSPL-1.0', dependencies: { redpeer: `file:${peer}` } }); + const own = dep('jest', { license: 'SSPL-1.0', dependencies: { redpeer: `file:${peer}` } }); const tgz = pack('top', { 'package.json': { name: 'top', version: '1.0.0', license: 'MIT', dependencies: { mid: `file:${mid}` }, optionalDependencies: { redopt: `file:${opt}`, reddarwin: `file:${darwin}`, redwin: `file:${win}` }, - // The package's own peer is the user's: named, not checked. - peerDependencies: { ownpeer: `file:${own}` }, + // The package's own peer (one of USER_PEERS) is the user's: named, + // not checked. + peerDependencies: { jest: `file:${own}` }, }, }); const r = run([tgz]); @@ -100,10 +101,38 @@ test('optional dependencies, those of other platforms, and the peers of dependen ' node_modules/redpeer (redpeer@1.0.0): license "EPL-2.0" is not allowed (field "EPL-2.0")', ' node_modules/redwin (redwin@1.0.0): license "MPL-2.0" is not allowed (field "MPL-2.0") (on win32/x64)', ]); - assert.match(r.stderr, /\nchecked linux\/x64: 4 packages, \d+ files; linux\/arm64: 4 packages, \d+ files; darwin\/arm64: 5 packages, \d+ files; darwin\/x64: 4 packages, \d+ files; win32\/x64: 5 packages, \d+ files; not checked, the user's own: 1 packages its peerDependencies install \(ownpeer@1\.0\.0\)\n$/); + assert.match(r.stderr, /\nchecked linux\/x64: 4 packages, \d+ files; linux\/arm64: 4 packages, \d+ files; darwin\/arm64: 5 packages, \d+ files; darwin\/x64: 4 packages, \d+ files; win32\/x64: 5 packages, \d+ files; not checked, the user's own: 1 packages its peerDependencies install \(jest@1\.0\.0\)\n$/); assert.deepEqual(r.left, []); }); +test('an npm config that leaves optional or peer dependencies out changes nothing', () => { + const dep = (name, extra) => pack(name, { 'package.json': { name, version: '1.0.0', ...extra } }); + const opt = dep('envopt', { license: 'GPL-3.0' }); + const peer = dep('envpeer', { license: 'EPL-2.0' }); + const mid = dep('envmid', { license: 'MIT', peerDependencies: { envpeer: `file:${peer}` } }); + const tgz = pack('envtop', { 'package.json': { name: 'envtop', version: '1.0.0', license: 'MIT', dependencies: { envmid: `file:${mid}` }, optionalDependencies: { envopt: `file:${opt}` } } }); + for (const omit of ['optional', 'peer']) { + const r = run([tgz], { npm_config_omit: omit, npm_config_legacy_peer_deps: 'true' }); + assert.equal(r.status, 1, r.stderr); + assert.deepEqual(r.stderr.split('\n').filter((l) => / is not allowed /.test(l)).sort(), [ + ' node_modules/envopt (envopt@1.0.0): license "GPL-3.0" is not allowed (field "GPL-3.0")', + ' node_modules/envpeer (envpeer@1.0.0): license "EPL-2.0" is not allowed (field "EPL-2.0")', + ], `npm_config_omit=${omit}`); + } +}); + +test('a peerDependency of the package other than those left to the user fails, and is checked', () => { + const other = pack('otherpeer', { 'package.json': { name: 'otherpeer', version: '1.0.0', license: 'MPL-2.0' } }); + const tgz = pack('peertop', { 'package.json': { name: 'peertop', version: '1.0.0', license: 'MIT', peerDependencies: { otherpeer: `file:${other}` } } }); + const r = run([tgz]); + assert.equal(r.status, 1, r.stderr); + assert.deepEqual(r.stderr.split('\n').filter((l) => /^ /.test(l)), [ + ' peertop: peerDependency "otherpeer" is not one left to the user (USER_PEERS), so it fails until it is added there', + ' node_modules/otherpeer (otherpeer@1.0.0): license "MPL-2.0" is not allowed (field "MPL-2.0")', + ' node_modules/otherpeer/package.json: MPL ("mpl 2 0")', + ]); +}); + test('a file that is not a package fails at the install', () => { const bad = join(work, 'not-a-package.tgz'); writeFileSync(bad, 'not a tarball'); diff --git a/test/license-gate.test.mjs b/test/license-gate.test.mjs index 10b3379..42f0c68 100644 --- a/test/license-gate.test.mjs +++ b/test/license-gate.test.mjs @@ -4,11 +4,11 @@ // license-gate-cli.test.mjs. import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; +import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { dirname, join } from 'node:path'; import { gzipSync } from 'node:zlib'; -import { ALLOWED, FLAGGED, KEEP, PLATFORMS, Scanner, check, expressionIds, fileFindings, licenseFindings, normalise, textFindings } from '../scripts/license-gate.mjs'; +import { ALLOWED, FLAGGED, KEEP, PLATFORMS, Scanner, USER_PEERS, check, expressionIds, fileFindings, licenseFindings, normalise, textFindings } from '../scripts/license-gate.mjs'; test('every allowed SPDX id passes, alone and in an expression', () => { const ids = ['MIT', 'BSD-2-Clause', 'BSD-3-Clause', '0BSD', 'Apache-2.0', 'ISC', 'Zlib', 'BSL-1.0', 'CC0-1.0', 'Unlicense', 'PostgreSQL', 'OFL-1.1']; @@ -88,6 +88,10 @@ const flagged = [ ['/* SPDX-License-Identifier: Unicode-3.0 */', ['license "unicode-3.0" is not allowed (SPDX-License-Identifier)']], ['// spdx-license-identifier:\t(MIT OR Artistic-2.0).', ['license "artistic-2.0" is not allowed (SPDX-License-Identifier)']], ['# SPDX-License-Identifier: BSD-3-Clause\nSPDX-License-Identifier: WTFPL AND ISC', ['license "wtfpl" is not allowed (SPDX-License-Identifier)']], + ['SPDX-License-Identifier: Foo-1.0', ['license "foo-1.0" is not allowed (SPDX-License-Identifier)']], + // Up to 8 spaces or tabs after the colon are not counted in the + // expression's 80 characters. + [`SPDX-License-Identifier:${' '.repeat(76)}Foo-1.0`, ['license "foo-1.0" is not allowed (SPDX-License-Identifier)']], ['UNICODE, INC. LICENSE AGREEMENT - DATA FILES AND SOFTWARE', ['Unicode terms ("unicode, inc license agreement")']], ['Unicode, Inc. Terms of Use', ['Unicode terms ("unicode, inc terms of use")']], ['Data Files and Software: Unicode Terms of Use apply', ['Unicode terms ("unicode terms of use")']], @@ -122,6 +126,10 @@ const flagged = [ ['Attribution-NonCommercial-ShareAlike', ['Creative Commons ("attribution noncommercial")']], ['Attribution-ShareAlike 4.0 International', ['Creative Commons ("attribution sharealike")']], ['https://creativecommons.org/licenses/by/4.0/', ['Creative Commons ("creativecommons org/licenses")']], + // The first lines of the 3.0 Unported legal code, as SPDX has it: they say + // neither "Creative Commons Attribution" nor "CC BY". + ['Creative Commons Legal Code\n\nAttribution 3.0 Unported\n\n CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE\n LEGAL SERVICES.', ['Creative Commons ("creative commons legal code attribution")']], + ['Creative Commons Legal Code\n\nAttribution-ShareAlike 3.0 Unported', ['Creative Commons ("creative commons legal code attribution")', 'Creative Commons ("attribution sharealike")']], ['ODbL-1.0', ['ODbL ("odbl 1 0")']], ['Open Database License', ['ODbL ("open database license")']], ['Elastic-2.0', ['Elastic ("elastic 2 0")']], @@ -160,6 +168,8 @@ const clean = [ '// SPDX-License-Identifier: (MIT OR Apache-2.0) */ and and SPDX-License-Identifier: 0BSD.', '// Check for surrogate pairs (reference Unicode 3.0 section "3.7 Surrogates").', 'elastic band; Unicode 15 tables; the commons; creative work; attribution required', + // CC0's legal code opens as the attribution licenses' do. + 'Creative Commons Legal Code\n\nCC0 1.0 Universal\n\n CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE', ]; test('a file naming allowed licenses only is clean', () => { @@ -269,7 +279,7 @@ test('the walk reads every package folder and file but the user\'s own peers, sc 'node_modules/.cache/x/data.json': '{}', // a dot folder is read, and is not a package 'node_modules/.cache/package.json': { name: 'cache' }, 'node_modules/stray.txt': 'Mozilla Public License', // a file outside any package is read too - 'node_modules/top/package.json': { name: 'top', version: '1.0.0', license: 'MIT', dependencies: { a: '1', '@s/b': '2' }, optionalDependencies: { d: '4', ghost: '9' }, peerDependencies: { p: '1', a: '1' } }, + 'node_modules/top/package.json': { name: 'top', version: '1.0.0', license: 'MIT', dependencies: { a: '1', '@s/b': '2' }, optionalDependencies: { d: '4', ghost: '9' }, peerDependencies: { jest: '1', a: '1' } }, 'node_modules/a/package.json': { name: 'a', version: '1.0.0', license: 'MIT' }, // Every file is read, whatever its extension or size. 'node_modules/a/index.js': 'module.exports = "ODbL-1.0";', @@ -297,13 +307,14 @@ test('the walk reads every package folder and file but the user\'s own peers, sc 'node_modules/a/test/fixture/node_modules/util/index.js': '// a fixture folder, not a package', 'node_modules/f/package.json': 'not json{', // The peer of a dependency is the package's to ship: checked, though the - // user's peer p needs it too. + // user's peer jest needs it too. 'node_modules/r/package.json': { name: 'r', version: '1.0.0', license: 'EPL-2.0' }, - // top's own peer, and what only it needs, are the user's: not checked. - // What top needs too (a) is checked all the same. - 'node_modules/p/package.json': { name: 'p', version: '1.0.0', license: 'GPL-3.0', dependencies: { q: '1', a: '1', r: '1' } }, - 'node_modules/p/COPYING': 'GNU General Public License', - 'node_modules/p/node_modules/q/package.json': { name: 'q', version: '1.0.0', license: 'MPL-2.0' }, + // top's own peer jest (one of USER_PEERS), and what only it needs, are + // the user's: not checked. What top needs too (a) is checked all the + // same, and a, a peer of top's not in USER_PEERS, is a finding. + 'node_modules/jest/package.json': { name: 'jest', version: '1.0.0', license: 'GPL-3.0', dependencies: { q: '1', a: '1', r: '1' } }, + 'node_modules/jest/COPYING': 'GNU General Public License', + 'node_modules/jest/node_modules/q/package.json': { name: 'q', version: '1.0.0', license: 'MPL-2.0' }, }); // npm's bin links are not read (the files they name are, where they are); // any other link is a finding. @@ -311,12 +322,13 @@ test('the walk reads every package folder and file but the user\'s own peers, sc symlinkSync('../a/index.js', join(root, 'node_modules/.bin/tool')); symlinkSync('../a/index.js', join(root, 'node_modules/d/linked')); const { findings, packages, files, user, peers } = await check(root, 'top'); - assert.deepEqual(user, ['p@1.0.0', 'q@1.0.0']); - assert.deepEqual(peers, ['p@1.0.0']); + assert.deepEqual(user, ['jest@1.0.0', 'q@1.0.0']); + assert.deepEqual(peers, ['jest@1.0.0']); assert.equal(packages, 8); assert.equal(files, 28); assert.deepEqual(findings.sort(), [ 'node_modules/stray.txt: MPL ("mozilla public license")', + 'top: peerDependency "a" is not one left to the user (USER_PEERS), so it fails until it is added there', 'node_modules/a/.package-lock.json: MPL ("mozilla public license")', 'node_modules/a/index.js: ODbL ("odbl 1 0")', 'node_modules/a/lib/x.cjs: GPL, LGPL or AGPL ("gpl 2 0")', @@ -340,9 +352,16 @@ test('the walk reads every package folder and file but the user\'s own peers, sc ].sort()); }); +test('the peers left to the user are those @keploy/jest and @keploy/playwright peer on, and no others', () => { + const peers = (dir) => Object.keys(JSON.parse(readFileSync(new URL(`../${dir}/package.json`, import.meta.url), 'utf8')).peerDependencies); + assert.deepEqual([...USER_PEERS].sort(), [...peers('jest'), ...peers('playwright')].sort()); + assert.deepEqual([...USER_PEERS].sort(), ['@playwright/test', 'jest', 'jest-environment-node']); +}); + test('a clean tree has no findings; a package that is not there is one', async () => { const root = tree({ - 'node_modules/a/package.json': { name: 'a', version: '1.0.0', license: 'MIT', peerDependencies: { gone: '1' } }, + // A peer left to the user that is not installed is no finding. + 'node_modules/a/package.json': { name: 'a', version: '1.0.0', license: 'MIT', peerDependencies: { jest: '1' } }, 'node_modules/a/LICENSE': 'MIT License\n\nPermission is hereby granted', }); const empty = tree({ 'package.json': { name: 'license-gate' } }); diff --git a/test/package.test.mjs b/test/package.test.mjs index 5e200c7..6b76417 100644 --- a/test/package.test.mjs +++ b/test/package.test.mjs @@ -29,22 +29,85 @@ test('every package the built code requires is a dependency', () => { } assert.deepEqual([...required].filter((n) => !(n in pkg.dependencies)), []); // The two the dedup entry points need, which 2.0.10 did not declare. - for (const n of ['body-parser', 'js-yaml']) assert.ok(required.has(n), n); + for (const n of ['body-parser', 'yaml']) assert.ok(required.has(n), n); }); -test('the entry points load from a clean install of the packed package', (t) => { - const work = mkdtempSync(join(tmpdir(), 'sdk-install-')); - t.after(() => rmSync(work, { recursive: true, force: true })); +// A clean install of the packed package, as a user's app gets it. +const work = mkdtempSync(join(tmpdir(), 'sdk-install-')); +test.after(() => rmSync(work, { recursive: true, force: true })); +const app = join(work, 'app'); +test.before(() => { const packed = spawnSync('npm', ['pack', '--ignore-scripts', '--pack-destination', work], { cwd: root, encoding: 'utf8' }); assert.equal(packed.status, 0, packed.stderr); const tgz = join(work, packed.stdout.trim().split('\n').pop()); - const app = join(work, 'app'); mkdirSync(app); writeFileSync(join(app, 'package.json'), JSON.stringify({ name: 'app', version: '0.0.0', private: true })); const inst = spawnSync('npm', ['install', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', tgz], { cwd: app, encoding: 'utf8' }); assert.equal(inst.status, 0, inst.stderr); +}); + +test('the entry points load from a clean install of the packed package', () => { for (const e of entries) { const r = spawnSync(process.execPath, ['-e', `require(${JSON.stringify(e)})`], { cwd: app, encoding: 'utf8' }); assert.equal(r.status, 0, `${e}: ${r.stderr}`); } }); + +// dedupData.yaml as 2.0.10's middleware wrote it, with js-yaml 3.15.2 and +// with js-yaml 4.3.2 (whichever the app had), before a request and after +// one more (test-set-0/test-3, lines 1 and 4 of index.js run). +const BEFORE = (winKey) => `- id: test-set-0/test-1 + executedLinesByFile: + /home/runner/app/src/index.js: + - 1 + - 2 + - 5 + - 12 + /home/runner/app/src/routes/user routes.js: [] +- id: undefined/test-2 + executedLinesByFile: + ${winKey}: + - 3 + /app/src/ünïcode/日本.js: + - 7 + - 8 +`; +const JS_YAML_3 = BEFORE(String.raw`'C:\Users\dev\app\src\index.js'`); +const JS_YAML_4 = BEFORE(String.raw`C:\Users\dev\app\src\index.js`); +const ADDED = `- id: test-set-0/test-3 + executedLinesByFile: + /home/runner/app/src/index.js: + - 1 + - 4 +`; + +// Runs the installed middleware on one request in a folder holding seed as +// dedupData.yaml, with lines 1 and 4 of each file in files run, and returns +// the file it leaves. +function request(seed, files = ['/home/runner/app/src/index.js']) { + const cwd = mkdtempSync(join(app, 'run-')); + writeFileSync(join(cwd, 'dedupData.yaml'), seed); + const r = spawnSync(process.execPath, ['-e', ` + global.__coverage__ = Object.fromEntries(${JSON.stringify(files)}.map((f) => [f, { statementMap: { 0: { start: { line: 1 } }, 1: { start: { line: 4 } }, 2: { start: { line: 9 } } }, s: { 0: 1, 1: 2, 2: 0 } }])); + const res = new (require('events').EventEmitter)(); + const h = { 'KEPLOY-TEST-ID': 'test-3', 'KEPLOY-TEST-SET-ID': 'test-set-0' }; + require('@keploy/sdk/dist/v2/dedup/middleware')()({ get: (n) => h[n] }, res, () => {}); + res.emit('finish'); + `], { cwd, encoding: 'utf8' }); + assert.equal(r.status, 0, r.stderr); + return readFileSync(join(cwd, 'dedupData.yaml'), 'utf8'); +} + +test('the middleware reads dedupData.yaml as 2.0.10 wrote it, and writes it as js-yaml 4 did', () => { + for (const seed of [JS_YAML_3, JS_YAML_4]) assert.equal(request(seed), JS_YAML_4 + ADDED); + assert.equal(request(''), ADDED); + // A name a YAML 1.1 reader takes for a boolean or a date is quoted, as + // js-yaml quoted it. + const lines = '\n - 1\n - 4\n'; + assert.equal(request('', ['no', '2026-11-04']), `- id: test-set-0/test-3\n executedLinesByFile:\n 'no':${lines} '2026-11-04':${lines}`); +}); + +test('the middleware loads no js/ tag as code', () => { + const file = request("- id: a/b\n executedLinesByFile: !!js/function 'function () { return 42 }'\n- !!js/regexp /x/\n- !!js/undefined ''\n"); + assert.equal(file, "- id: a/b\n executedLinesByFile: function () { return 42 }\n- /x/\n- ''\n" + ADDED); +}); diff --git a/test/workflows.test.mjs b/test/workflows.test.mjs index 5f82337..5277a47 100644 --- a/test/workflows.test.mjs +++ b/test/workflows.test.mjs @@ -1,27 +1,55 @@ -// Rows over the publish workflows themselves: each publishes the tarball its -// license check passed, and no job can hold a runner for GitHub's 6 hours. +// Rows over the workflows that publish to npm: each publishes the tarball its +// license check passed, and nothing in it skips that check or these rows; +// every pull request that changes one of them, these rows or the check runs +// these rows; and every job, those of a workflow one calls included, has a +// timeout, so none can hold a runner for GitHub's 6 hours. import { test } from 'node:test'; import assert from 'node:assert/strict'; -import { readFileSync } from 'node:fs'; +import { readdirSync, readFileSync } from 'node:fs'; import { createRequire } from 'node:module'; -const yaml = createRequire(import.meta.url)('js-yaml'); -const load = (name) => yaml.load(readFileSync(new URL(`../.github/workflows/${name}`, import.meta.url), 'utf8')); +const YAML = createRequire(import.meta.url)('yaml'); +const dir = new URL('../.github/workflows/', import.meta.url); +const text = (name) => readFileSync(new URL(name, dir), 'utf8'); +const load = (name) => YAML.parse(text(name)); const TARBALL = '${{ steps.pack.outputs.tarball }}'; -// The steps of job, in order: each one's name (or command), run, env and id. +// The steps of job, in order: each one's name (or command), run, env, id and +// the keys that can skip it or let it fail. function steps(job) { - return job.steps.map((s) => ({ name: s.name || s.run || s.uses, run: s.run || '', env: s.env || {}, id: s.id })); + return job.steps.map((s) => ({ + name: s.name || s.run || s.uses, run: s.run || '', env: s.env || {}, id: s.id, + if: s.if, continueOnError: s['continue-on-error'], cwd: s['working-directory'], + })); } +// Whether GitHub's paths filter pattern p matches path: ** matches any +// characters, * any but a slash. +function matches(p, path) { + const re = p.split(/(\*\*|\*)/).map((s) => (s === '**' ? '.*' : s === '*' ? '[^/]*' : s.replace(/[.+?^${}()|[\]\\]/g, '\\$&'))).join(''); + return new RegExp(`^${re}$`).test(path); +} + +// What runs these rows from the repository's root in each job: the root's +// own rows (this file among them), or, in a job that works in a package's +// folder, the root's install (no scripts: only the YAML parser is needed) and +// this file. +const ALL_ROWS = { name: 'Rows', run: 'node --test test/*.test.mjs', cwd: undefined }; +const WORKFLOW_ROWS = { name: 'Workflow rows', run: 'npm ci --ignore-scripts\nnode --test test/workflows.test.mjs\n', cwd: '.' }; + const workflows = [ - // [file, publishing job, pull request job, the gate's path from the job's folder] - ['sdk-publish.yml', 'publish', 'check', 'scripts/license-gate.mjs'], - ['playwright-publish.yml', 'publish', 'dry-run', '../scripts/license-gate.mjs'], - ['jest-publish.yml', 'publish', 'dry-run', '../scripts/license-gate.mjs'], + // [file, tag, folder, publishing job, pull request job, the rows step, the gate's path from the folder] + ['sdk-publish.yml', 'sdk-v*', undefined, 'publish', 'check', ALL_ROWS, 'scripts/license-gate.mjs'], + ['playwright-publish.yml', 'playwright-v*', 'playwright', 'publish', 'dry-run', WORKFLOW_ROWS, '../scripts/license-gate.mjs'], + ['jest-publish.yml', 'jest-v*', 'jest', 'publish', 'dry-run', WORKFLOW_ROWS, '../scripts/license-gate.mjs'], ]; -for (const [file, publish, pr, gate] of workflows) { +test('every workflow that publishes to npm is one these rows hold', () => { + const publishing = readdirSync(dir).filter((n) => /\.ya?ml$/.test(n) && /\bnpm publish\b/.test(text(n))).sort(); + assert.deepEqual(publishing, workflows.map(([file]) => file).sort()); +}); + +for (const [file, tag, folder, publish, pr, rows, gate] of workflows) { test(`${file}: the tarball packed once is license-checked, then published`, () => { const wf = load(file); for (const job of [publish, pr]) { @@ -46,11 +74,64 @@ for (const [file, publish, pr, gate] of workflows) { assert.equal(steps(wf.jobs[publish]).find((x) => x.name === 'Publish').run, 'npm publish "$TARBALL" --access public --provenance'); }); - test(`${file}: every job that runs on a runner has a timeout`, () => { + test(`${file}: nothing skips the license check or the rows, or lets them fail`, () => { const wf = load(file); - for (const [name, job] of Object.entries(wf.jobs)) { - if (job.uses) continue; // a called workflow's jobs carry their own + for (const [job, event] of [[publish, 'push'], [pr, 'pull_request']]) { + const j = wf.jobs[job]; + // The job runs on its event, and on nothing narrower. + assert.equal(j.if, `github.event_name == '${event}'`, `${job}: if`); + assert.equal(j['continue-on-error'], undefined, `${job}: continue-on-error`); + assert.equal(j.defaults.run['working-directory'], folder, `${job}: its folder`); + const s = steps(j); + for (const x of s) { + assert.equal(x.if, undefined, `${job}: ${x.name}: if`); + assert.equal(x.continueOnError, undefined, `${job}: ${x.name}: continue-on-error`); + } + const r = s.findIndex((x) => x.name === rows.name); + assert.ok(r >= 0, `${job}: a ${rows.name} step`); + assert.deepEqual([s[r].run, s[r].cwd], [rows.run, rows.cwd], `${job}: ${rows.name} runs from the root`); + assert.ok(r < s.findIndex((x) => x.name === 'Licenses'), `${job}: ${rows.name} before Licenses`); + } + // A tag of this package's name starts the publish; any pull request that + // touches what its paths name starts the check. + assert.deepEqual(wf.on.push, { tags: [tag] }); + assert.deepEqual(Object.keys(wf.on.pull_request), ['paths']); + assert.ok(wf.on.pull_request.paths.every((p) => !p.startsWith('!')), 'no path is excluded'); + }); + + test(`${file}: a pull request that changes it, or the check, runs its own check and these rows`, () => { + const own = load(file).on.pull_request.paths; + const root = load('sdk-publish.yml').on.pull_request.paths; + for (const path of [`.github/workflows/${file}`, 'scripts/license-gate.mjs', ...(folder ? [`${folder}/package.json`] : [])]) { + assert.ok(own.some((p) => matches(p, path)), `${file} runs on a change to ${path}`); + } + // sdk-publish.yml's check runs every root row, these among them. + for (const path of [`.github/workflows/${file}`, 'test/workflows.test.mjs', 'scripts/license-gate.mjs', 'package.json', 'package-lock.json']) { + assert.ok(root.some((p) => matches(p, path)), `sdk-publish.yml runs on a change to ${path}`); + } + }); + + test(`${file}: every job, a called workflow's included, has a timeout`, () => { + const jobs = (wf, from) => Object.entries(wf.jobs).flatMap(([name, job]) => { + if (!job.uses) return [[`${from}: ${name}`, job]]; + const called = job.uses.match(/^\.\/\.github\/workflows\/([^/@]+)$/); + assert.ok(called, `${from}: ${name}: calls a workflow of this repository (${job.uses})`); + return jobs(load(called[1]), called[1]); + }); + const all = jobs(load(file), file); + assert.ok(all.length >= 2); + for (const [name, job] of all) { assert.ok(Number.isInteger(job['timeout-minutes']) && job['timeout-minutes'] <= 60, `${name}: timeout-minutes`); } }); } + +test('the paths filter matches as GitHub does', () => { + assert.ok(matches('scripts/**', 'scripts/license-gate.mjs')); + assert.ok(matches('v2/**', 'v2/dedup/middleware.ts')); + assert.ok(matches('.github/workflows/jest-*.yml', '.github/workflows/jest-sdk.yml')); + assert.ok(!matches('.github/workflows/*.yml', '.github/workflows/sub/x.yml')); + assert.ok(!matches('scripts/**', 'test/scripts/x.mjs')); + assert.ok(!matches('package.json', 'jest/package.json')); + assert.ok(!matches('a.b', 'axb')); +}); diff --git a/v2/dedup/middleware.ts b/v2/dedup/middleware.ts index b13ca9e..c5ab6e9 100644 --- a/v2/dedup/middleware.ts +++ b/v2/dedup/middleware.ts @@ -2,7 +2,10 @@ import { log } from "console"; import { Request, Response, NextFunction } from "express"; const fs = require('fs'); -const yaml = require('js-yaml'); +// yaml (ISC, no dependencies). The file is written as js-yaml 4 wrote it +// (single quotes, and YAML 1.1's quoting of y, no and dates); the files +// js-yaml 3 and 4 wrote are read the same; no js/ tag loads as code. +const yaml = require('yaml'); const filePath = 'dedupData.yaml'; @@ -43,7 +46,7 @@ export function afterMiddleware(req: Request, res: Response) { try { const fileContent = fs.readFileSync(filePath, 'utf-8'); - existingData = yaml.load(fileContent) || []; + existingData = yaml.parse(fileContent) || []; } catch (error) { // Handle the case where the file doesn't exist or is not valid YAML console.error("Error reading existing file:", error); @@ -58,7 +61,7 @@ export function afterMiddleware(req: Request, res: Response) { existingData.push(currentData); // Convert the array to YAML format - const yamlData = yaml.dump(existingData); + const yamlData = yaml.stringify(existingData, { singleQuote: true, version: '1.1' }); // Write the updated YAML data back to the file fs.writeFileSync(filePath, yamlData, 'utf-8'); From dd3971f1b9fb169beba73c35b0df56287e01f5f6 Mon Sep 17 00:00:00 2001 From: slayerjain Date: Wed, 4 Nov 2026 21:23:08 +0530 Subject: [PATCH 7/9] fix: append dedup entries without a YAML package; gate musl installs The dedup middleware read, parsed, re-serialised and rewrote the whole dedupData.yaml on every request, so each request cost time and memory in proportion to a file that only grows, with the event loop blocked. It now appends the new entry's text, written by our own code (v2/dedup/entry.ts) as js-yaml 4's dump wrote it, and needs no YAML package: yaml leaves the dependencies (it stays a devDependency for the workflow rows), and nothing new joins the installed tree. Rows: one request on a file of 1000 entries makes one appendFileSync of its own entry's bytes and reads nothing; 2.0.10's files, as js-yaml 3 and 4 wrote them, keep their bytes and both readers read back every entry; over 335 names, ids and lines the entry is js-yaml 4's text but where entry.ts's limits say, and both readers read the same values. The license gate installs for musl Linux too: linux x64 and arm64 each with glibc and with musl, --libc passed on every linux leg, so a red optional dependency for one C library only is found whatever npm_config_libc says. A row covers both C libraries under three settings. The workflow rows pin each Licenses step to its name, TARBALL and its command. Signed-off-by: slayerjain --- package-lock.json | 3 +- package.json | 6 +- scripts/license-gate.mjs | 24 ++++--- test/license-gate-cli.test.mjs | 23 ++++-- test/license-gate.test.mjs | 5 +- test/package.test.mjs | 126 ++++++++++++++++++++++++++++----- test/workflows.test.mjs | 5 ++ v2/dedup/entry.ts | 76 ++++++++++++++++++++ v2/dedup/middleware.ts | 37 ++-------- 9 files changed, 236 insertions(+), 69 deletions(-) create mode 100644 v2/dedup/entry.ts diff --git a/package-lock.json b/package-lock.json index 72648d3..cbfbb2f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -568,7 +568,8 @@ "yaml": { "version": "2.9.1", "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.1.tgz", - "integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==" + "integrity": "sha512-3NxN8+78OdzbT7C/WjGsyfPAtJaN3FNDsWxv7Y7mcDsT/oOmgW8BpyQQFFBnvZE3j9Y2Sdz1ULFLezL7Eb2yFw==", + "dev": true } } } diff --git a/package.json b/package.json index ec233e6..ec0280c 100644 --- a/package.json +++ b/package.json @@ -32,12 +32,12 @@ "cors": "^2.8.5", "lodash": "^4.17.21", "require-in-the-middle": "^7.3.0", - "tree-kill": "^1.2.2", - "yaml": "^2.9.1" + "tree-kill": "^1.2.2" }, "devDependencies": { "@types/axios": "^0.14.0", "@types/cors": "^2.8.17", - "typescript": "^5.3.3" + "typescript": "^5.3.3", + "yaml": "^2.9.1" } } diff --git a/scripts/license-gate.mjs b/scripts/license-gate.mjs index a92e516..371798d 100644 --- a/scripts/license-gate.mjs +++ b/scripts/license-gate.mjs @@ -4,7 +4,7 @@ // The check installs it into an empty project the way a user's `npm install` // does, with install scripts off (no devDependencies; optional and peer // dependencies in, whatever an npmrc or an npm_config_ variable says), once -// for each platform in PLATFORMS (npm's --os and --cpu), so that a dependency +// for each platform in PLATFORMS (npm's --os, --cpu and --libc), so that a dependency // npm installs only on one of those platforms is checked too. It refuses the package // when anything those installs put on disk for it names a license outside the // allow-list below: @@ -44,8 +44,6 @@ // unless it also names a flagged id, English name or URL. // - only gzip is decompressed, one level deep: zip, xz, bzip2 and brotli // data, and UTF-16 text with no byte order mark, are read as their bytes. -// - npm cannot install for another C library: an optional dependency for -// musl (Alpine) Linux only is not installed, so not checked. // - only the platforms in PLATFORMS are installed for: an optional // dependency only for another (win32/arm64, linux/arm, ia32, the BSDs, // AIX, say) is not installed, so not checked. @@ -70,9 +68,12 @@ export const ALLOWED = new Set([ export const USER_PEERS = new Set(['jest', 'jest-environment-node', '@playwright/test']); // The platforms the tarball is installed for: the runner's, and those of -// other users, as npm's --os and --cpu name them. +// other users, as npm's --os, --cpu and (on Linux) --libc name them: glibc +// and musl (Alpine) Linux both, so that an optional dependency for one C +// library only (sharp's LGPL libvips for musl, say) is checked. export const PLATFORMS = [ - ['linux', 'x64'], ['linux', 'arm64'], ['darwin', 'arm64'], ['darwin', 'x64'], ['win32', 'x64'], + ['linux', 'x64', 'glibc'], ['linux', 'x64', 'musl'], ['linux', 'arm64', 'glibc'], ['linux', 'arm64', 'musl'], + ['darwin', 'arm64'], ['darwin', 'x64'], ['win32', 'x64'], ]; // Text as it is matched: lower case, with each run of spaces, hyphens (any @@ -402,18 +403,19 @@ export async function check(project, root) { } // Installs tarball into the empty project, as a user's `npm install` does on -// platform ([os, cpu]) (npm never installs a dependency's devDependencies; +// platform ([os, cpu, libc]) (npm never installs a dependency's devDependencies; // no install script runs; the flags put optional and peer dependencies in -// over any npmrc or npm_config_ variable), and returns the package's name: the one +// over any npmrc or npm_config_ variable, the C library too), and returns the package's name: the one // dependency npm saved to the project's package.json. Throws, saying why, on // a failure. -export function install(tarball, project, [os, cpu] = PLATFORMS[0]) { +export function install(tarball, project, platform = PLATFORMS[0]) { + const [os, cpu, libc] = platform; writeFileSync(join(project, 'package.json'), JSON.stringify({ name: 'license-gate', version: '0.0.0', private: true })); - const args = ['install', '--ignore-scripts', '--include=optional', '--include=peer', '--legacy-peer-deps=false', '--no-audit', '--no-fund', '--no-package-lock', `--os=${os}`, `--cpu=${cpu}`, resolve(tarball)]; + const args = ['install', '--ignore-scripts', '--include=optional', '--include=peer', '--legacy-peer-deps=false', '--no-audit', '--no-fund', '--no-package-lock', `--os=${os}`, `--cpu=${cpu}`, ...(libc ? [`--libc=${libc}`] : []), resolve(tarball)]; const r = spawnSync('npm', args, { cwd: project, encoding: 'utf8' }); - if (r.status !== 0) throw new Error(`npm install ${tarball} for ${os}/${cpu} failed (${r.error ? r.error.message : `exit ${r.status}`}): ${(r.stderr || '').trim()}`); + if (r.status !== 0) throw new Error(`npm install ${tarball} for ${platform.join('/')} failed (${r.error ? r.error.message : `exit ${r.status}`}): ${(r.stderr || '').trim()}`); const names = needs(JSON.parse(readFileSync(join(project, 'package.json'), 'utf8')), ['dependencies']); - if (names.length !== 1) throw new Error(`installing ${tarball} for ${os}/${cpu} saved ${names.length} packages to the project, not 1`); + if (names.length !== 1) throw new Error(`installing ${tarball} for ${platform.join('/')} saved ${names.length} packages to the project, not 1`); return names[0]; } diff --git a/test/license-gate-cli.test.mjs b/test/license-gate-cli.test.mjs index 80fd245..70f44db 100644 --- a/test/license-gate-cli.test.mjs +++ b/test/license-gate-cli.test.mjs @@ -35,7 +35,7 @@ function run(args, env = {}, cwd = undefined) { } const quote = (s) => s.replace(/[.*+?^${}()|[\]\\]/g, '\\$&'); -const counts = (n) => ['linux/x64', 'linux/arm64', 'darwin/arm64', 'darwin/x64', 'win32/x64'].map((p) => `${p}: ${n} packages, \\d+ files`).join('; '); +const counts = (n) => ['linux/x64/glibc', 'linux/x64/musl', 'linux/arm64/glibc', 'linux/arm64/musl', 'darwin/arm64', 'darwin/x64', 'win32/x64'].map((p) => `${p}: ${n} packages, \\d+ files`).join('; '); test('a clean package passes on every platform, and the projects it was installed into are gone', () => { const tgz = pack('clean', { 'package.json': { name: 'clean', version: '1.0.0', license: 'Apache-2.0' }, 'index.js': 'module.exports = 1;' }); @@ -101,10 +101,25 @@ test('optional dependencies, those of other platforms, and the peers of dependen ' node_modules/redpeer (redpeer@1.0.0): license "EPL-2.0" is not allowed (field "EPL-2.0")', ' node_modules/redwin (redwin@1.0.0): license "MPL-2.0" is not allowed (field "MPL-2.0") (on win32/x64)', ]); - assert.match(r.stderr, /\nchecked linux\/x64: 4 packages, \d+ files; linux\/arm64: 4 packages, \d+ files; darwin\/arm64: 5 packages, \d+ files; darwin\/x64: 4 packages, \d+ files; win32\/x64: 5 packages, \d+ files; not checked, the user's own: 1 packages its peerDependencies install \(jest@1\.0\.0\)\n$/); + assert.match(r.stderr, /\nchecked linux\/x64\/glibc: 4 packages, \d+ files; linux\/x64\/musl: 4 packages, \d+ files; linux\/arm64\/glibc: 4 packages, \d+ files; linux\/arm64\/musl: 4 packages, \d+ files; darwin\/arm64: 5 packages, \d+ files; darwin\/x64: 4 packages, \d+ files; win32\/x64: 5 packages, \d+ files; not checked, the user's own: 1 packages its peerDependencies install \(jest@1\.0\.0\)\n$/); assert.deepEqual(r.left, []); }); +test('an optional dependency for one C library only is installed and checked, whatever npm_config_libc says', () => { + const dep = (name, extra) => pack(name, { 'package.json': { name, version: '1.0.0', ...extra } }); + const musl = dep('redmusl', { license: 'LGPL-3.0-or-later', os: ['linux'], libc: ['musl'] }); + const glibc = dep('redglibc', { license: 'GPL-2.0', os: ['linux'], libc: ['glibc'] }); + const tgz = pack('libctop', { 'package.json': { name: 'libctop', version: '1.0.0', license: 'MIT', optionalDependencies: { redmusl: `file:${musl}`, redglibc: `file:${glibc}` } } }); + for (const env of [{}, { npm_config_libc: 'musl' }, { npm_config_libc: 'glibc' }]) { + const r = run([tgz], env); + assert.equal(r.status, 1, r.stderr); + assert.deepEqual(r.stderr.split('\n').filter((l) => / is not allowed /.test(l)).sort(), [ + ' node_modules/redglibc (redglibc@1.0.0): license "GPL-2.0" is not allowed (field "GPL-2.0") (on linux/x64/glibc, linux/arm64/glibc)', + ' node_modules/redmusl (redmusl@1.0.0): license "LGPL-3.0-or-later" is not allowed (field "LGPL-3.0-or-later") (on linux/x64/musl, linux/arm64/musl)', + ], JSON.stringify(env)); + } +}); + test('an npm config that leaves optional or peer dependencies out changes nothing', () => { const dep = (name, extra) => pack(name, { 'package.json': { name, version: '1.0.0', ...extra } }); const opt = dep('envopt', { license: 'GPL-3.0' }); @@ -138,7 +153,7 @@ test('a file that is not a package fails at the install', () => { writeFileSync(bad, 'not a tarball'); const r = run([bad]); assert.equal(r.status, 1); - assert.match(r.stderr, /^license gate: npm install .*not-a-package\.tgz for linux\/x64 failed \(exit \d+\): /); + assert.match(r.stderr, /^license gate: npm install .*not-a-package\.tgz for linux\/x64\/glibc failed \(exit \d+\): /); assert.deepEqual(r.left, []); }); @@ -158,6 +173,6 @@ test('an install that puts nothing on disk fails: nothing checked is not clean', const tgz = pack('dry', { 'package.json': { name: 'dry', version: '1.0.0', license: 'MIT' } }); const r = run([tgz], { npm_config_dry_run: 'true' }); assert.equal(r.status, 1); - assert.equal(r.stderr, `license gate: installing ${tgz} for linux/x64 saved 0 packages to the project, not 1\n`); + assert.equal(r.stderr, `license gate: installing ${tgz} for linux/x64/glibc saved 0 packages to the project, not 1\n`); assert.deepEqual(r.left, []); }); diff --git a/test/license-gate.test.mjs b/test/license-gate.test.mjs index 42f0c68..d8b9661 100644 --- a/test/license-gate.test.mjs +++ b/test/license-gate.test.mjs @@ -370,5 +370,8 @@ test('a clean tree has no findings; a package that is not there is one', async ( }); test('the tarball is installed for the runner and for other users\' platforms', () => { - assert.deepEqual(PLATFORMS, [['linux', 'x64'], ['linux', 'arm64'], ['darwin', 'arm64'], ['darwin', 'x64'], ['win32', 'x64']]); + assert.deepEqual(PLATFORMS, [ + ['linux', 'x64', 'glibc'], ['linux', 'x64', 'musl'], ['linux', 'arm64', 'glibc'], ['linux', 'arm64', 'musl'], + ['darwin', 'arm64'], ['darwin', 'x64'], ['win32', 'x64'], + ]); }); diff --git a/test/package.test.mjs b/test/package.test.mjs index 6b76417..0038e36 100644 --- a/test/package.test.mjs +++ b/test/package.test.mjs @@ -4,7 +4,7 @@ import { test } from 'node:test'; import assert from 'node:assert/strict'; import { spawnSync } from 'node:child_process'; import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs'; -import { builtinModules } from 'node:module'; +import { builtinModules, createRequire } from 'node:module'; import { tmpdir } from 'node:os'; import { join } from 'node:path'; import { fileURLToPath } from 'node:url'; @@ -28,14 +28,18 @@ test('every package the built code requires is a dependency', () => { } } assert.deepEqual([...required].filter((n) => !(n in pkg.dependencies)), []); - // The two the dedup entry points need, which 2.0.10 did not declare. - for (const n of ['body-parser', 'yaml']) assert.ok(required.has(n), n); + // The one the dedup entry points need, which 2.0.10 did not declare; and + // no YAML package: the middleware writes its entries with our own code. + assert.ok(required.has('body-parser')); + for (const n of ['yaml', 'js-yaml']) assert.ok(!required.has(n) && !(n in pkg.dependencies), n); }); // A clean install of the packed package, as a user's app gets it. const work = mkdtempSync(join(tmpdir(), 'sdk-install-')); test.after(() => rmSync(work, { recursive: true, force: true })); const app = join(work, 'app'); +const readers = join(work, 'readers'); +const reader = (v) => createRequire(join(readers, 'package.json'))(`js-yaml-${v}`); test.before(() => { const packed = spawnSync('npm', ['pack', '--ignore-scripts', '--pack-destination', work], { cwd: root, encoding: 'utf8' }); assert.equal(packed.status, 0, packed.stderr); @@ -44,6 +48,11 @@ test.before(() => { writeFileSync(join(app, 'package.json'), JSON.stringify({ name: 'app', version: '0.0.0', private: true })); const inst = spawnSync('npm', ['install', '--omit=dev', '--ignore-scripts', '--no-audit', '--no-fund', tgz], { cwd: app, encoding: 'utf8' }); assert.equal(inst.status, 0, inst.stderr); + // The YAML readers a user's app may have had: js-yaml 3 (nyc's) and 4. + mkdirSync(readers); + writeFileSync(join(readers, 'package.json'), JSON.stringify({ name: 'readers', version: '0.0.0', private: true })); + const r = spawnSync('npm', ['install', '--ignore-scripts', '--no-audit', '--no-fund', '--no-package-lock', 'js-yaml-3@npm:js-yaml@3.15.2', 'js-yaml-4@npm:js-yaml@4.3.2'], { cwd: readers, encoding: 'utf8' }); + assert.equal(r.status, 0, r.stderr); }); test('the entry points load from a clean install of the packed package', () => { @@ -51,6 +60,7 @@ test('the entry points load from a clean install of the packed package', () => { const r = spawnSync(process.execPath, ['-e', `require(${JSON.stringify(e)})`], { cwd: app, encoding: 'utf8' }); assert.equal(r.status, 0, `${e}: ${r.stderr}`); } + for (const n of ['yaml', 'js-yaml', 'esprima']) assert.ok(!existsSync(join(app, 'node_modules', n)), `${n} is installed`); }); // dedupData.yaml as 2.0.10's middleware wrote it, with js-yaml 3.15.2 and @@ -81,25 +91,37 @@ const ADDED = `- id: test-set-0/test-3 - 4 `; +// The middleware's request: test-set-0/test-3, with lines 1 and 4 of each +// file in files run. +const REQUEST = (files) => ` + global.__coverage__ = Object.fromEntries(${JSON.stringify(files)}.map((f) => [f, { statementMap: { 0: { start: { line: 1 } }, 1: { start: { line: 4 } }, 2: { start: { line: 9 } } }, s: { 0: 1, 1: 2, 2: 0 } }])); + const res = new (require('events').EventEmitter)(); + const h = { 'KEPLOY-TEST-ID': 'test-3', 'KEPLOY-TEST-SET-ID': 'test-set-0' }; + require('@keploy/sdk/dist/v2/dedup/middleware')()({ get: (n) => h[n] }, res, () => {}); +`; + // Runs the installed middleware on one request in a folder holding seed as -// dedupData.yaml, with lines 1 and 4 of each file in files run, and returns -// the file it leaves. +// dedupData.yaml, and returns the file it leaves. function request(seed, files = ['/home/runner/app/src/index.js']) { const cwd = mkdtempSync(join(app, 'run-')); writeFileSync(join(cwd, 'dedupData.yaml'), seed); - const r = spawnSync(process.execPath, ['-e', ` - global.__coverage__ = Object.fromEntries(${JSON.stringify(files)}.map((f) => [f, { statementMap: { 0: { start: { line: 1 } }, 1: { start: { line: 4 } }, 2: { start: { line: 9 } } }, s: { 0: 1, 1: 2, 2: 0 } }])); - const res = new (require('events').EventEmitter)(); - const h = { 'KEPLOY-TEST-ID': 'test-3', 'KEPLOY-TEST-SET-ID': 'test-set-0' }; - require('@keploy/sdk/dist/v2/dedup/middleware')()({ get: (n) => h[n] }, res, () => {}); - res.emit('finish'); - `], { cwd, encoding: 'utf8' }); + const r = spawnSync(process.execPath, ['-e', REQUEST(files) + "res.emit('finish');"], { cwd, encoding: 'utf8' }); assert.equal(r.status, 0, r.stderr); return readFileSync(join(cwd, 'dedupData.yaml'), 'utf8'); } -test('the middleware reads dedupData.yaml as 2.0.10 wrote it, and writes it as js-yaml 4 did', () => { - for (const seed of [JS_YAML_3, JS_YAML_4]) assert.equal(request(seed), JS_YAML_4 + ADDED); +test('the middleware appends to dedupData.yaml as 2.0.10 wrote it, and js-yaml 3 and 4 read it back the same', () => { + const entry = (id, executedLinesByFile) => ({ id, executedLinesByFile }); + const want = [ + entry('test-set-0/test-1', { '/home/runner/app/src/index.js': [1, 2, 5, 12], '/home/runner/app/src/routes/user routes.js': [] }), + entry('undefined/test-2', { [String.raw`C:\Users\dev\app\src\index.js`]: [3], '/app/src/ünïcode/日本.js': [7, 8] }), + entry('test-set-0/test-3', { '/home/runner/app/src/index.js': [1, 4] }), + ]; + for (const seed of [JS_YAML_3, JS_YAML_4]) { + const file = request(seed); + assert.equal(file, seed + ADDED); + for (const v of [3, 4]) assert.deepStrictEqual(reader(v).load(file), want, `js-yaml ${v}`); + } assert.equal(request(''), ADDED); // A name a YAML 1.1 reader takes for a boolean or a date is quoted, as // js-yaml quoted it. @@ -107,7 +129,77 @@ test('the middleware reads dedupData.yaml as 2.0.10 wrote it, and writes it as j assert.equal(request('', ['no', '2026-11-04']), `- id: test-set-0/test-3\n executedLinesByFile:\n 'no':${lines} '2026-11-04':${lines}`); }); -test('the middleware loads no js/ tag as code', () => { - const file = request("- id: a/b\n executedLinesByFile: !!js/function 'function () { return 42 }'\n- !!js/regexp /x/\n- !!js/undefined ''\n"); - assert.equal(file, "- id: a/b\n executedLinesByFile: function () { return 42 }\n- /x/\n- ''\n" + ADDED); +test('the middleware leaves what the file holds as it is, and loads none of it', () => { + const tags = "- id: a/b\n executedLinesByFile: !!js/function 'function () { return 42 }'\n- !!js/regexp /x/\n- !!js/undefined ''\n"; + assert.equal(request(tags), tags + ADDED); +}); + +test('on a file of 1000 entries, a request reads nothing from it and writes only its own entry', () => { + const cwd = mkdtempSync(join(app, 'big-')); + // 1000 tests, each with 60 lines run in each of 40 files: 28 MiB. + const body = Array.from({ length: 40 }, (_, f) => ` /home/runner/app/src/file-${f}.js:\n` + Array.from({ length: 60 }, (_, l) => ` - ${l + 1}\n`).join('')).join(''); + const seed = Array.from({ length: 1000 }, (_, t) => `- id: test-set-0/test-${t}\n executedLinesByFile:\n${body}`).join(''); + writeFileSync(join(cwd, 'dedupData.yaml'), seed); + // Every call into fs the request makes (not those one of them makes), with + // its first argument and the size of its second. + const r = spawnSync(process.execPath, ['-e', REQUEST(['/home/runner/app/src/index.js']) + ` + const fs = require('fs'); + const calls = []; + const saved = {}; + let depth = 0; + for (const k of Object.keys(fs)) { + if (typeof fs[k] !== 'function' || /^[A-Z]/.test(k)) continue; + const f = (saved[k] = fs[k]); + fs[k] = function (...a) { + if (depth === 0) calls.push([k, a[0], typeof a[1] === 'string' ? Buffer.byteLength(a[1]) : a[1] && a[1].length]); + depth++; + try { return f.apply(this, a); } finally { depth--; } + }; + } + res.emit('finish'); + Object.assign(fs, saved); + process.stdout.write(JSON.stringify(calls)); + `], { cwd, encoding: 'utf8' }); + assert.equal(r.status, 0, r.stderr); + assert.deepEqual(JSON.parse(r.stdout), [['appendFileSync', 'dedupData.yaml', Buffer.byteLength(ADDED)]]); + assert.ok(readFileSync(join(cwd, 'dedupData.yaml'), 'utf8') === seed + ADDED, 'the file is the seed, then the new entry'); +}); + +// Names and ids a coverage tool and a request can give, YAML's own words, +// indicators and escapes among them, and lines. +const NAMES = ['/home/runner/app/src/index.js', String.raw`C:\Users\dev\app\src\index.js`, '/a b/c.js', '/app/ünïcode/日本.js', '/x/😀.js', + 'a#b', 'a #b', 'a: b', 'a:b', 'a:', 'a ', ' a', ' ', ':', '-a', '- a', '?a', ':a', ',a', '[a]', '{a}', '#a', '&a', '*a', '!a', '|a', '=a', '>a', "'a", '"a', '%a', '@a', '`a', + "it's", 'a"b', 'a\\b', 'no', 'No', 'NO', 'yes', 'y', 'n', 'on', 'off', 'OFF', 'true', 'True', 'false', 'null', 'Null', 'NULL', '~', '', '<<', '0o12', '+0o7', '1_000', '+.5', '.5', '08', + '0x1F', '0b101', '1:20', '_:_', '1:20.5', '2026-11-04', '2026-11-04T10:00:00Z', '.inf', '-.inf', '.NaN', '.nan', '1e5', '123', '-1', '...', '---', + 'a\tb', 'a\nb', 'a\rb', '\x00', '\x7f', '\x85', 'a\xa0b', '\u2028', 'a\u2029b', 'a\ufeffb', '\ufeffa', 'a\ud800b', '\ud83d', 'a\x1fb', 'a~b', 'a\xa1b', 'a\ud7ffb', 'a\ue000b', + 'a\ufffdb', 'a\ufffeb', 'a\x80b', 'a\x01b', 'a\t"\\b', '\x1b\x07\x08\x0b\x0c', + 'x'.repeat(1023) + "'", 'x'.repeat(1024), 'x'.repeat(1025), "'".repeat(600), '/p/' + 'q'.repeat(2000)]; +const LINES = [1, 2, 0, 12, 1.5, NaN, Infinity, -Infinity, -0, 1e-7, 1e21, 2 ** 53, -3]; + +test('each entry is the text js-yaml 4 wrote but where the limits say, and js-yaml 3 and 4 read back its values', () => { + const { entryYaml } = createRequire(join(app, 'package.json'))('@keploy/sdk/dist/v2/dedup/entry'); + const short = (s) => JSON.stringify(s).slice(0, 24); + const cases = [ + ...NAMES.flatMap((s) => [[`key ${short(s)}`, 't/1', { [s]: [1, 4] }], [`empty ${short(s)}`, 't/1', { [s]: [] }], [`id ${short(s)}`, s, { '/a.js': [1] }]]), + ...LINES.map((n) => [`line ${n}`, 't/1', { '/a.js': [n] }]), + ['no files', 't/1', {}], ['not numbers', 't/1', { '/a.js': ['5', null, undefined, true] }], + ['three files', 'undefined/test-2', { '/a.js': [1, 2], 'C:\\x.js': [], '/b.js': [3] }], + ...[76, 77].flatMap((n) => [[`id of ${n}`, 'i'.repeat(n), {}], [`id of ${n} after a space`, ' ' + 'i'.repeat(n), {}]]), + ]; + const differ = []; + for (const [label, id, lines] of cases) { + const text = entryYaml(id, lines); + const theirs = reader(4).dump([{ id, executedLinesByFile: lines }]); + for (const v of [3, 4]) { + let got; + assert.doesNotThrow(() => { got = reader(v).load(text); }, `${label}: js-yaml ${v} reads it`); + assert.deepStrictEqual(got, reader(4).load(theirs), `${label}: js-yaml ${v}`); + } + if (text !== theirs) differ.push(label); + } + // The limits: names that start like a number, ids js-yaml 4 writes as a + // block, and lines that are not numbers. + const x = `id "${'x'.repeat(23)}`; + assert.deepEqual(differ, ['key "1_000"', 'empty "1_000"', 'id "1_000"', 'key "+.5"', 'empty "+.5"', 'id "+.5"', 'id "a\\nb"', + x, x, x, `id "${"'".repeat(23)}`, 'id "/p/qqqqqqqqqqqqqqqqqqqq', 'not numbers', 'id of 77']); }); diff --git a/test/workflows.test.mjs b/test/workflows.test.mjs index 5277a47..c681b54 100644 --- a/test/workflows.test.mjs +++ b/test/workflows.test.mjs @@ -63,6 +63,11 @@ for (const [file, tag, folder, publish, pr, rows, gate] of workflows) { assert.ok(lic > pack, `${job}: a Licenses step after Pack`); assert.equal(s[lic].run, `node ${gate} "$TARBALL"`); assert.equal(s[lic].env.TARBALL, TARBALL); + // Nothing else on the step: no shell of its own, and no other variable + // (one that preloads code, say). + const raw = wf.jobs[job].steps[lic]; + assert.deepEqual(Object.keys(raw).sort(), ['env', 'name', 'run'], `${job}: the Licenses step's keys`); + assert.deepEqual(Object.keys(raw.env), ['TARBALL'], `${job}: the Licenses step's env`); // npm packs or publishes nothing else. assert.deepEqual(s.filter((x) => /\bnpm (?:pack|publish)\b/.test(x.run)).map((x) => x.name), job === publish ? ['Pack', 'Publish'] : file === 'sdk-publish.yml' ? ['Pack'] : ['Pack', 'Publish, dry run']); const pub = s.findIndex((x) => /\bnpm publish\b/.test(x.run)); diff --git a/v2/dedup/entry.ts b/v2/dedup/entry.ts new file mode 100644 index 0000000..ca299ca --- /dev/null +++ b/v2/dedup/entry.ts @@ -0,0 +1,76 @@ +// One dedupData.yaml entry as YAML text: the bytes js-yaml 4's dump() gave +// it as an item of the file's list, so that the middleware appends the new +// entry to the file instead of reading, parsing and rewriting the whole file +// on every request. Our own code: no YAML package is needed. +// Limits, each written another way that every YAML 1.1 and 1.2 reader +// (js-yaml 3 and 4 among them) reads back as the same value: +// - a name that starts like a number (a sign, then a digit, or a dot and a +// digit), or is ~, null, <<, .inf, .nan or a YAML 1.1 or 1.2 boolean, in +// any letter case, is single-quoted where js-yaml 4 may leave it plain; +// - an id over 76 characters or with a line break, which js-yaml 4 writes +// as a block, is double-quoted; +// - a line that is not a number (no coverage tool gives one) is JSON. + +const ESCAPES: { [c: number]: string } = { + 0x00: '\\0', 0x07: '\\a', 0x08: '\\b', 0x09: '\\t', 0x0a: '\\n', 0x0b: '\\v', 0x0c: '\\f', 0x0d: '\\r', + 0x1b: '\\e', 0x22: '\\"', 0x5c: '\\\\', 0x85: '\\N', 0xa0: '\\_', 0x2028: '\\L', 0x2029: '\\P', +}; +// js-yaml 4 single-quotes these, for YAML 1.1 readers, at any length. +const BASE60 = /^[-+]?[0-9_]+(?::[0-9_]+)+(?:\.[0-9_]*)?$/; +// Every text a YAML 1.1 or 1.2 reader takes for a null, boolean, number, +// timestamp or merge key matches this, in js-yaml 3 and 4 too. +const NOT_A_STRING = /^(?:~|null|true|false|y|yes|n|no|on|off|<<|[-+]?\.(?:inf|nan))$|^[-+]?\.?[0-9]/i; +// The characters a plain scalar may not start with: space and YAML's indicators. +const NOT_FIRST = ' -?:,[]{}#&*!|=>\'"%@`'; + +// A character js-yaml 4 writes as itself. +function printable(c: number): boolean { + return (c >= 0x20 && c <= 0x7e) || (c >= 0xa1 && c <= 0xd7ff && c !== 0x2028 && c !== 0x2029) || + (c >= 0xe000 && c <= 0xfffd && c !== 0xfeff) || c >= 0x10000; +} + +function escape(c: number): string { + if (ESCAPES[c] !== undefined) return ESCAPES[c]; + if (printable(c)) return String.fromCodePoint(c); + // Every character over U+FFFF is printable, so four digits are enough. + return c <= 0xff ? '\\x' + ('0' + c.toString(16).toUpperCase()).slice(-2) : '\\u' + ('000' + c.toString(16).toUpperCase()).slice(-4); +} + +// s as a scalar. width: the longest one-line value js-yaml 4 writes there, or +// 0 for a mapping key, which it never folds. +export function scalar(s: string, width: number): string { + if (s === '') return "''"; + if (BASE60.test(s)) return "'" + s + "'"; + const chars = Array.from(s, (ch) => ch.codePointAt(0) as number); + if (!chars.every(printable) || (width > 0 && s.length > width && s[0] !== ' ')) { + return '"' + chars.map(escape).join('') + '"'; + } + const plain = !NOT_FIRST.includes(s[0]) && !/[ :]$/.test(s) && !/: | #/.test(s) && !NOT_A_STRING.test(s); + return plain ? s : "'" + s.replace(/'/g, "''") + "'"; +} + +// n as js-yaml 4 writes a number. +export function number(n: number): string { + if (Number.isNaN(n)) return '.nan'; + if (n === Infinity) return '.inf'; + if (n === -Infinity) return '-.inf'; + if (Object.is(n, -0)) return '-0.0'; + const s = String(n); + return n % 1 !== 0 && /^[-+]?[0-9]+e/.test(s) ? s.replace('e', '.e') : s; +} + +// The entry for one request, ending in a line break. +export function entryYaml(id: string, executedLinesByFile: { [filename: string]: unknown[] }): string { + let out = '- id: ' + scalar(id, 76) + '\n executedLinesByFile:'; + const files = Object.keys(executedLinesByFile); + if (files.length === 0) return out + ' {}\n'; + for (const file of files) { + const key = scalar(file, 0); + const items = executedLinesByFile[file].map((n) => '- ' + (typeof n === 'number' ? number(n) : JSON.stringify(n) ?? 'null')); + // A key over 1024 characters is written explicit (? key, then : value), + // as YAML allows no longer implicit key. + if (key.length > 1024) out += '\n ? ' + key + '\n : ' + (items.length ? items.join('\n ') : '[]'); + else out += '\n ' + key + ':' + (items.length ? '\n ' + items.join('\n ') : ' []'); + } + return out + '\n'; +} diff --git a/v2/dedup/middleware.ts b/v2/dedup/middleware.ts index c5ab6e9..2649c72 100644 --- a/v2/dedup/middleware.ts +++ b/v2/dedup/middleware.ts @@ -2,10 +2,7 @@ import { log } from "console"; import { Request, Response, NextFunction } from "express"; const fs = require('fs'); -// yaml (ISC, no dependencies). The file is written as js-yaml 4 wrote it -// (single quotes, and YAML 1.1's quoting of y, no and dates); the files -// js-yaml 3 and 4 wrote are read the same; no js/ tag loads as code. -const yaml = require('yaml'); +import { entryYaml } from "./entry"; const filePath = 'dedupData.yaml'; @@ -37,34 +34,10 @@ export function afterMiddleware(req: Request, res: Response) { } let executedLinesByFile = GetCoverage(); - let currentData = { - id: testSet + "/" + id, - executedLinesByFile: executedLinesByFile - }; - - let existingData = []; - - try { - const fileContent = fs.readFileSync(filePath, 'utf-8'); - existingData = yaml.parse(fileContent) || []; - } catch (error) { - // Handle the case where the file doesn't exist or is not valid YAML - console.error("Error reading existing file:", error); - } - - if (!Array.isArray(existingData)) { - console.error('Expected an array for existingData, but got:', typeof existingData); - existingData = []; // Reset to an empty array or handle accordingly - } - - // Add or update the entry for the current id - existingData.push(currentData); - - // Convert the array to YAML format - const yamlData = yaml.stringify(existingData, { singleQuote: true, version: '1.1' }); - - // Write the updated YAML data back to the file - fs.writeFileSync(filePath, yamlData, 'utf-8'); + // The file is a list that only grows: append this request's entry, as + // js-yaml 4 wrote it, rather than read, parse and rewrite the whole file, + // which cost each request time and memory in proportion to the file. + fs.appendFileSync(filePath, entryYaml(testSet + "/" + id, executedLinesByFile), 'utf-8'); // Log to the console // console.log("Executed lines by file:", executedLinesByFile); From a9b22b84339549d7885265c5237b820677fb2c11 Mon Sep 17 00:00:00 2001 From: slayerjain Date: Wed, 4 Nov 2026 21:49:49 +0530 Subject: [PATCH 8/9] fix: write a long dedup id on one line, as its readers take it entry.ts double-quoted an id over 76 characters, as a stand-in for the folded block js-yaml 4 writes there. No reader needs it: a plain value of any length on one line is valid YAML, and only an implicit key has a length limit (over 1024 characters a key is still written explicit). The rule had no row that could fail on it, so it goes: a long id is plain or quoted by the same rules as a short one. The row over 335 names, ids and lines still shows js-yaml 3 and 4 read every entry back as the same values, with the same list of entries whose text differs from js-yaml 4's. Signed-off-by: slayerjain --- v2/dedup/entry.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/v2/dedup/entry.ts b/v2/dedup/entry.ts index ca299ca..ed5ea14 100644 --- a/v2/dedup/entry.ts +++ b/v2/dedup/entry.ts @@ -7,8 +7,9 @@ // - a name that starts like a number (a sign, then a digit, or a dot and a // digit), or is ~, null, <<, .inf, .nan or a YAML 1.1 or 1.2 boolean, in // any letter case, is single-quoted where js-yaml 4 may leave it plain; -// - an id over 76 characters or with a line break, which js-yaml 4 writes -// as a block, is double-quoted; +// - an id with a line break, which js-yaml 4 writes as a block, is +// double-quoted, and one over 76 characters, which it folds, stays on one +// line; // - a line that is not a number (no coverage tool gives one) is JSON. const ESCAPES: { [c: number]: string } = { @@ -36,13 +37,12 @@ function escape(c: number): string { return c <= 0xff ? '\\x' + ('0' + c.toString(16).toUpperCase()).slice(-2) : '\\u' + ('000' + c.toString(16).toUpperCase()).slice(-4); } -// s as a scalar. width: the longest one-line value js-yaml 4 writes there, or -// 0 for a mapping key, which it never folds. -export function scalar(s: string, width: number): string { +// s as a scalar, on one line. +export function scalar(s: string): string { if (s === '') return "''"; if (BASE60.test(s)) return "'" + s + "'"; const chars = Array.from(s, (ch) => ch.codePointAt(0) as number); - if (!chars.every(printable) || (width > 0 && s.length > width && s[0] !== ' ')) { + if (!chars.every(printable)) { return '"' + chars.map(escape).join('') + '"'; } const plain = !NOT_FIRST.includes(s[0]) && !/[ :]$/.test(s) && !/: | #/.test(s) && !NOT_A_STRING.test(s); @@ -61,11 +61,11 @@ export function number(n: number): string { // The entry for one request, ending in a line break. export function entryYaml(id: string, executedLinesByFile: { [filename: string]: unknown[] }): string { - let out = '- id: ' + scalar(id, 76) + '\n executedLinesByFile:'; + let out = '- id: ' + scalar(id) + '\n executedLinesByFile:'; const files = Object.keys(executedLinesByFile); if (files.length === 0) return out + ' {}\n'; for (const file of files) { - const key = scalar(file, 0); + const key = scalar(file); const items = executedLinesByFile[file].map((n) => '- ' + (typeof n === 'number' ? number(n) : JSON.stringify(n) ?? 'null')); // A key over 1024 characters is written explicit (? key, then : value), // as YAML allows no longer implicit key. From b8f80ba8011ee733d724d6f316568076876cef62 Mon Sep 17 00:00:00 2001 From: slayerjain Date: Thu, 5 Nov 2026 00:43:28 +0530 Subject: [PATCH 9/9] fix: create the dedup file at setup without emptying it; pin job env The dedup middleware's setup checked for dedupData.yaml asynchronously and, when the check found it missing, wrote it empty in its callback. Requests that finished before that callback lost their entries: 40 requests in the setup's tick left 0 bytes. The setup now creates a missing file at once with an append of nothing, which never empties it, and as before writes nothing to a file that exists. Rows run those 40 requests and read back 40 entries, and start an app whose dedupData.yaml is a folder. A name that starts with "._" is now quoted: js-yaml 3 reads plain "._1" as 0.1 and "._e0" as NaN. The differential row covers three such names. The bytes row wraps fs and fs.promises (require('fs/promises') is the same object) before the middleware loads and logs each call as it is made, from the setup to the exit: only the setup's existsSync and the request's append may show. A whole-file read at setup, through promises, deferred, in an exit listener or through a function bound at load fails it. The workflow rows now fail on an env on a publish workflow or any of its jobs; a pack, check or publish job key outside the allowed set (a container, say), a runner other than ubuntu-latest, a defaults.run other than bash and the job's folder, or an action other than the pinned checkout and setup-node with their three inputs (the registry npmjs.org, the scope @keploy); a step env beyond TARBALL, NPM_TOKEN and NODE_AUTH_TOKEN; a rows step key beyond its name, command and folder; and a command that names GITHUB_ENV, GITHUB_PATH, ::set-env or ::add-path, in any case. Each could give the license check's node a NODE_OPTIONS that exits 0. playwright-publish.yml now pins its actions by commit, as the other two publish workflows do. The license gate's header names no unreleased tool. Signed-off-by: slayerjain --- .github/workflows/playwright-publish.yml | 12 ++- scripts/license-gate.mjs | 2 +- test/package.test.mjs | 111 ++++++++++++++++++----- test/workflows.test.mjs | 34 ++++++- v2/dedup/entry.ts | 11 ++- v2/dedup/middleware.ts | 12 +-- 6 files changed, 140 insertions(+), 42 deletions(-) diff --git a/.github/workflows/playwright-publish.yml b/.github/workflows/playwright-publish.yml index 1824dc9..de875ef 100644 --- a/.github/workflows/playwright-publish.yml +++ b/.github/workflows/playwright-publish.yml @@ -7,7 +7,9 @@ name: "@keploy/playwright publish" # playwright-vX.Y.Z on that commit. The rows run on the tagged commit first; # the publish runs only when every leg passes, and refuses a tag that does not # name package.json's version (scripts/release-version.mjs). npm itself -# refuses a version it already has. +# refuses a version it already has. Every action the publish and dry-run +# jobs run is pinned by commit: the publish job holds the npm token and +# signs the package's provenance. # # Push playwright-vX.Y.Z as a plain git tag, never as a GitHub release: a # published GitHub release also starts any workflow of this repository that @@ -57,8 +59,8 @@ jobs: shell: bash # -eo pipefail: a failed command before a pipe fails the step working-directory: playwright steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 # Writes the token line for registry.npmjs.org and pins the @keploy @@ -130,8 +132,8 @@ jobs: shell: bash # -eo pipefail: a failed command before a pipe fails the step working-directory: playwright steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 + - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0 + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 with: node-version: 22 - name: Workflow rows diff --git a/scripts/license-gate.mjs b/scripts/license-gate.mjs index 371798d..5561540 100644 --- a/scripts/license-gate.mjs +++ b/scripts/license-gate.mjs @@ -31,7 +31,7 @@ // Accepting a flagged file with a notice is not enough, so there is no // exception list: a finding fails the publish. // -// Limits, until licensegate's npm mode replaces this script: +// Limits, until a fuller license scanner replaces this script: // - a license that FLAGGED does not name passes when only a file's text // names it (the Artistic License, say); a license field naming it fails. // - the id Unicode-3.0 is matched only in an SPDX-License-Identifier line diff --git a/test/package.test.mjs b/test/package.test.mjs index 0038e36..d99502a 100644 --- a/test/package.test.mjs +++ b/test/package.test.mjs @@ -91,10 +91,14 @@ const ADDED = `- id: test-set-0/test-3 - 4 `; +// The coverage of a run with lines 1 and 4 of each file in files run. +const COVERAGE = (files) => ` + global.__coverage__ = Object.fromEntries(${JSON.stringify(files)}.map((f) => [f, { statementMap: { 0: { start: { line: 1 } }, 1: { start: { line: 4 } }, 2: { start: { line: 9 } } }, s: { 0: 1, 1: 2, 2: 0 } }])); +`; + // The middleware's request: test-set-0/test-3, with lines 1 and 4 of each // file in files run. -const REQUEST = (files) => ` - global.__coverage__ = Object.fromEntries(${JSON.stringify(files)}.map((f) => [f, { statementMap: { 0: { start: { line: 1 } }, 1: { start: { line: 4 } }, 2: { start: { line: 9 } } }, s: { 0: 1, 1: 2, 2: 0 } }])); +const REQUEST = (files) => COVERAGE(files) + ` const res = new (require('events').EventEmitter)(); const h = { 'KEPLOY-TEST-ID': 'test-3', 'KEPLOY-TEST-SET-ID': 'test-set-0' }; require('@keploy/sdk/dist/v2/dedup/middleware')()({ get: (n) => h[n] }, res, () => {}); @@ -134,42 +138,99 @@ test('the middleware leaves what the file holds as it is, and loads none of it', assert.equal(request(tags), tags + ADDED); }); -test('on a file of 1000 entries, a request reads nothing from it and writes only its own entry', () => { +test('on a file of 1000 entries, the setup only checks that it exists, and a request only appends its own entry', () => { const cwd = mkdtempSync(join(app, 'big-')); - // 1000 tests, each with 60 lines run in each of 40 files: 28 MiB. + // 1000 tests, each with 60 lines run in each of 40 files: 26 MiB (27,558,890 bytes). const body = Array.from({ length: 40 }, (_, f) => ` /home/runner/app/src/file-${f}.js:\n` + Array.from({ length: 60 }, (_, l) => ` - ${l + 1}\n`).join('')).join(''); const seed = Array.from({ length: 1000 }, (_, t) => `- id: test-set-0/test-${t}\n executedLinesByFile:\n${body}`).join(''); writeFileSync(join(cwd, 'dedupData.yaml'), seed); - // Every call into fs the request makes (not those one of them makes), with - // its first argument and the size of its second. - const r = spawnSync(process.execPath, ['-e', REQUEST(['/home/runner/app/src/index.js']) + ` + // Every call into fs from the setup on, until the process exits (not those + // one of them makes), logged to calls.jsonl as it is made, with its first + // argument and the size of its second: to fs's functions, and to those of + // fs.promises, which require('fs/promises') is too. They are wrapped before + // the middleware loads, so that one it binds then is wrapped too; only the + // load is not logged. + const r = spawnSync(process.execPath, ['-e', ` const fs = require('fs'); - const calls = []; - const saved = {}; + const log = fs.appendFileSync; let depth = 0; - for (const k of Object.keys(fs)) { - if (typeof fs[k] !== 'function' || /^[A-Z]/.test(k)) continue; - const f = (saved[k] = fs[k]); - fs[k] = function (...a) { - if (depth === 0) calls.push([k, a[0], typeof a[1] === 'string' ? Buffer.byteLength(a[1]) : a[1] && a[1].length]); - depth++; - try { return f.apply(this, a); } finally { depth--; } - }; + for (const [api, o] of [['', fs], ['promises.', fs.promises]]) { + for (const k of Object.keys(o)) { + if (typeof o[k] !== 'function' || /^[A-Z]/.test(k)) continue; + const f = o[k]; + o[k] = function (...a) { + if (depth++ === 0) log('calls.jsonl', JSON.stringify([api + k, a[0], typeof a[1] === 'string' ? Buffer.byteLength(a[1]) : a[1] && a[1].length]) + '\\n'); + try { return f.apply(this, a); } finally { depth--; } + }; + } } + depth++; + const middleware = require('@keploy/sdk/dist/v2/dedup/middleware'); + depth--; + ` + COVERAGE(['/home/runner/app/src/index.js']) + ` + const res = new (require('events').EventEmitter)(); + const h = { 'KEPLOY-TEST-ID': 'test-3', 'KEPLOY-TEST-SET-ID': 'test-set-0' }; + middleware()({ get: (n) => h[n] }, res, () => {}); res.emit('finish'); - Object.assign(fs, saved); - process.stdout.write(JSON.stringify(calls)); `], { cwd, encoding: 'utf8' }); assert.equal(r.status, 0, r.stderr); - assert.deepEqual(JSON.parse(r.stdout), [['appendFileSync', 'dedupData.yaml', Buffer.byteLength(ADDED)]]); + const logged = join(cwd, 'calls.jsonl'); + const calls = existsSync(logged) ? readFileSync(logged, 'utf8').split('\n').filter(Boolean).map((l) => JSON.parse(l)) : []; + assert.deepEqual(calls, [['existsSync', 'dedupData.yaml', null], ['appendFileSync', 'dedupData.yaml', Buffer.byteLength(ADDED)]]); assert.ok(readFileSync(join(cwd, 'dedupData.yaml'), 'utf8') === seed + ADDED, 'the file is the seed, then the new entry'); }); +test('requests that finish in the tick that set the middleware up keep their entries', () => { + // No dedupData.yaml yet. The app sets the middleware up and, in that same + // tick, 40 requests finish. The tick sleeps 200 ms first, as start-up work + // would take, so that what the setup started in the thread pool has run: + // 2.0.10's check found the file missing there and, once the tick ended, + // wrote it empty, after the 40 had written their entries. + const cwd = mkdtempSync(join(app, 'setup-')); + const r = spawnSync(process.execPath, ['-e', COVERAGE(['/home/runner/app/src/index.js']) + ` + const handle = require('@keploy/sdk/dist/v2/dedup/middleware')(); + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 200); + for (let i = 0; i < 40; i++) { + const res = new (require('events').EventEmitter)(); + const h = { 'KEPLOY-TEST-ID': 'test-' + i, 'KEPLOY-TEST-SET-ID': 'test-set-0' }; + handle({ get: (n) => h[n] }, res, () => {}); + res.emit('finish'); + } + `], { cwd, encoding: 'utf8' }); + assert.equal(r.status, 0, r.stderr); + const file = readFileSync(join(cwd, 'dedupData.yaml'), 'utf8'); + const ids = Array.from({ length: 40 }, (_, i) => `test-set-0/test-${i}`); + for (const v of [3, 4]) assert.deepEqual((reader(v).load(file) || []).map((e) => e.id), ids, `js-yaml ${v}`); + // The first ran lines 1 and 4; the others, nothing new. + assert.equal(file, ids.map((id, i) => `- id: ${id}\n executedLinesByFile:\n /home/runner/app/src/index.js:${i ? ' []\n' : '\n - 1\n - 4\n'}`).join('')); + // With no request, the setup leaves the file there, empty. + const idle = mkdtempSync(join(app, 'setup-')); + const s = spawnSync(process.execPath, ['-e', "require('@keploy/sdk/dist/v2/dedup/middleware')()"], { cwd: idle, encoding: 'utf8' }); + assert.equal(s.status, 0, s.stderr); + assert.ok(existsSync(join(idle, 'dedupData.yaml')), 'the setup creates the file'); + assert.equal(readFileSync(join(idle, 'dedupData.yaml'), 'utf8'), ''); +}); + +test('the setup writes nothing to a dedupData.yaml that exists', () => { + // As 2.0.10's: an app whose dedupData.yaml it may not write (here a folder + // of that name, which no user may write, root included) starts, and a + // request with no test id leaves it alone. + const cwd = mkdtempSync(join(app, 'setup-')); + mkdirSync(join(cwd, 'dedupData.yaml')); + const r = spawnSync(process.execPath, ['-e', ` + const res = new (require('events').EventEmitter)(); + require('@keploy/sdk/dist/v2/dedup/middleware')()({ get: () => undefined }, res, () => {}); + res.emit('finish'); + `], { cwd, encoding: 'utf8' }); + assert.equal(r.status, 0, r.stderr); + assert.match(r.stderr, /No test ID found in the request headers/); +}); + // Names and ids a coverage tool and a request can give, YAML's own words, // indicators and escapes among them, and lines. const NAMES = ['/home/runner/app/src/index.js', String.raw`C:\Users\dev\app\src\index.js`, '/a b/c.js', '/app/ünïcode/日本.js', '/x/😀.js', 'a#b', 'a #b', 'a: b', 'a:b', 'a:', 'a ', ' a', ' ', ':', '-a', '- a', '?a', ':a', ',a', '[a]', '{a}', '#a', '&a', '*a', '!a', '|a', '=a', '>a', "'a", '"a', '%a', '@a', '`a', - "it's", 'a"b', 'a\\b', 'no', 'No', 'NO', 'yes', 'y', 'n', 'on', 'off', 'OFF', 'true', 'True', 'false', 'null', 'Null', 'NULL', '~', '', '<<', '0o12', '+0o7', '1_000', '+.5', '.5', '08', + "it's", 'a"b', 'a\\b', 'no', 'No', 'NO', 'yes', 'y', 'n', 'on', 'off', 'OFF', 'true', 'True', 'false', 'null', 'Null', 'NULL', '~', '', '<<', '0o12', '+0o7', '1_000', '+.5', '._1', '._97', '._e0', '.5', '08', '0x1F', '0b101', '1:20', '_:_', '1:20.5', '2026-11-04', '2026-11-04T10:00:00Z', '.inf', '-.inf', '.NaN', '.nan', '1e5', '123', '-1', '...', '---', 'a\tb', 'a\nb', 'a\rb', '\x00', '\x7f', '\x85', 'a\xa0b', '\u2028', 'a\u2029b', 'a\ufeffb', '\ufeffa', 'a\ud800b', '\ud83d', 'a\x1fb', 'a~b', 'a\xa1b', 'a\ud7ffb', 'a\ue000b', 'a\ufffdb', 'a\ufffeb', 'a\x80b', 'a\x01b', 'a\t"\\b', '\x1b\x07\x08\x0b\x0c', @@ -197,9 +258,11 @@ test('each entry is the text js-yaml 4 wrote but where the limits say, and js-ya } if (text !== theirs) differ.push(label); } - // The limits: names that start like a number, ids js-yaml 4 writes as a - // block, and lines that are not numbers. + // The limits: names that start like a number or with `._` (js-yaml 3 reads + // js-yaml 4's plain `._1` as 0.1 and `._e0` as NaN), ids js-yaml 4 writes + // as a block, and lines that are not numbers. + const quoted = ['1_000', '+.5', '._1', '._97', '._e0'].flatMap((s) => [`key "${s}"`, `empty "${s}"`, `id "${s}"`]); const x = `id "${'x'.repeat(23)}`; - assert.deepEqual(differ, ['key "1_000"', 'empty "1_000"', 'id "1_000"', 'key "+.5"', 'empty "+.5"', 'id "+.5"', 'id "a\\nb"', + assert.deepEqual(differ, [...quoted, 'id "a\\nb"', x, x, x, `id "${"'".repeat(23)}`, 'id "/p/qqqqqqqqqqqqqqqqqqqq', 'not numbers', 'id of 77']); }); diff --git a/test/workflows.test.mjs b/test/workflows.test.mjs index c681b54..98540de 100644 --- a/test/workflows.test.mjs +++ b/test/workflows.test.mjs @@ -37,6 +37,23 @@ function matches(p, path) { const ALL_ROWS = { name: 'Rows', run: 'node --test test/*.test.mjs', cwd: undefined }; const WORKFLOW_ROWS = { name: 'Workflow rows', run: 'npm ci --ignore-scripts\nnode --test test/workflows.test.mjs\n', cwd: '.' }; +// The keys a workflow, and a job that packs, checks or publishes, may have: +// none of them sets the environment of every step (env), or runs the steps +// somewhere else (container). A NODE_OPTIONS that preloads process.exit(0), +// on the workflow or on a job, would pass every node in the job, the license +// check and these rows among them. +const WORKFLOW_KEYS = ['name', 'on', 'permissions', 'jobs']; +const JOB_KEYS = ['name', 'if', 'needs', 'runs-on', 'timeout-minutes', 'permissions', 'defaults', 'steps']; +// In such a job: the actions a step may run, each pinned by commit, and their +// inputs (an action may set any variable for the steps after it); and the +// variables a step may set for itself. +const ACTIONS = ['actions/checkout@11d5960a326750d5838078e36cf38b85af677262', 'actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020']; +const ACTION_INPUTS = ['node-version', 'registry-url', 'scope']; +// The registry and scope setup-node writes into npm's config, where present: +// the license check installs from that registry. +const ACTION_VALUES = { 'registry-url': 'https://registry.npmjs.org', scope: '@keploy' }; +const STEP_ENV = ['TARBALL', 'NPM_TOKEN', 'NODE_AUTH_TOKEN']; + const workflows = [ // [file, tag, folder, publishing job, pull request job, the rows step, the gate's path from the folder] ['sdk-publish.yml', 'sdk-v*', undefined, 'publish', 'check', ALL_ROWS, 'scripts/license-gate.mjs'], @@ -81,20 +98,35 @@ for (const [file, tag, folder, publish, pr, rows, gate] of workflows) { test(`${file}: nothing skips the license check or the rows, or lets them fail`, () => { const wf = load(file); + assert.deepEqual(Object.keys(wf).filter((k) => !WORKFLOW_KEYS.includes(k)), [], 'the workflow\'s keys'); + for (const [name, j] of Object.entries(wf.jobs)) assert.equal(j.env, undefined, `${name}: env`); for (const [job, event] of [[publish, 'push'], [pr, 'pull_request']]) { const j = wf.jobs[job]; + assert.deepEqual(Object.keys(j).filter((k) => !JOB_KEYS.includes(k)), [], `${job}: its keys`); + assert.equal(j['runs-on'], 'ubuntu-latest', `${job}: runs-on`); // The job runs on its event, and on nothing narrower. assert.equal(j.if, `github.event_name == '${event}'`, `${job}: if`); assert.equal(j['continue-on-error'], undefined, `${job}: continue-on-error`); - assert.equal(j.defaults.run['working-directory'], folder, `${job}: its folder`); + // Its steps run in bash (-eo pipefail), in its folder, and nothing else. + assert.deepEqual(j.defaults, { run: { shell: 'bash', ...(folder ? { 'working-directory': folder } : {}) } }, `${job}: defaults`); const s = steps(j); for (const x of s) { assert.equal(x.if, undefined, `${job}: ${x.name}: if`); assert.equal(x.continueOnError, undefined, `${job}: ${x.name}: continue-on-error`); + // No step sets a variable or a path for the steps after it. + assert.doesNotMatch(x.run, /\bGITHUB_(?:ENV|PATH)\b|::(?:set-env|add-path)\b/i, `${job}: ${x.name}: sets a variable or a path for later steps`); + assert.deepEqual(Object.keys(x.env).filter((k) => !STEP_ENV.includes(k)), [], `${job}: ${x.name}: env`); + } + for (const st of j.steps.filter((st) => st.uses)) { + assert.ok(ACTIONS.includes(st.uses), `${job}: ${st.uses}: an action pinned here`); + assert.deepEqual(Object.keys(st.with || {}).filter((k) => !ACTION_INPUTS.includes(k)), [], `${job}: ${st.uses}: its inputs`); + for (const [k, v] of Object.entries(ACTION_VALUES)) if (st.with && k in st.with) assert.equal(st.with[k], v, `${job}: ${st.uses}: ${k}`); } const r = s.findIndex((x) => x.name === rows.name); assert.ok(r >= 0, `${job}: a ${rows.name} step`); assert.deepEqual([s[r].run, s[r].cwd], [rows.run, rows.cwd], `${job}: ${rows.name} runs from the root`); + // Nothing else on the rows step: no env, no shell of its own. + assert.deepEqual(Object.keys(j.steps[r]).sort(), rows.cwd ? ['name', 'run', 'working-directory'] : ['name', 'run'], `${job}: the ${rows.name} step's keys`); assert.ok(r < s.findIndex((x) => x.name === 'Licenses'), `${job}: ${rows.name} before Licenses`); } // A tag of this package's name starts the publish; any pull request that diff --git a/v2/dedup/entry.ts b/v2/dedup/entry.ts index ed5ea14..98fb5dc 100644 --- a/v2/dedup/entry.ts +++ b/v2/dedup/entry.ts @@ -5,8 +5,9 @@ // Limits, each written another way that every YAML 1.1 and 1.2 reader // (js-yaml 3 and 4 among them) reads back as the same value: // - a name that starts like a number (a sign, then a digit, or a dot and a -// digit), or is ~, null, <<, .inf, .nan or a YAML 1.1 or 1.2 boolean, in -// any letter case, is single-quoted where js-yaml 4 may leave it plain; +// digit), or with a dot and an underscore (js-yaml 3 reads `._1` as 0.1), +// or is ~, null, <<, .inf, .nan or a YAML 1.1 or 1.2 boolean, in any +// letter case, is single-quoted where js-yaml 4 may leave it plain; // - an id with a line break, which js-yaml 4 writes as a block, is // double-quoted, and one over 76 characters, which it folds, stays on one // line; @@ -19,8 +20,10 @@ const ESCAPES: { [c: number]: string } = { // js-yaml 4 single-quotes these, for YAML 1.1 readers, at any length. const BASE60 = /^[-+]?[0-9_]+(?::[0-9_]+)+(?:\.[0-9_]*)?$/; // Every text a YAML 1.1 or 1.2 reader takes for a null, boolean, number, -// timestamp or merge key matches this, in js-yaml 3 and 4 too. -const NOT_A_STRING = /^(?:~|null|true|false|y|yes|n|no|on|off|<<|[-+]?\.(?:inf|nan))$|^[-+]?\.?[0-9]/i; +// timestamp or merge key matches this, in js-yaml 3 and 4 too. js-yaml 3's +// float takes a dot then digits or underscores, so it reads plain `._1` as +// 0.1 and `._e0` as NaN: a name that starts with `._` is quoted. +const NOT_A_STRING = /^(?:~|null|true|false|y|yes|n|no|on|off|<<|[-+]?\.(?:inf|nan))$|^\._|^[-+]?\.?[0-9]/i; // The characters a plain scalar may not start with: space and YAML's indicators. const NOT_FIRST = ' -?:,[]{}#&*!|=>\'"%@`'; diff --git a/v2/dedup/middleware.ts b/v2/dedup/middleware.ts index 2649c72..dfbb432 100644 --- a/v2/dedup/middleware.ts +++ b/v2/dedup/middleware.ts @@ -10,13 +10,11 @@ const filePath = 'dedupData.yaml'; export default function middleware(): (req: Request, res: Response, next: NextFunction) => void { // console.log("Inside middleware..."); - // @ts-ignore - fs.access(filePath, fs.constants.F_OK, (err) => { - if (err) { - // Create the file if it doesn't exist - fs.writeFileSync(filePath, '', 'utf-8'); - } - }); + // Create the file if it does not exist, now, with an append of nothing, + // which never empties it; a file that exists is not written. Checking for + // it asynchronously and then writing it empty, as before, dropped the + // entries of requests that finished between the check and its callback. + if (!fs.existsSync(filePath)) fs.appendFileSync(filePath, '', 'utf-8'); return (req: Request, res: Response, next: NextFunction) => { res.on("finish", () => { afterMiddleware(req, res);