diff --git a/.fallowrc.json b/.fallowrc.json index 43f4763993..21d5b7eb20 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -75,7 +75,7 @@ // A hand-written type surface for a plain-Node .mjs the typechecked program (and Fallow's // graph) cannot import; its only consumer is the .mjs's real importers, resolved by tsc, not // by an import edge. Same shape as the help-conformance declaration files above. - "scripts/size-report-package.d.mts" + "scripts/size-report-package*.d.mts" ], "dynamicallyLoaded": [ "packages/daemon/src/execution/interaction/index.ts", diff --git a/.github/workflows/size.yml b/.github/workflows/size.yml index 66f11fe633..be7c51abab 100644 --- a/.github/workflows/size.yml +++ b/.github/workflows/size.yml @@ -71,7 +71,7 @@ jobs: - name: Measure base size run: | git checkout --detach "${{ github.event.pull_request.base.sha }}" - cp scripts/size-report-package.mjs /tmp/agent-device-size-report/ + cp scripts/size-report-package*.mjs /tmp/agent-device-size-report/ pnpm install --frozen-lockfile if [ "${{ steps.base-dist-cache.outputs.cache-hit }}" != "true" ]; then pnpm build diff --git a/LICENSES/Apache-2.0.txt b/LICENSES/Apache-2.0.txt new file mode 100644 index 0000000000..d645695673 --- /dev/null +++ b/LICENSES/Apache-2.0.txt @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/LICENSES/jpeg-js.txt b/LICENSES/jpeg-js.txt new file mode 100644 index 0000000000..6dcbbfa4c0 --- /dev/null +++ b/LICENSES/jpeg-js.txt @@ -0,0 +1,53 @@ +jpeg-js 0.4.4 source-file notices + +lib/decoder.js: + + Copyright 2011 notmasteryet + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +The full Apache License 2.0 accompanies this notice. + +lib/encoder.js: + + Copyright (c) 2008, Adobe Systems Incorporated + All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + * Neither the name of Adobe Systems Incorporated nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS + IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, + THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +JPEG encoder ported to JavaScript and optimized by Andreas Ritter, www.bytestrom.eu, 11/2009 diff --git a/THIRD_PARTY_NOTICES.md b/THIRD_PARTY_NOTICES.md new file mode 100644 index 0000000000..7d8868279c --- /dev/null +++ b/THIRD_PARTY_NOTICES.md @@ -0,0 +1,40 @@ +# Third-party material + +Third-party material retains its original license. The root MIT license does not +replace the licenses listed here. + +## Bundled JavaScript dependencies + +Each build generates `dist/src/THIRD_PARTY_NOTICES.txt` from the dependencies +included in the bundle. That file ships in the npm package with the dependencies' +license texts and applicable NOTICE contents. Dependencies resolved separately at +runtime retain the license files in their own packages. + +`jpeg-js` also carries source-file notices for its decoder and encoder, beyond +the package-level BSD license. These are retained in +[`LICENSES/jpeg-js.txt`](LICENSES/jpeg-js.txt) and included with the Apache License +2.0 in the generated notice file whenever `jpeg-js` is bundled. + +## Maestro conformance fixtures + +The 42 upstream flows in `scripts/maestro-conformance/corpus/upstream/` are copied +without modification from [Maestro v2.5.1](https://github.com/mobile-dev-inc/Maestro/tree/v2.5.1), +commit `a4c7c95f5ba1884858f7e35efa6b8e0165db9448`, under the Apache License 2.0. +Their original paths and SHA-256 hashes are recorded in the adjacent +`scripts/maestro-conformance/corpus/manifest.json`. + +The license is retained in [LICENSES/Apache-2.0.txt](LICENSES/Apache-2.0.txt). +The conformance fixtures are development inputs and do not ship in the npm package. + +## Gradle wrapper + +The Gradle wrapper in `scripts/maestro-conformance/jvm-harness/` is licensed under +the [Apache License 2.0](LICENSES/Apache-2.0.txt). +Its wrapper scripts retain the upstream copyright notices for the original authors. + +## Simulator accessibility bridge + +`apple/snapshot-bridge/SnapshotBridgeRuntime.m` contains code adapted from Meta's +idb v1.5.2. Its MIT copyright notice and license are retained in +[`apple/snapshot-bridge/LICENSE.idb`](apple/snapshot-bridge/LICENSE.idb), which also +ships with the bridge in the npm package. diff --git a/packages/apple-runner/src/__tests__/runner-lifecycle-request-cancellation.test.ts b/packages/apple-runner/src/__tests__/runner-lifecycle-request-cancellation.test.ts index d2ce983b8d..9493297cbd 100644 --- a/packages/apple-runner/src/__tests__/runner-lifecycle-request-cancellation.test.ts +++ b/packages/apple-runner/src/__tests__/runner-lifecycle-request-cancellation.test.ts @@ -387,7 +387,7 @@ test('a wedged build with no caller left is ended by the default start budget an (build.signal?.reason as AppError | undefined)?.details?.reason, 'runner_start_budget_exhausted', ); - await settleUnderFakeTimers(next); + await next; }); assert.match(diagnostics, /ios_runner_detached_start_failed/); diff --git a/packages/platform-android/src/__tests__/test-utils/fake-timers.ts b/packages/platform-android/src/__tests__/test-utils/fake-timers.ts index cbb60f6584..d4a9ad8bc6 100644 --- a/packages/platform-android/src/__tests__/test-utils/fake-timers.ts +++ b/packages/platform-android/src/__tests__/test-utils/fake-timers.ts @@ -1,8 +1,16 @@ import { vi } from 'vitest'; /** - * Runs `run` with its sleeps advanced instead of waited out. The clock moves in slices shorter than - * a poll interval, so a helper session's own timers never fire ahead of its replies. + * How far the clock moves per step. One step awaits a real event-loop turn, so every turn a session + * reply still needs burns this much of the clock. A reply crossing a real loopback socket takes + * several turns: at 10 ms the polling helper's own interval fired ahead of the reply it was waiting + * for, and the deadline was reached a sample or two early. + */ +const CLOCK_STEP_MS = 1; + +/** + * Runs `run` with its sleeps advanced instead of waited out. The clock moves in slices much shorter + * than a poll interval, so a helper session's own timers never fire ahead of its replies. */ export async function underFakeTimers(run: () => Promise): Promise { vi.useFakeTimers(); @@ -12,7 +20,7 @@ export async function underFakeTimers(run: () => Promise): Promise { done = true; }); outcome.catch(() => {}); - while (!done) await vi.advanceTimersByTimeAsync(10); + while (!done) await vi.advanceTimersByTimeAsync(CLOCK_STEP_MS); return await outcome; } finally { vi.useRealTimers(); diff --git a/scripts/__tests__/bundled-license-notices.test.ts b/scripts/__tests__/bundled-license-notices.test.ts new file mode 100644 index 0000000000..60a08c7d7c --- /dev/null +++ b/scripts/__tests__/bundled-license-notices.test.ts @@ -0,0 +1,359 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; +import { test } from 'vitest'; +import { mkdtempForTest } from '@agent-device/test-kit'; + +const execFileAsync = promisify(execFile); + +async function fixture(files: Record): Promise { + const root = await fs.realpath(await mkdtempForTest('bundled-licenses-')); + const contents = { + 'package.json': JSON.stringify({ name: 'fixture-project', type: 'module' }), + 'tsconfig.json': JSON.stringify({ + compilerOptions: { target: 'es2022', module: 'nodenext', skipLibCheck: true }, + include: ['input.ts'], + }), + ...files, + }; + for (const [relative, source] of Object.entries(contents)) { + const file = path.join(root, relative); + await fs.mkdir(path.dirname(file), { recursive: true }); + await fs.writeFile(file, source); + } + return root; +} + +async function bundle(root: string, dts = false, outputDirectory = 'dist'): Promise { + await execFileAsync(process.execPath, [ + '--experimental-strip-types', + fileURLToPath(new URL('./fixtures/bundled-license-build.mts', import.meta.url)), + root, + String(dts), + outputDirectory, + ]); +} + +test('bundled licenses retain full license and NOTICE text after minification', async () => { + const root = await fixture({ + 'input.ts': [ + "import { value } from 'included';", + "import { ignored } from 'unused';", + "import { external } from 'external';", + 'export const result = value + external;', + ].join('\n'), + 'node_modules/included/package.json': JSON.stringify({ + name: 'included', + version: '1.2.3', + license: 'MIT', + author: 'Fixture author ', + main: 'index.js', + }), + 'node_modules/included/index.js': + "export const value = process.env.FIXTURE_NUMBER ?? 'fixture';", + 'node_modules/included/LICENSE': 'Copyright fixture author\nFull license terms retained.', + 'node_modules/included/NOTICE': 'Upstream attribution retained.', + 'node_modules/unused/package.json': JSON.stringify({ + name: 'unused', + version: '1.0.0', + main: 'index.js', + sideEffects: false, + }), + 'node_modules/unused/index.js': 'export const ignored = 99;', + 'node_modules/external/package.json': JSON.stringify({ + name: 'external', + version: '1.0.0', + main: 'index.js', + }), + 'node_modules/external/index.js': 'export const external = 1;', + }); + await bundle(root); + const notices = await fs.readFile(path.join(root, 'dist/THIRD_PARTY_NOTICES.txt'), 'utf8'); + assert.match(notices, /included/); + assert.match(notices, /1\.2\.3/); + assert.match(notices, /Copyright fixture author\nFull license terms retained\./); + assert.match(notices, /Upstream attribution retained\./); + assert.match(notices, /Author: Fixture author /); + assert.doesNotMatch(notices, /unused|external|fixture-project/); + // The populated notice ends like any other text file: exactly one trailing newline, so a reader + // who cats it and a tool that counts its lines both see a finished last license block. + assert.ok(notices.endsWith('\n'), 'notices end with a newline'); + assert.ok(!notices.endsWith('\n\n'), 'notices carry exactly one trailing newline'); +}); + +test('bundling fails when a dependency has a license identifier but no license text', async () => { + const root = await fixture({ + 'input.ts': "export { value } from 'included';", + 'node_modules/included/package.json': JSON.stringify({ + name: 'included', + version: '1.0.0', + license: 'MIT', + main: 'index.js', + }), + 'node_modules/included/index.js': 'export const value = 42;', + }); + await assert.rejects(bundle(root), /included@1\.0\.0 is missing its license or license text/); +}); + +// A licensed package without a version once shipped a notice reading `included@null (MIT)`, +// attributing code to a release that never existed. The version is checked first, so the failure +// names the version-less package rather than its license. +test.each([ + { name: 'no version field', version: undefined }, + { name: 'blank version', version: ' ' }, +])('bundling rejects a licensed dependency with a missing version: $name', async ({ version }) => { + const manifest: Record = { name: 'included', license: 'MIT', main: 'index.js' }; + if (version !== undefined) manifest.version = version; + const root = await fixture({ + 'input.ts': "export { value } from 'included';", + 'node_modules/included/package.json': JSON.stringify(manifest), + 'node_modules/included/index.js': 'export const value = 42;', + 'node_modules/included/LICENSE': 'Copyright fixture author\nFull license terms.', + }); + await assert.rejects(bundle(root), /Bundled dependency included is missing its version/); + await assert.rejects(fs.stat(path.join(root, 'dist/THIRD_PARTY_NOTICES.txt')), { + code: 'ENOENT', + }); +}); + +// The build enumerates the bundled modules itself. A package whose `package.json` names neither a +// version nor a license, or a module with no `package.json` at all, is still bundled code: beside a +// licensed neighbor it must stop the build, not drop out of a notices file that names the neighbor. +test.each([ + { + name: 'package with neither license nor version', + files: { + 'node_modules/omitted/package.json': JSON.stringify({ name: 'omitted', main: 'index.js' }), + 'node_modules/omitted/index.js': 'export const value = process.env.OMITTED;', + }, + failure: /Bundled dependency omitted is missing its version/, + }, + { + name: 'module with no package.json', + files: { 'node_modules/omitted/index.js': 'export const value = process.env.OMITTED;' }, + failure: + /Bundled module node_modules\/omitted\/index\.js has no package\.json naming its package/, + }, +])( + 'bundling rejects an unattributable module beside a licensed one: $name', + async (fixtureCase) => { + const root = await fixture({ + 'input.ts': [ + "export { value } from 'included';", + "export { value as omitted } from 'omitted';", + ].join('\n'), + 'node_modules/included/package.json': JSON.stringify({ + name: 'included', + version: '1.0.0', + license: 'MIT', + main: 'index.js', + }), + 'node_modules/included/index.js': 'export const value = process.env.INCLUDED;', + 'node_modules/included/LICENSE': 'Copyright fixture author\nFull license terms.', + ...fixtureCase.files, + }); + await assert.rejects(bundle(root), fixtureCase.failure); + await assert.rejects(fs.stat(path.join(root, 'dist/THIRD_PARTY_NOTICES.txt')), { + code: 'ENOENT', + }); + }, +); + +// The attribution block is one line per package, so manifest text that spans lines must not split it. +test('multiline license and author metadata render as one attribution line', async () => { + const root = await fixture({ + 'input.ts': "export { value } from 'included';", + 'node_modules/included/package.json': JSON.stringify({ + name: 'included', + version: '1.0.0', + license: 'MIT\nOR Apache-2.0', + author: 'Multi\nline author ', + main: 'index.js', + }), + 'node_modules/included/index.js': 'export const value = process.env.INCLUDED;', + 'node_modules/included/LICENSE': 'Copyright fixture author\nFull license terms.', + }); + await bundle(root); + const notices = await fs.readFile(path.join(root, 'dist/THIRD_PARTY_NOTICES.txt'), 'utf8'); + assert.equal( + notices.split('\n\n')[0], + 'included@1.0.0 (MIT OR Apache-2.0): [1]; Author: Multi line author ', + ); +}); + +test('identical notices are shared without losing packages or distinct copyrights', async () => { + const sharedLicense = 'Copyright shared author\nFull shared license terms.'; + const sharedNotice = 'Shared upstream attribution.'; + const distinctLicense = 'Copyright different author\nFull shared license terms.'; + const files: Record = { + 'input.ts': [ + "export { value as first } from 'included';", + "export { value as second } from 'shared';", + "export { value as third } from 'distinct';", + ].join('\n'), + }; + for (const name of ['included', 'shared', 'distinct']) { + files[`node_modules/${name}/package.json`] = JSON.stringify({ + name, + version: '1.0.0', + license: 'MIT', + main: 'index.js', + }); + files[`node_modules/${name}/index.js`] = `export const value = process.env.${name};`; + files[`node_modules/${name}/LICENSE`] = name === 'distinct' ? distinctLicense : sharedLicense; + files[`node_modules/${name}/NOTICE`] = sharedNotice; + } + const root = await fixture(files); + await bundle(root); + const notices = await fs.readFile(path.join(root, 'dist/THIRD_PARTY_NOTICES.txt'), 'utf8'); + assert.equal(notices.split(sharedLicense).length - 1, 1); + assert.equal(notices.split(sharedNotice).length - 1, 1); + assert.equal(notices.split(distinctLicense).length - 1, 1); + const references = ['included', 'shared', 'distinct'].map((name) => { + const line = new RegExp(`^${name}@1\\.0\\.0 \\(MIT\\): (.+)$`, 'm'); + assert.match(notices, line); + return notices.match(line)?.[1]; + }); + assert.equal(references[0], references[1]); + assert.notEqual(references[0], references[2]); + await fs.writeFile( + path.join(root, 'input.ts'), + [ + "export { value as third } from 'distinct';", + "export { value as second } from 'shared';", + "export { value as first } from 'included';", + ].join('\n'), + ); + await bundle(root); + assert.equal(await fs.readFile(path.join(root, 'dist/THIRD_PARTY_NOTICES.txt'), 'utf8'), notices); +}); + +test('notice references use code-unit ordering for package names', async () => { + const names = ['included_extra', 'included-extra', 'included.extra']; + const files: Record = { + 'input.ts': names + .map((name, index) => `export { value as value${index} } from '${name}';`) + .join('\n'), + }; + for (const name of names) { + files[`node_modules/${name}/package.json`] = JSON.stringify({ + name, + version: '1.0.0', + license: 'MIT', + main: 'index.js', + }); + files[`node_modules/${name}/index.js`] = `export const value = '${name}';`; + files[`node_modules/${name}/LICENSE`] = 'Copyright ordering fixture\nShared license terms.'; + } + const root = await fixture(files); + await bundle(root); + const notices = await fs.readFile(path.join(root, 'dist/THIRD_PARTY_NOTICES.txt'), 'utf8'); + assert.equal( + notices.split('\n\n')[0], + [ + 'included-extra@1.0.0 (MIT): [1]', + 'included.extra@1.0.0 (MIT): [1]', + 'included_extra@1.0.0 (MIT): [1]', + ].join('\n'), + ); +}); + +test('declaration generation preserves JavaScript dependency notices', async () => { + const root = await fixture({ + 'input.ts': "export { value } from 'included';", + 'node_modules/included/package.json': JSON.stringify({ + name: 'included', + version: '1.0.0', + license: 'MIT', + main: 'index.js', + types: 'index.d.ts', + }), + 'node_modules/included/index.js': "export const value = 'included';", + 'node_modules/included/index.d.ts': 'export declare const value: string;', + 'node_modules/included/LICENSE': 'Copyright included author\nIncluded license terms.', + }); + await bundle(root); + const notices = await fs.readFile(path.join(root, 'dist/THIRD_PARTY_NOTICES.txt'), 'utf8'); + assert.match(notices, /included@1\.0\.0/); + await bundle(root, true); + assert.ok((await fs.stat(path.join(root, 'dist/input.d.ts'))).size > 0); + assert.equal(await fs.readFile(path.join(root, 'dist/THIRD_PARTY_NOTICES.txt'), 'utf8'), notices); +}); + +test.each(['0.4.4', '0.4.5'])( + 'bundled jpeg-js@%s requires matching source-file notices', + async (version) => { + const [sourceNotices, apacheLicense] = await Promise.all([ + fs.readFile(new URL('../../LICENSES/jpeg-js.txt', import.meta.url), 'utf8'), + fs.readFile(new URL('../../LICENSES/Apache-2.0.txt', import.meta.url), 'utf8'), + ]); + const root = await fixture({ + 'input.ts': "export { value } from 'jpeg-js';", + 'node_modules/jpeg-js/package.json': JSON.stringify({ + name: 'jpeg-js', + version, + license: 'BSD-3-Clause', + main: 'index.js', + }), + 'node_modules/jpeg-js/index.js': 'export const value = process.env.FIXTURE_NUMBER;', + 'node_modules/jpeg-js/LICENSE': 'Package-level BSD terms.', + 'LICENSES/jpeg-js.txt': sourceNotices, + 'LICENSES/Apache-2.0.txt': apacheLicense, + }); + if (version === '0.4.5') { + await assert.rejects(bundle(root), /LICENSES\/jpeg-js\.txt.*jpeg-js@0\.4\.5/); + return; + } + await bundle(root); + const notices = await fs.readFile(path.join(root, 'dist/THIRD_PARTY_NOTICES.txt'), 'utf8'); + assert.match(notices, /Package-level BSD terms/); + assert.match(notices, /Copyright 2011 notmasteryet/); + assert.match(notices, /Adobe Systems Incorporated/); + assert.match(notices, /JPEG encoder ported to JavaScript and optimized by Andreas Ritter/); + assert.ok(notices.includes(sourceNotices.replaceAll('\r\n', '\n').trim())); + assert.ok(notices.includes(apacheLicense.replaceAll('\r\n', '\n').trim())); + }, +); + +// A project that genuinely bundles nothing keeps getting the generic notice: the content policy that +// holds the published CLI to naming somebody is the publish gate's, not this formatter's. Pinning it +// here keeps a future "reject the placeholder" edit at the formatter from breaking an empty build. +test('a bundle with no third-party code emits the generic notice unchanged', async () => { + const root = await fixture({ + 'input.ts': "import { external } from 'external';\nexport const result = external;", + 'node_modules/external/package.json': JSON.stringify({ + name: 'external', + version: '1.0.0', + main: 'index.js', + }), + 'node_modules/external/index.js': 'export const external = 1;', + }); + await bundle(root); + assert.equal( + await fs.readFile(path.join(root, 'dist/THIRD_PARTY_NOTICES.txt'), 'utf8'), + 'No third-party dependencies are bundled.\n', + ); +}); + +test('output overrides write notices beside the bundle without creating the default dist', async () => { + const root = await fixture({ + 'input.ts': "export { value } from 'included';", + 'node_modules/included/package.json': JSON.stringify({ + name: 'included', + version: '1.0.0', + license: 'MIT', + main: 'index.js', + }), + 'node_modules/included/index.js': 'export const value = 42;', + 'node_modules/included/LICENSE': 'Copyright fixture author\nFull license terms.', + }); + await bundle(root, false, 'redirected'); + assert.match( + await fs.readFile(path.join(root, 'redirected/THIRD_PARTY_NOTICES.txt'), 'utf8'), + /included@1\.0\.0/, + ); + await assert.rejects(fs.stat(path.join(root, 'dist')), { code: 'ENOENT' }); +}); diff --git a/scripts/__tests__/fixtures/bundled-license-build.mts b/scripts/__tests__/fixtures/bundled-license-build.mts new file mode 100644 index 0000000000..69604fabd2 --- /dev/null +++ b/scripts/__tests__/fixtures/bundled-license-build.mts @@ -0,0 +1,32 @@ +import path from 'node:path'; +import { build } from 'tsdown'; +import { bundledLicenseNotices } from '../../lib/bundled-license-notices.ts'; + +const root = process.argv[2]; +if (!root) throw new Error('Missing license fixture root.'); + +await build({ + config: false, + cwd: root, + tsconfig: path.join(root, 'tsconfig.json'), + entry: ['input.ts'], + outDir: process.argv[4] ?? 'dist', + outExtensions: () => ({ js: '.js', dts: '.d.ts' }), + dts: process.argv[3] === 'true', + minify: true, + deps: { + onlyBundle: [ + 'included', + 'omitted', + 'shared', + 'distinct', + 'unused', + 'jpeg-js', + 'included-extra', + 'included.extra', + 'included_extra', + ], + }, + plugins: [bundledLicenseNotices(root)], + logLevel: 'silent', +}); diff --git a/scripts/__tests__/fixtures/size-report-npm-pack.json b/scripts/__tests__/fixtures/size-report-npm-pack.json index 8494a3fcc3..2a229a1a22 100644 --- a/scripts/__tests__/fixtures/size-report-npm-pack.json +++ b/scripts/__tests__/fixtures/size-report-npm-pack.json @@ -1,6 +1,10 @@ { "unpackedSize": 2319, "files": [ + { + "path": "dist/src/THIRD_PARTY_NOTICES.txt", + "size": 100 + }, { "path": "dist/src/index.js", "size": 401 @@ -82,6 +86,6 @@ "size": 426 } ], - "entryCount": 20, + "entryCount": 21, "size": 3376 } diff --git a/scripts/__tests__/size-report-package.test.ts b/scripts/__tests__/size-report-package.test.ts index 137cadee5b..4454ced1c5 100644 --- a/scripts/__tests__/size-report-package.test.ts +++ b/scripts/__tests__/size-report-package.test.ts @@ -1,14 +1,23 @@ import assert from 'node:assert/strict'; -import { readFile } from 'node:fs/promises'; +import { execFile } from 'node:child_process'; +import { mkdir, readFile, realpath, writeFile } from 'node:fs/promises'; import { join } from 'node:path'; +import { promisify } from 'node:util'; import { test } from 'vitest'; +import { mkdtempForTest } from '@agent-device/test-kit'; import { formatMarkdown } from '../size-report.mjs'; -import { assertPublishPackageContents } from '../size-report-package.mjs'; +import { + assertBundledLicenseNotices, + assertPublishPackageContents, + parseNpmPackOutput, +} from '../size-report-package.mjs'; const fixturePack = JSON.parse( await readFile(join(import.meta.dirname, 'fixtures', 'size-report-npm-pack.json'), 'utf8'), ); +const execFileAsync = promisify(execFile); + // The fixture pack ships the macOS helper (manifest, sources, and per-target PackStub.swift), // so every assertion here names the manifest source the guard would read from the packed tree; // without it the guard falls back to this checkout's manifest via repoRoot. @@ -23,6 +32,467 @@ const helperManifestSource = [ ' name: "AgentDeviceMacOSInputTests",', ].join('\n'); +test('publish package requires bundled dependency license notices', () => { + assert.throws( + () => + assertPublishPackageContents( + fixturePack.files.filter((entry) => entry.path !== 'dist/src/THIRD_PARTY_NOTICES.txt'), + { macosHelperManifestSource: helperManifestSource }, + ), + /THIRD_PARTY_NOTICES\.txt/, + ); + assert.throws( + () => + assertPublishPackageContents( + fixturePack.files.map((entry) => + entry.path === 'dist/src/THIRD_PARTY_NOTICES.txt' ? { ...entry, size: 0 } : entry, + ), + { macosHelperManifestSource: helperManifestSource }, + ), + /non-empty.*THIRD_PARTY_NOTICES\.txt/, + ); +}); + +// The publish gate's own copy of what a shipped notices file must contain. Written out rather than +// produced by the bundler: a fixture the producer generates cannot show the producer emitted the wrong +// thing. The attribution lines, the reference markers and the license bodies are the shape +// `renderLicenseNotices` in scripts/size-report-package-notices.mjs writes for a bundled build, and the +// placeholder below is the line that same function writes when the bundle holds no third-party code. +const REAL_NOTICES = [ + '@agent-device/proxy@0.21.37-dev (MIT): [1]', + 'undici@7.1.1 (MIT): [1]', + '', + '[1]', + 'Copyright (c) 2023 Callstack', + 'Permission is hereby granted, free of charge, to any person obtaining a copy of this software.', +].join('\n'); +const EMPTY_BUNDLE_NOTICES = 'No third-party dependencies are bundled.\n'; + +// Two distinct license bodies, one of them shared by two attributions: the shape the bundler writes +// when it deduplicates identical texts, which the gate must keep accepting. +const TWO_LICENSES_NOTICES = [ + '@agent-device/proxy@0.21.37-dev (MIT): [1]', + 'undici@7.1.1 (MIT): [2]', + 'yauzl@3.4.0 (MIT): [1], [2]', + '', + '[1]', + 'Copyright (c) 2023 Callstack', + 'Permission is hereby granted, free of charge, to any person obtaining a copy of this software.', + '', + '[2]', + 'Copyright (c) 2018 Nathan LaFreniere and other contributors', + 'Permission is hereby granted, free of charge, to any person obtaining a copy of this software.', +].join('\n'); + +/** The pack manifest's notices entry — non-empty by bytes, which is all a manifest can say. */ +function noticesEntry() { + return fixturePack.files.find((entry) => entry.path === 'dist/src/THIRD_PARTY_NOTICES.txt'); +} + +/** + * A pack manifest plus the tree it was built from, so the gate is asked about real content and not + * only about a byte count. `noticesRoot` is the half the placeholder slipped past before. + */ +async function treeWithNotices(contents: string | null): Promise { + const root = await mkdtempForTest('notices-tree-'); + await mkdir(join(root, 'dist', 'src'), { recursive: true }); + if (contents !== null) { + await writeFile(join(root, 'dist', 'src', 'THIRD_PARTY_NOTICES.txt'), contents); + } + return root; +} + +test('pack gate rejects the zero-dependency placeholder a non-empty manifest reports', async () => { + // The failure this exists for: the bundler stops bundling dependencies, the bundler reports none, + // and the tarball still ships a non-empty notices file. The manifest entry below is exactly the one + // a healthy build produces, so only the bytes tell the two apart. + const root = await treeWithNotices(EMPTY_BUNDLE_NOTICES); + assert.throws( + () => assertBundledLicenseNotices([noticesEntry()], { noticesRoot: root, context: 'npm pack' }), + /no bundled dependency notices[\s\S]*found 0 attributions and 0 license texts/, + ); + // Attribution lines with no license text behind them are the same defect by half. + const halfWritten = await treeWithNotices('@agent-device/proxy@0.21.37-dev (MIT): [1]\n'); + assert.throws( + () => + assertBundledLicenseNotices([noticesEntry()], { + noticesRoot: halfWritten, + context: 'npm pack', + }), + /found 1 attributions and 0 license texts/, + ); +}); + +test('pack gate accepts a tree whose notices attribute the bundled dependencies', async () => { + const root = await treeWithNotices(REAL_NOTICES); + assert.doesNotThrow(() => + assertBundledLicenseNotices([noticesEntry()], { noticesRoot: root, context: 'npm pack' }), + ); +}); + +test('pack gate rejects attributions whose references hold no license text', async () => { + // A marker is a heading, not a license: this file has the shape the bundler writes and zero terms + // in it, and a gate that only counts the two kinds of line lets it through. + const markerWithoutTerms = await treeWithNotices( + '@agent-device/proxy@0.21.37-dev (MIT): [1]\n\n[1]\n', + ); + assert.throws( + () => + assertBundledLicenseNotices([noticesEntry()], { + noticesRoot: markerWithoutTerms, + context: 'npm pack', + }), + /no bundled dependency notices[\s\S]*1 attributions and 0 license texts[\s\S]*unresolved: @agent-device\/proxy@0\.21\.37-dev \(MIT\) -> \[1\]/, + ); + // The other half: the reference names an id the file never writes. + const dangling = await treeWithNotices( + [ + '@agent-device/proxy@0.21.37-dev (MIT): [3]', + '', + '[1]', + 'Copyright (c) 2023 Callstack', + 'Permission is hereby granted, free of charge, to any person obtaining a copy of this software.', + ].join('\n'), + ); + assert.throws( + () => + assertBundledLicenseNotices([noticesEntry()], { noticesRoot: dangling, context: 'npm pack' }), + /unresolved: @agent-device\/proxy@0\.21\.37-dev \(MIT\) -> \[3\]/, + ); + // An attribution with nothing after the colon promises a license and carries none. + const noReference = await treeWithNotices( + [ + '@agent-device/proxy@0.21.37-dev (MIT): ', + '', + '[1]', + 'Copyright (c) 2023 Callstack', + 'Permission is hereby granted, free of charge, to any person obtaining a copy of this software.', + ].join('\n'), + ); + assert.throws( + () => + assertBundledLicenseNotices([noticesEntry()], { + noticesRoot: noReference, + context: 'npm pack', + }), + /unresolved: @agent-device\/proxy@0\.21\.37-dev \(MIT\) -> no reference/, + ); +}); + +test('pack gate accepts every reference a deduplicated notices file makes', async () => { + // The bundler numbers each distinct license text once, so one package can point at two texts and + // two packages can share one. Resolving references must not read that as a missing license. + const root = await treeWithNotices(TWO_LICENSES_NOTICES); + assert.doesNotThrow(() => + assertBundledLicenseNotices([noticesEntry()], { noticesRoot: root, context: 'npm pack' }), + ); + // The shapes the real build writes and a naive reader would reject: an attribution that names an + // author after the references, and an Apache-style license whose body carries blank lines. + await writeFile( + join(root, 'dist', 'src', 'THIRD_PARTY_NOTICES.txt'), + [ + '@agent-device/proxy@0.21.37-dev (MIT): [1]; Author: Callstack', + 'undici@7.1.1 (Apache-2.0): [1], [2]; Author: Nathan LaFreniere ', + '', + '[1]', + 'Copyright (c) 2023 Callstack', + 'Permission is hereby granted, free of charge, to any person obtaining a copy of this software.', + '', + '[2]', + 'Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file', + 'except in compliance with the License.', + '', + 'You may obtain a copy of the License at', + '', + ' https://www.apache.org/licenses/LICENSE-2.0', + ].join('\n'), + ); + assert.doesNotThrow(() => + assertBundledLicenseNotices([noticesEntry()], { noticesRoot: root, context: 'npm pack' }), + ); +}); + +test('pack gate names the tree it could not read notices from', async () => { + const root = await treeWithNotices(null); + assert.throws( + () => assertBundledLicenseNotices([noticesEntry()], { noticesRoot: root, context: 'npm pack' }), + /npm pack has no dist\/src\/THIRD_PARTY_NOTICES\.txt/, + ); +}); + +test('pack gate reads content without a tree, staying a manifest-only check', () => { + // The manifest entry alone is a presence-and-bytes claim, so that is all this call can prove; the + // content policy needs the tree whose bytes it is standing in for. Production callers — `collectNpmPack` + // and both calls in `check-package.ts` — always name that tree, so the packed and installed copies + // are read, never only the manifest. + assert.doesNotThrow(() => assertBundledLicenseNotices([noticesEntry()])); +}); + +/** A dependency the fixture build bundles, with the manifest fields that shape its attribution. */ +function dependencyManifest( + name: string, + fields: Record, +): [string, string][] { + return [ + [ + `node_modules/${name}/package.json`, + JSON.stringify({ name, version: '1.2.3', main: 'index.js', ...fields }), + ], + [ + `node_modules/${name}/index.js`, + `export const ${name.replaceAll(/[^A-Za-z0-9_$]/g, '_')} = process.env.${name.toUpperCase()};`, + ], + [`node_modules/${name}/LICENSE`, `Copyright ${name} author\nFull ${name} license terms.`], + ]; +} + +type FixtureDependency = [string, Record, Record?]; + +/** + * Real renderer bytes: runs the same `bundledLicenseNotices` plugin the published build uses (through + * the shared fixture build script) over a fixture tree, so the attribution lines the pack gate reads + * are the ones `renderLicenseNotices` writes — grouped SPDX field, author clause and all — rather than a + * hand-written approximation of them. The build writes into the tree's `dist/src`, so the fixture + * root is the tree the gate reads, whether or not the build succeeded. + */ +async function bundledTree(dependencies: FixtureDependency[]): Promise { + const root = await realpath(await mkdtempForTest('pack-gate-render-')); + const files: [string, string][] = [ + ['package.json', JSON.stringify({ name: 'fixture-project', type: 'module' })], + [ + 'tsconfig.json', + JSON.stringify({ + compilerOptions: { target: 'es2022', module: 'nodenext', skipLibCheck: true }, + include: ['input.ts'], + }), + ], + ['input.ts', dependencies.map(([name]) => `export { ${name} } from '${name}';`).join('\n')], + ]; + for (const [name, fields, extras] of dependencies) { + files.push(...dependencyManifest(name, fields)); + if (extras) files.push(...Object.entries(extras)); + } + for (const [relative, source] of files) { + const file = join(root, relative); + await mkdir(join(file, '..'), { recursive: true }); + await writeFile(file, source); + } + return root; +} + +async function buildTree(root: string): Promise { + await execFileAsync( + process.execPath, + [ + '--experimental-strip-types', + join(import.meta.dirname, 'fixtures', 'bundled-license-build.mts'), + root, + 'false', + 'dist/src', + ], + { cwd: import.meta.dirname }, + ); +} + +async function renderedNotices(dependencies: FixtureDependency[]): Promise { + const root = await bundledTree(dependencies); + await buildTree(root); + return root; +} + +test('a licensed dependency cannot carry an unattributable one past build and pack gate', async () => { + // The former license plugin reported only packages whose manifest named a version or a license, so a + // bundled `omitted` package with neither left a notices file that attributed its licensed + // neighbor, and the gate accepted it. The build enumerates bundled modules itself: it fails, writes + // no notices, and the gate then names the missing file instead of passing a partial one. + const root = await bundledTree([ + ['included', { license: 'MIT' }], + ['omitted', { version: undefined }], + ]); + await assert.rejects(buildTree(root), /Bundled dependency omitted is missing its version/); + assert.throws( + () => assertBundledLicenseNotices([noticesEntry()], { noticesRoot: root, context: 'npm pack' }), + /npm pack has no dist\/src\/THIRD_PARTY_NOTICES\.txt/, + ); +}); + +test('pack gate accepts the attribution the renderer writes for a grouped SPDX expression', async () => { + // A `package.json` license may be a grouped SPDX expression, and `renderLicenseNotices` wraps the + // manifest's license in its own parentheses, so the attribution carries nested ones. Those bytes are + // valid producer output: the gate has to read them, not report them as naming nobody. + const root = await renderedNotices([ + ['shared', { license: '(MIT OR Apache-2.0)', author: 'Grouped author ' }], + ]); + const rendered = await readFile(join(root, 'dist', 'src', 'THIRD_PARTY_NOTICES.txt'), 'utf8'); + assert.match(rendered, /^shared@1\.2\.3 \(\(MIT OR Apache-2\.0\)\): \[1\]; Author: /m); + assert.doesNotThrow(() => + assertBundledLicenseNotices([noticesEntry()], { noticesRoot: root, context: 'npm pack' }), + ); +}); + +test('pack gate reads the license reference, not the author clause, from real bytes', async () => { + // The author line is attribution metadata: a bracketed footnote inside it is not a license + // reference. Resolving it as one would both reject valid bytes (no such section) and let an + // attribution with no reference at all pass on an author's footnote. + const root = await renderedNotices([ + ['distinct', { license: 'MIT', author: 'Bracket author [7] ' }], + ]); + const rendered = await readFile(join(root, 'dist', 'src', 'THIRD_PARTY_NOTICES.txt'), 'utf8'); + assert.match(rendered, /\[1\]; Author: Bracket author \[7\]/); + assert.doesNotThrow(() => + assertBundledLicenseNotices([noticesEntry()], { noticesRoot: root, context: 'npm pack' }), + ); +}); + +test('pack gate reads no attribution from license or NOTICE body text', async () => { + // A numbered section holds upstream license or NOTICE text verbatim, at column zero: the renderer + // neither escapes nor indents it. Prose shaped like an attribution — an email contact, a line + // pointing at "section [9]" of some other package — is ordinary upstream content, and the only + // attributions the bundler itself writes are the block before the first heading. Reading a body + // line as an attribution would reject publishable bytes: the dangling [9] and the empty reference + // below are unparseable as anything but unresolved attributions. + const root = await renderedNotices([ + [ + 'included', + { license: 'MIT' }, + { + 'node_modules/included/NOTICE': [ + 'This distribution bundles software from several upstreams:', + 'helper-pkg@0.1.0 (MIT): see section [9] for those terms', + 'maintainer@example.test (Apache-2.0): ', + ].join('\n'), + }, + ], + ]); + const rendered = await readFile(join(root, 'dist', 'src', 'THIRD_PARTY_NOTICES.txt'), 'utf8'); + assert.match(rendered, /^helper-pkg@0\.1\.0 \(MIT\): see section \[9\]/m); + assert.doesNotThrow(() => + assertBundledLicenseNotices([noticesEntry()], { noticesRoot: root, context: 'npm pack' }), + ); +}); + +test('pack gate rejects a file that attributes nobody but whose license text names somebody', async () => { + // The gate exists to catch a notices file that attributes nobody. A build that lost the leading + // attribution block keeps the heading and bodies, and one body can happen to carry a line shaped + // like an attribution pointing at its own section. That file must not clear the gate on prose it + // preserves verbatim: only the block the renderer writes before the first heading attributes. + const bodyOnly = await treeWithNotices( + [ + '[1]', + 'Copyright (c) 2023 Callstack', + 'Permission is hereby granted, free of charge, to any person obtaining a copy of this software.', + '@agent-device/proxy@0.21.37-dev (MIT): [1]', + ].join('\n'), + ); + assert.throws( + () => + assertBundledLicenseNotices([noticesEntry()], { + noticesRoot: bodyOnly, + context: 'npm pack', + }), + /no bundled dependency notices[\s\S]*found 0 attributions and 1 license texts/, + ); +}); + +test('pack gate still rejects a grouped attribution whose reference is missing or absent', async () => { + // The mixed file is the point: a simple attribution must not stand in for the grouped one, so the + // grouped line has to be inspected and named rather than skipped as unparseable. + const groupedDangling = await treeWithNotices( + [ + '@agent-device/proxy@0.21.37-dev (MIT): [1]', + 'undici@7.1.1 (MIT OR (Apache-2.0 AND CC0-1.0)): [9]', + '', + '[1]', + 'Copyright (c) 2023 Callstack', + 'Permission is hereby granted, free of charge, to any person obtaining a copy of this software.', + ].join('\n'), + ); + assert.throws( + () => + assertBundledLicenseNotices([noticesEntry()], { + noticesRoot: groupedDangling, + context: 'npm pack', + }), + /2 attributions and 1 license texts[\s\S]*unresolved: undici@7\.1\.1 \(MIT OR \(Apache-2\.0 AND CC0-1\.0\)\) -> \[9\]/, + ); + const groupedEmpty = await treeWithNotices( + [ + '@agent-device/proxy@0.21.37-dev (MIT): [1]', + 'undici@7.1.1 (MIT OR Apache-2.0): ', + '', + '[1]', + 'Copyright (c) 2023 Callstack', + 'Permission is hereby granted, free of charge, to any person obtaining a copy of this software.', + ].join('\n'), + ); + assert.throws( + () => + assertBundledLicenseNotices([noticesEntry()], { + noticesRoot: groupedEmpty, + context: 'npm pack', + }), + /unresolved: undici@7\.1\.1 \(MIT OR Apache-2\.0\) -> no reference/, + ); +}); + +test('pack gate takes no reference from an author clause', async () => { + // Written out rather than rendered: the shape the renderer only produces when a package's author + // field carries a footnote, which the real-bytes case above already covers end to end. + const authorSupplied = await treeWithNotices( + [ + '@agent-device/proxy@0.21.37-dev (MIT): ; Author: Footnote author [1]', + '', + '[1]', + 'Copyright (c) 2023 Callstack', + 'Permission is hereby granted, free of charge, to any person obtaining a copy of this software.', + ].join('\n'), + ); + assert.throws( + () => + assertBundledLicenseNotices([noticesEntry()], { + noticesRoot: authorSupplied, + context: 'npm pack', + }), + /unresolved: @agent-device\/proxy@0\.21\.37-dev \(MIT\) -> no reference/, + ); + // And the same file with a real reference beside the footnote passes: the author clause is + // ignored, not treated as a rejection by default. + const withReference = await treeWithNotices( + [ + '@agent-device/proxy@0.21.37-dev (MIT): [1]; Author: Footnote author [1]', + '', + '[1]', + 'Copyright (c) 2023 Callstack', + 'Permission is hereby granted, free of charge, to any person obtaining a copy of this software.', + ].join('\n'), + ); + assert.doesNotThrow(() => + assertBundledLicenseNotices([noticesEntry()], { + noticesRoot: withReference, + context: 'npm pack', + }), + ); +}); + +test('publish package gate rejects placeholder notices from the tree it packed', async () => { + const root = await treeWithNotices(EMPTY_BUNDLE_NOTICES); + assert.throws( + () => + assertPublishPackageContents(fixturePack.files, { + noticesRoot: root, + macosHelperManifestSource: helperManifestSource, + }), + /no bundled dependency notices/, + ); + await writeFile(join(root, 'dist', 'src', 'THIRD_PARTY_NOTICES.txt'), REAL_NOTICES); + assert.doesNotThrow(() => + assertPublishPackageContents(fixturePack.files, { + noticesRoot: root, + macosHelperManifestSource: helperManifestSource, + }), + ); +}); + test('publish package requires both Android helpers and excludes benchmark scripts', () => { assert.doesNotThrow(() => assertPublishPackageContents(fixturePack.files, { @@ -168,6 +638,67 @@ test('base measurement uses the measured revision package asset policy', async ( .split(' - name: Save base dist cache')[0]!; assert.match( baseStep, - /git checkout --detach[\s\S]*cp scripts\/size-report-package\.mjs \/tmp\/agent-device-size-report\/[\s\S]*node \/tmp\/agent-device-size-report\/size-report\.mjs/, + /git checkout --detach[\s\S]*cp scripts\/size-report-package\*\.mjs \/tmp\/agent-device-size-report\/[\s\S]*node \/tmp\/agent-device-size-report\/size-report\.mjs/, + ); +}); + +// The size lane copies `scripts/size-report*.mjs` out of the checkout and recopies the measured +// revision's `scripts/size-report-package*.mjs`, then runs the copy with no node_modules. Each +// module the package policy imports must fall in both copies, and the notices format owner, which +// the build also imports, must import nothing. +test('the package policy and its notices format run from the standalone size-report copy', async () => { + const policy = await readFile(join(import.meta.dirname, '../size-report-package.mjs'), 'utf8'); + const localImports = [...policy.matchAll(/from '\.\/([^']+)'/g)].map((match) => match[1]); + assert.deepEqual(localImports, ['size-report-package-notices.mjs']); + for (const local of localImports) assert.match(local, /^size-report-package[^/]*\.mjs$/); + const format = await readFile( + join(import.meta.dirname, '../size-report-package-notices.mjs'), + 'utf8', + ); + assert.doesNotMatch(format, /^\s*import\b|\bimport\(|\brequire\(/m); +}); + +test.each(['1', -1, 0.5, null])('npm pack rejects malformed file sizes (%s)', (size) => { + assert.throws( + () => + parseNpmPackOutput( + JSON.stringify([ + { + filename: 'fixture.tgz', + size: 1, + unpackedSize: 1, + files: [{ path: 'dist/src/THIRD_PARTY_NOTICES.txt', size }], + }, + ]), + ), + /invalid file entry/, ); }); + +test.each( + ['size', 'unpackedSize'].flatMap((field) => + [undefined, null, '1', -1, 0.5, Number.MAX_SAFE_INTEGER + 1].map((value) => ({ field, value })), + ), +)('npm pack rejects invalid aggregate $field ($value)', ({ field, value }) => { + assert.throws( + () => + parseNpmPackOutput( + JSON.stringify({ filename: 'fixture.tgz', ...fixturePack, [field]: value }), + ), + new RegExp(`invalid ${field}`), + ); +}); + +test.each([0, Number.MAX_SAFE_INTEGER])('npm pack retains valid aggregate sizes (%s)', (size) => { + const parsed = parseNpmPackOutput( + JSON.stringify({ filename: 'fixture.tgz', ...fixturePack, size, unpackedSize: size }), + ); + assert.equal(parsed.size, size); + assert.equal(parsed.unpackedSize, size); +}); + +test('npm pack parser shares validated file entries across both output shapes', () => { + const pack = { filename: 'fixture.tgz', ...fixturePack }; + assert.deepEqual(parseNpmPackOutput(JSON.stringify([pack])).files, fixturePack.files); + assert.deepEqual(parseNpmPackOutput(JSON.stringify(pack)).files, fixturePack.files); +}); diff --git a/scripts/check-affected/command.ts b/scripts/check-affected/command.ts index 46ab92f5ad..07540b2298 100644 --- a/scripts/check-affected/command.ts +++ b/scripts/check-affected/command.ts @@ -18,13 +18,5 @@ runEntrypoint('validation command', async () => { process.stderr.write(chunk); }, }); - await Promise.all( - [process.stdout, process.stderr].map( - (stream) => - new Promise((resolve, reject) => - stream.write('', (error) => (error ? reject(error) : resolve())), - ), - ), - ); return result.exitCode; }); diff --git a/scripts/check-package.ts b/scripts/check-package.ts index 5b7291301e..0bb72f6733 100644 --- a/scripts/check-package.ts +++ b/scripts/check-package.ts @@ -31,6 +31,11 @@ import { type PackedManifest as PackedDependencies, } from './lib/shipped-imports.ts'; import { assertInstalledSnapshotBridge } from './size-report-install.mjs'; +import { + assertBundledLicenseNotices, + parseNpmPackOutput, + type NpmPackEntry, +} from './size-report-package.mjs'; import { testOnlyContentFailures, testOnlyPathFailures } from './lib/shipped-test-code.ts'; import { DAEMON_SOURCE_ENTRY, isSourceCheckoutProjectRoot } from '@agent-device/host-kit/version'; @@ -91,21 +96,21 @@ function step(message: string): void { * npm reports is the package's own declaration of what it ships, which is what the test-code guard * reads — a clean-install tree would also hold the installed dependencies' files. */ -function packTarball(): { tarball: string; packedPaths: string[] } { +function packTarball(): { tarball: string; packedEntries: readonly NpmPackEntry[] } { if (!fs.existsSync(path.join(repoRoot, 'dist', 'src'))) { throw new Error('No dist/src build found. Run `pnpm build` first.'); } fs.mkdirSync(packDestination, { recursive: true }); - const packed = JSON.parse( + const packed = parseNpmPackOutput( run( 'npm', ['pack', '--ignore-scripts', '--json', '--pack-destination', packDestination], repoRoot, ), - ) as [{ filename: string; files: { path: string }[] }]; + ); return { - tarball: path.join(packDestination, packed[0].filename), - packedPaths: packed[0].files.map((file) => file.path), + tarball: path.join(packDestination, packed.filename), + packedEntries: packed.files, }; } @@ -237,9 +242,20 @@ function smokeTestBin(installedRoot: string, manifest: PackedManifest): void { step(`Packing and verifying agent-device in ${workDir}`); try { - const { tarball, packedPaths } = packTarball(); + const { tarball, packedEntries } = packTarball(); + assertBundledLicenseNotices(packedEntries, { + noticesRoot: repoRoot, + context: `The build packed from ${repoRoot}`, + }); + const packedPaths = packedEntries.map((file) => file.path); lintTarball(tarball); const installedRoot = installIntoCleanConsumer(tarball); + // A user's machine is the tree that has to carry the attribution, so the content policy runs again + // against the installed file, not only the working-tree copy the tarball was built from. + assertBundledLicenseNotices(packedEntries, { + noticesRoot: installedRoot, + context: `The installed package at ${installedRoot}`, + }); assertInstalledSnapshotBridge(installedRoot); assertInstalledTreeIsNotASourceCheckout(installedRoot); assertPackedTreeCarriesNoTestCode(installedRoot, packedPaths); diff --git a/scripts/lib/bundled-license-notices.ts b/scripts/lib/bundled-license-notices.ts new file mode 100644 index 0000000000..e390625fcb --- /dev/null +++ b/scripts/lib/bundled-license-notices.ts @@ -0,0 +1,141 @@ +import fs from 'node:fs'; +import path from 'node:path'; +import type { TsdownPlugin } from 'tsdown'; +import { renderLicenseNotices, type NoticeDependency } from '../size-report-package-notices.mjs'; + +type Manifest = { + name?: unknown; + version?: unknown; + license?: unknown; + author?: unknown; + private?: unknown; +}; + +/** + * Enumerates every module rendered into the bundle, attributes each third-party one to its package, + * and emits the notices beside the bundle. Extra notices are read from root's LICENSES directory. + */ +export function bundledLicenseNotices(root: string): TsdownPlugin { + return { + name: 'bundled-license-notices', + generateBundle(_options, bundle) { + const packages = new Map(); + for (const output of Object.values(bundle)) { + if (output.type !== 'chunk') continue; + for (const [id, module] of Object.entries(output.modules)) { + if (module.renderedLength === 0) continue; + const owner = thirdPartyPackage(root, id); + if (owner) packages.set(owner.directory, owner.manifest); + } + } + const dependencies = Array.from(packages, ([directory, manifest]) => + attributableDependency(root, directory, manifest), + ); + this.emitFile({ + type: 'asset', + fileName: 'THIRD_PARTY_NOTICES.txt', + source: renderLicenseNotices(deduplicate(dependencies)), + }); + }, + }; +} + +function readManifest(directory: string): Manifest | undefined { + const file = path.join(directory, 'package.json'); + if (!fs.existsSync(file)) return undefined; + const manifest = JSON.parse(fs.readFileSync(file, 'utf8')) as Manifest; + return typeof manifest.name === 'string' && manifest.name ? manifest : undefined; +} + +/** + * The package that owns a bundled module: the nearest `package.json` with a name. A `package.json` + * without one only marks a module type. Root and private workspace packages are this project's code. + * A module under `node_modules` must find its package before it leaves that `node_modules`. + */ +function thirdPartyPackage( + root: string, + moduleId: string, +): { directory: string; manifest: Manifest } | undefined { + const file = moduleId.replace(/^\0/, ''); + if (!path.isAbsolute(file)) return undefined; + for (let directory = path.dirname(file); directory !== root;) { + const manifest = readManifest(directory); + if (manifest) { + const vendored = directory.split(path.sep).includes('node_modules'); + return manifest.private === true && !vendored ? undefined : { directory, manifest }; + } + const parent = path.dirname(directory); + if (path.basename(directory) === 'node_modules' || parent === directory) { + throw new Error( + `Bundled module ${path.relative(root, file)} has no package.json naming its package.`, + ); + } + directory = parent; + } + return undefined; +} + +function readNamedFile(directory: string, prefix: RegExp): string | undefined { + const name = fs + .readdirSync(directory, { withFileTypes: true }) + .filter((entry) => entry.isFile() && prefix.test(entry.name)) + .map((entry) => entry.name) + .sort()[0]; + const text = name === undefined ? undefined : fs.readFileSync(path.join(directory, name), 'utf8'); + return text?.trim() ? text : undefined; +} + +function authorText(author: unknown): string | undefined { + if (typeof author === 'string') return author.trim() || undefined; + if (typeof author !== 'object' || author === null) return undefined; + const { name, email, url } = author as Record; + if (typeof name !== 'string' || !name) return undefined; + return `${name}${typeof email === 'string' && email ? ` <${email}>` : ''}${typeof url === 'string' && url ? ` (${url})` : ''}`; +} + +/** + * A bundled package must name its release, its license, and the license text before it is listed. + * The version is checked first so every later diagnostic names the package as the notices do. + */ +function attributableDependency( + root: string, + directory: string, + manifest: Manifest, +): NoticeDependency { + const name = String(manifest.name); + const version = typeof manifest.version === 'string' ? manifest.version.trim() : ''; + if (!version) throw new Error(`Bundled dependency ${name} is missing its version.`); + const license = typeof manifest.license === 'string' ? manifest.license.trim() : ''; + const licenseText = readNamedFile(directory, /^licen[cs]e/i); + if (!license || !licenseText) { + throw new Error( + `Bundled dependency ${name}@${version} is missing its license or license text.`, + ); + } + const texts = [licenseText]; + const noticeText = readNamedFile(directory, /^notice/i); + if (noticeText) texts.push(noticeText); + if (name === 'jpeg-js') { + const sourceNotices = fs.readFileSync(path.join(root, 'LICENSES/jpeg-js.txt'), 'utf8'); + const noticeVersion = sourceNotices + .split(/\r?\n/, 1)[0] + ?.match(/^jpeg-js (\S+) source-file notices$/)?.[1]; + if (noticeVersion !== version) { + throw new Error( + `Refresh LICENSES/jpeg-js.txt for bundled jpeg-js@${version}; its first line must name that version.`, + ); + } + texts.push(sourceNotices, fs.readFileSync(path.join(root, 'LICENSES/Apache-2.0.txt'), 'utf8')); + } + const author = authorText(manifest.author); + return { name, version, license, ...(author ? { author } : {}), texts }; +} + +/** One attribution per release: two installed copies of one `name@version` are one package. */ +function deduplicate(dependencies: readonly NoticeDependency[]): NoticeDependency[] { + return [ + ...new Map( + dependencies.map((dependency) => [`${dependency.name}@${dependency.version}`, dependency]), + ).values(), + ]; +} diff --git a/scripts/lib/cli-entrypoint.test.ts b/scripts/lib/cli-entrypoint.test.ts new file mode 100644 index 0000000000..0b359219c6 --- /dev/null +++ b/scripts/lib/cli-entrypoint.test.ts @@ -0,0 +1,305 @@ +import assert from 'node:assert/strict'; +import { spawn } from 'node:child_process'; +import { once } from 'node:events'; +import { test } from 'vitest'; + +test.each([false, true])('script exit flushes piped diagnostics (throws=%s)', async (throws) => { + const bytes = 4 * 1024 * 1024; + const stdoutTail = 'stdout diagnostic tail\n'; + const stderrTail = 'stderr diagnostic tail\n'; + const failure = 'fixture: expected failure\n'; + const source = [ + `import { runEntrypoint } from ${JSON.stringify(new URL('./cli-entrypoint.ts', import.meta.url).href)};`, + 'runEntrypoint("fixture", async () => {', + `process.stdout.write('o'.repeat(${bytes}) + ${JSON.stringify(stdoutTail)});`, + `process.stderr.write('e'.repeat(${bytes}) + ${JSON.stringify(stderrTail)});`, + throws ? 'throw new Error("expected failure");' : 'return 7;', + '});', + ].join('\n'); + const child = spawn( + process.execPath, + ['--experimental-strip-types', '--input-type=module', '--eval', source], + { stdio: ['ignore', 'pipe', 'pipe'] }, + ); + let stdout = ''; + let stderr = ''; + child.stdout.setEncoding('utf8').on('data', (chunk: string) => (stdout += chunk)); + child.stderr.setEncoding('utf8').on('data', (chunk: string) => (stderr += chunk)); + const [code, signal] = await once(child, 'close'); + assert.equal(signal, null); + assert.equal(code, throws ? 1 : 7); + assert.equal(stdout.length, bytes + stdoutTail.length); + assert.ok(stdout.endsWith(stdoutTail)); + assert.equal(stderr.length, bytes + stderrTail.length + (throws ? failure.length : 0)); + assert.ok(stderr.endsWith(throws ? failure : stderrTail)); +}); + +test.each([0, 7])('closed output pipe preserves command exit code %s', async (exitCode) => { + const source = [ + `import { runEntrypoint } from ${JSON.stringify(new URL('./cli-entrypoint.ts', import.meta.url).href)};`, + 'process.once("message", () => {', + ' process.disconnect();', + ` runEntrypoint("fixture", async () => { process.stdout.write("closed-consumer output"); return ${exitCode}; });`, + '});', + 'process.send("ready");', + ].join('\n'); + const child = spawn( + process.execPath, + ['--experimental-strip-types', '--input-type=module', '--eval', source], + { stdio: ['ignore', 'pipe', 'pipe', 'ipc'] }, + ); + let stderr = ''; + child.stderr.setEncoding('utf8').on('data', (chunk: string) => (stderr += chunk)); + const ready = once(child, 'message'); + const outputClosed = once(child.stdout, 'close'); + child.stdout.destroy(); + await outputClosed; + await ready; + const exited = once(child, 'close'); + child.send('run'); + const [code, signal] = await exited; + assert.equal(signal, null); + assert.equal(code, exitCode); + assert.equal(stderr, ''); +}); + +test.each( + ['EIO', 'ENOSPC', 'ERR_STREAM_DESTROYED'].flatMap((errorCode) => + [0, 7].map((exitCode) => ({ errorCode, exitCode })), + ), +)( + 'unexpected output error is reported without losing command failure ($errorCode, code=$exitCode)', + async ({ errorCode, exitCode }) => { + const source = [ + `import { runEntrypoint } from ${JSON.stringify(new URL('./cli-entrypoint.ts', import.meta.url).href)};`, + "import { Writable } from 'node:stream';", + `const output = new Writable({ write(_chunk, _encoding, done) { done(Object.assign(new Error('fixture write failure'), { code: ${JSON.stringify(errorCode)} })); } });`, + "Object.defineProperty(process, 'stdout', { value: output });", + `runEntrypoint('fixture', async () => ${exitCode});`, + ].join('\n'); + const child = spawn( + process.execPath, + ['--experimental-strip-types', '--input-type=module', '--eval', source], + { stdio: ['ignore', 'pipe', 'pipe'] }, + ); + let stderr = ''; + child.stderr.setEncoding('utf8').on('data', (chunk: string) => (stderr += chunk)); + const [code, signal] = await once(child, 'close'); + assert.equal(signal, null); + assert.equal(code, exitCode || 1); + assert.equal(stderr, 'fixture: unable to flush stdout: fixture write failure\n'); + }, +); + +test.each( + ['EIO', 'EPIPE'].flatMap((errorCode) => + [0, 7].flatMap((exitCode) => + [false, true].map((settles) => ({ errorCode, exitCode, settles })), + ), + ), +)( + 'early output error preserves status and reports non-EPIPE failures once ($errorCode, code=$exitCode, settles=$settles)', + async ({ errorCode, exitCode, settles }) => { + const source = [ + `import { runEntrypoint } from ${JSON.stringify(new URL('./cli-entrypoint.ts', import.meta.url).href)};`, + "import { Writable } from 'node:stream';", + `const output = new Writable({ write(_chunk, _encoding, done) { done(Object.assign(new Error('fixture write failure'), { code: ${JSON.stringify(errorCode)} })); } });`, + "Object.defineProperty(process, 'stdout', { value: output });", + `process.exitCode = ${exitCode};`, + `runEntrypoint('fixture', () => { process.stdout.write('trigger'); return ${settles ? `new Promise(resolve => setImmediate(() => resolve(${exitCode})))` : 'new Promise(() => {})'}; });`, + ].join('\n'); + const child = spawn( + process.execPath, + ['--experimental-strip-types', '--input-type=module', '--eval', source], + { stdio: ['ignore', 'pipe', 'pipe'] }, + ); + let stderr = ''; + child.stderr.setEncoding('utf8').on('data', (chunk: string) => (stderr += chunk)); + const [code, signal] = await once(child, 'close'); + assert.equal(signal, null); + assert.equal(code, exitCode || (errorCode === 'EPIPE' ? 0 : 1)); + assert.equal( + stderr, + errorCode === 'EPIPE' ? '' : 'fixture: unable to flush stdout: fixture write failure\n', + ); + }, +); + +test.each([0, 7])( + 'an early diagnostic queued after the initial drain is flushed (code=%s)', + async (exitCode) => { + const source = [ + `import { runEntrypoint } from ${JSON.stringify(new URL('./cli-entrypoint.ts', import.meta.url).href)};`, + "import { Writable } from 'node:stream';", + 'const stderr = process.stderr;', + "const output = new Writable({ write(_chunk, _encoding, done) { done(Object.assign(new Error('fixture write failure'), { code: 'EIO' })); } });", + 'const delayed = new Writable({ write(chunk, encoding, done) {', + ' if (chunk.length > 0) setTimeout(() => stderr.write(chunk, encoding, done), 50);', + ' else stderr.write(chunk, encoding, done);', + '} });', + "Object.defineProperty(process, 'stdout', { value: output });", + "Object.defineProperty(process, 'stderr', { value: delayed });", + `runEntrypoint('fixture', async () => ${exitCode});`, + ].join('\n'); + const child = spawn( + process.execPath, + ['--experimental-strip-types', '--input-type=module', '--eval', source], + { stdio: ['ignore', 'pipe', 'pipe'] }, + ); + let stderr = ''; + child.stderr.setEncoding('utf8').on('data', (chunk: string) => (stderr += chunk)); + const [code, signal] = await once(child, 'close'); + assert.equal(signal, null); + assert.equal(code, exitCode || 1); + assert.equal(stderr, 'fixture: unable to flush stdout: fixture write failure\n'); + }, +); + +test.each( + ['stdout', 'stderr'].flatMap((stream) => + ['EIO', 'EPIPE'].flatMap((errorCode) => + [0, 7].map((exitCode) => ({ stream, errorCode, exitCode })), + ), + ), +)( + 'a late $stream error during the other drain preserves status ($errorCode, code=$exitCode)', + async ({ stream, errorCode, exitCode }) => { + const source = [ + `import { runEntrypoint } from ${JSON.stringify(new URL('./cli-entrypoint.ts', import.meta.url).href)};`, + "import { Writable } from 'node:stream';", + 'const [errorCode, stream, exitCode] = process.argv.slice(1);', + 'const stderr = process.stderr;', + 'const faulty = new Writable({ write(_chunk, _encoding, done) { done(); } });', + 'let injected = false;', + 'const waiting = new Writable({ write(chunk, encoding, done) {', + ' if (!injected && chunk.length === 0) {', + ' injected = true;', + " setImmediate(() => { faulty.destroy(Object.assign(new Error('late write failure'), { code: errorCode })); setImmediate(done); });", + ' } else stderr.write(chunk, encoding, done);', + '} });', + 'Object.defineProperty(process, stream, { value: faulty });', + "Object.defineProperty(process, stream === 'stdout' ? 'stderr' : 'stdout', { value: waiting });", + "runEntrypoint('fixture', async () => Number(exitCode));", + ].join('\n'); + const child = spawn( + process.execPath, + [ + '--experimental-strip-types', + '--input-type=module', + '--eval', + source, + errorCode, + stream, + String(exitCode), + ], + { stdio: ['ignore', 'pipe', 'pipe'] }, + ); + let stderr = ''; + child.stderr.setEncoding('utf8').on('data', (chunk: string) => (stderr += chunk)); + const [code, signal] = await once(child, 'close'); + assert.equal(signal, null); + assert.equal(code, exitCode || (errorCode === 'EPIPE' ? 0 : 1)); + assert.equal( + stderr, + stream === 'stdout' && errorCode !== 'EPIPE' + ? 'fixture: unable to flush stdout: late write failure\n' + : '', + ); + }, +); + +// A real pipe must stay unread: synthetic write callbacks cannot prove process termination under backpressure. +test.each([ + { exitCode: 0, diagnosticDelayMs: 0 }, + { exitCode: 7, diagnosticDelayMs: 0 }, + { exitCode: 0, diagnosticDelayMs: 50 }, +])( + 'a stalled output reader cannot keep a finished command alive (code=$exitCode, diagnostic delay=$diagnosticDelayMs)', + async ({ exitCode, diagnosticDelayMs }) => { + const source = [ + `import { runEntrypoint } from ${JSON.stringify(new URL('./cli-entrypoint.ts', import.meta.url).href)};`, + "import { Writable } from 'node:stream';", + `if (${diagnosticDelayMs} > 0) {`, + ' const stderr = process.stderr;', + ' const delayed = new Writable({ write(chunk, encoding, done) {', + ` if (chunk.length > 0) setTimeout(() => stderr.write(chunk, encoding, done), ${diagnosticDelayMs});`, + ' else stderr.write(chunk, encoding, done);', + ' } });', + " Object.defineProperty(process, 'stderr', { value: delayed });", + '}', + 'setTimeout(() => process.exit(99), 3000);', + 'runEntrypoint("fixture", async () => {', + " process.stdout.write('o'.repeat(4 * 1024 * 1024));", + ' process.send({ queuedBytes: process.stdout.writableLength });', + ` return ${exitCode};`, + '});', + ].join('\n'); + const child = spawn( + process.execPath, + ['--experimental-strip-types', '--input-type=module', '--eval', source], + { stdio: ['ignore', 'pipe', 'pipe', 'ipc'] }, + ); + let stderr = ''; + child.stderr.setEncoding('utf8').on('data', (chunk: string) => (stderr += chunk)); + const stderrEnded = once(child.stderr, 'end'); + const exited = once(child, 'exit'); + try { + const [message] = await once(child, 'message'); + assert.ok( + message.queuedBytes > 0, + 'the real producer must have queued output behind the unread pipe', + ); + const [code, signal] = await exited; + await stderrEnded; + assert.equal(signal, null); + assert.equal( + code, + exitCode || 1, + 'the flush deadline must win before the independent leak guard', + ); + assert.equal(stderr, 'fixture: unable to flush stdout: output drain timed out\n'); + } finally { + child.stdout.destroy(); + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + } + }, +); + +// The diagnostic for an early stdout failure is written while main is still pending. Behind a real +// stderr pipe nobody reads, that write never completes, and a pending main does not keep Node alive +// on its own: only the stalled write does. The diagnostic must end at its deadline so the process can +// exit with the failure it retained, instead of waiting on the reader forever. +test('an early diagnostic behind an unread stderr pipe cannot keep a pending command alive', async () => { + const source = [ + `import { runEntrypoint } from ${JSON.stringify(new URL('./cli-entrypoint.ts', import.meta.url).href)};`, + "import { Writable } from 'node:stream';", + "const output = new Writable({ write(_chunk, _encoding, done) { done(Object.assign(new Error('fixture write failure'), { code: 'EIO' })); } });", + "Object.defineProperty(process, 'stdout', { value: output });", + 'setTimeout(() => process.exit(99), 4000).unref();', + "runEntrypoint('fixture', () => {", + " process.stderr.write('e'.repeat(4 * 1024 * 1024));", + ' process.send({ queuedBytes: process.stderr.writableLength }, () => process.disconnect());', + " process.stdout.write('trigger');", + ' return new Promise(() => {});', + '});', + ].join('\n'); + const child = spawn( + process.execPath, + ['--experimental-strip-types', '--input-type=module', '--eval', source], + { stdio: ['ignore', 'pipe', 'pipe', 'ipc'] }, + ); + const exited = once(child, 'exit'); + try { + const [message] = await once(child, 'message'); + assert.ok( + message.queuedBytes > 0, + 'the diagnostic must queue behind output the unread pipe has not taken', + ); + const [code, signal] = await exited; + assert.equal(signal, null); + assert.equal(code, 1, 'the diagnostic deadline must win before the independent leak guard'); + } finally { + child.stderr.destroy(); + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + } +}); diff --git a/scripts/lib/cli-entrypoint.ts b/scripts/lib/cli-entrypoint.ts index f75fcd7849..93df285d1a 100644 --- a/scripts/lib/cli-entrypoint.ts +++ b/scripts/lib/cli-entrypoint.ts @@ -1,12 +1,102 @@ -// Shared bootstrap for script entrypoints: run `main`, exit on its code, and turn a -// thrown error into one prefixed line rather than a stack trace. +const OUTPUT_DRAIN_TIMEOUT_MS = 2000; +function isClosedPipe(error: unknown): boolean { + return typeof error === 'object' && error !== null && 'code' in error && error.code === 'EPIPE'; +} + +function outputFailure(stream: NodeJS.WriteStream, error?: unknown): unknown { + const cause = stream.errored ?? error; + return cause != null && !isClosedPipe(cause) ? cause : undefined; +} + +class OutputDeadlineExceeded extends Error { + constructor() { + super('output drain timed out'); + } +} + +/** + * The one owner of output completion: settles with the stream's failure once what was written to it + * has flushed, or with `OutputDeadlineExceeded` at `deadline`. + */ +function settleOutput(stream: NodeJS.WriteStream, deadline: number): Promise { + return new Promise((resolve) => { + const finish = (error?: unknown): void => { + clearTimeout(timer); + stream.off('error', finish); + resolve(outputFailure(stream, error)); + }; + const timer = setTimeout( + () => finish(new OutputDeadlineExceeded()), + Math.max(0, deadline - Date.now()), + ); + if (stream.destroyed || !stream.writable) { + finish(stream.errored); + return; + } + stream.once('error', finish); + try { + stream.write('', finish); + } catch (error) { + finish(error); + } + }); +} + +// Shared bootstrap for script entrypoints. Output failure fails a successful command; a closed +// pipe preserves its code. Primary output and the stdout-failure diagnostic each settle by a deadline. +// A pending write holds the process even while main is pending, and Node cannot close stdio, so a +// diagnostic still unflushed at its deadline exits with the failure it reports. export function runEntrypoint(prefix: string, main: () => Promise): void { - main().then( - (code) => process.exit(code), - (error: unknown) => { - process.stderr.write(`${prefix}: ${error instanceof Error ? error.message : error}\n`); - process.exit(1); - }, - ); + let mainSettled = false; + // A pending main promise does not keep Node alive; retain early failures for natural exit. + const retainOutputFailure = (error: unknown): void => { + if (!isClosedPipe(error) && !Number(process.exitCode)) process.exitCode = 1; + }; + let diagnostic: Promise | undefined; + const reportStdoutFailure = (error: unknown): void => { + if ( + diagnostic || + process.stderr.destroyed || + !process.stderr.writable || + process.stderr.errored != null + ) + return; + const message = error instanceof Error ? error.message : String(error); + try { + process.stderr.write(`${prefix}: unable to flush stdout: ${message}\n`); + } catch (error) { + retainOutputFailure(error); + } + diagnostic = settleOutput(process.stderr, Date.now() + OUTPUT_DRAIN_TIMEOUT_MS); + void diagnostic.then((failure) => { + if (!mainSettled && failure instanceof OutputDeadlineExceeded) { + process.exit(Number(process.exitCode) || 1); + } + }); + }; + process.stderr.on('error', retainOutputFailure); + process.stdout.on('error', (error: unknown) => { + retainOutputFailure(error); + if (!isClosedPipe(error)) reportStdoutFailure(error); + }); + const exitAfterOutput = async (code: number): Promise => { + mainSettled = true; + const deadline = Date.now() + OUTPUT_DRAIN_TIMEOUT_MS; + const [initialStdoutError, initialStderrError] = await Promise.all([ + settleOutput(process.stdout, deadline), + settleOutput(process.stderr, deadline), + ]); + const stdoutError = initialStdoutError ?? outputFailure(process.stdout); + let stderrError = initialStderrError ?? outputFailure(process.stderr); + if (stdoutError != null && stderrError == null) { + reportStdoutFailure(stdoutError); + stderrError = await diagnostic; + } + process.exit(code || (stdoutError != null || stderrError != null ? 1 : 0)); + }; + main().then(exitAfterOutput, (error: unknown) => { + process.stderr.write(`${prefix}: ${error instanceof Error ? error.message : error}\n`); + return exitAfterOutput(1); + }); } diff --git a/scripts/size-report-package-notices.d.mts b/scripts/size-report-package-notices.d.mts new file mode 100644 index 0000000000..42d2b26324 --- /dev/null +++ b/scripts/size-report-package-notices.d.mts @@ -0,0 +1,13 @@ +// Type surface for the plain-Node notices format module; keep it in step with its exports. +export type NoticeDependency = Readonly<{ + name: string; + version: string; + license: string; + author?: string; + texts: readonly string[]; +}>; +export function renderLicenseNotices(dependencies: readonly NoticeDependency[]): string; +export function readLicenseNotices(contents: string): Readonly<{ + attributions: readonly Readonly<{ label: string; references: readonly number[] }>[]; + licenseTextIds: ReadonlySet; +}>; diff --git a/scripts/size-report-package-notices.mjs b/scripts/size-report-package-notices.mjs new file mode 100644 index 0000000000..d1546eb5b9 --- /dev/null +++ b/scripts/size-report-package-notices.mjs @@ -0,0 +1,106 @@ +// The THIRD_PARTY_NOTICES.txt format: the build renders it and the pack gate reads it. It imports +// nothing, so the standalone size-report copy runs it without the repository's dependencies. + +const NO_BUNDLED_DEPENDENCIES_NOTICE = 'No third-party dependencies are bundled.\n'; + +/** + * The heading that opens a numbered license section. Every attribution is written before the first + * one, so this line is also the boundary between the attributions and the license text. + */ +const LICENSE_SECTION_HEADING = /^\[(\d+)\]$/; +const AUTHOR_CLAUSE = '; Author: '; + +/** The attribution block is one line per package, so manifest text that spans lines is joined. */ +function singleLine(value) { + return value.replaceAll(/\s+/g, ' ').trim(); +} + +function normalizeText(text) { + return text.replaceAll('\r\n', '\n').trim(); +} + +/** + * Renders one `name@version (license): [n], …; Author: …` line per dependency, sorted by code unit, + * then each distinct text once under its `[n]` heading. + */ +export function renderLicenseNotices(dependencies) { + if (dependencies.length === 0) return NO_BUNDLED_DEPENDENCIES_NOTICE; + const texts = new Map(); + const reference = (text) => { + const normalized = normalizeText(text); + let id = texts.get(normalized); + if (id === undefined) { + id = texts.size + 1; + texts.set(normalized, id); + } + return `[${id}]`; + }; + const attributions = [...dependencies] + .sort((left, right) => { + const a = `${left.name}@${left.version}`; + const b = `${right.name}@${right.version}`; + return a < b ? -1 : a > b ? 1 : 0; + }) + .map( + (dependency) => + `${dependency.name}@${dependency.version} (${singleLine(dependency.license)}): ` + + dependency.texts.map(reference).join(', ') + + (dependency.author ? `${AUTHOR_CLAUSE}${singleLine(dependency.author)}` : ''), + ); + const sections = Array.from(texts, ([text, id]) => `[${id}]\n${text}`); + return `${[attributions.join('\n'), ...sections].join('\n\n')}\n`; +} + +/** + * The ids of the numbered sections that hold text. A section runs from its heading to the next + * heading, so a license body with blank lines keeps that body. A heading-shaped line inside a body + * only shortens its own section; it cannot fill an empty one. + */ +function readLicenseTextIds(lines) { + const idsWithText = new Set(); + let openId; + let seenText = false; + for (const line of lines) { + const heading = line.match(LICENSE_SECTION_HEADING); + if (heading) { + if (seenText) idsWithText.add(openId); + openId = Number(heading[1]); + seenText = false; + } else if (openId !== undefined && line.trim()) { + seenText = true; + } + } + if (seenText) idsWithText.add(openId); + return idsWithText; +} + +/** + * One attribution line, or undefined for any other line. The license field sits in the renderer's + * parentheses, so a grouped SPDX expression nests inside it and the field ends at the first `):`. + * References run to the author clause: an author holding `[n]` is metadata, not a reference. + */ +function readAttribution(line) { + const attribution = line.match(/^(\S+@\S+ \((.+?)\)): ?(.*)$/); + if (!attribution) return undefined; + const references = attribution[3].split(AUTHOR_CLAUSE, 1)[0]; + return { + label: attribution[1], + references: [...references.matchAll(/\[(\d+)\]/g)].map((reference) => Number(reference[1])), + }; +} + +/** + * Reads attributions only from the block before the first section: sections hold upstream license + * and NOTICE text verbatim, and prose there shaped like an attribution is not one. + */ +export function readLicenseNotices(contents) { + const lines = contents.replaceAll('\r\n', '\n').split('\n'); + const firstSection = lines.findIndex((line) => LICENSE_SECTION_HEADING.test(line)); + const attributionLines = firstSection === -1 ? lines : lines.slice(0, firstSection); + return { + attributions: attributionLines + .map(readAttribution) + .filter((attribution) => attribution !== undefined), + licenseTextIds: readLicenseTextIds(lines), + }; +} diff --git a/scripts/size-report-package.d.mts b/scripts/size-report-package.d.mts index 3ba2bff28a..2056499a20 100644 --- a/scripts/size-report-package.d.mts +++ b/scripts/size-report-package.d.mts @@ -4,11 +4,23 @@ export const DOCUMENTED_PACK_STUBS: readonly string[]; export function readMacosHelperTestTargets(manifestSource: string): string[]; export type NpmPackEntry = Readonly<{ path: string; size: number }>; +export function assertBundledLicenseNotices( + entries: readonly NpmPackEntry[], + options?: Readonly<{ noticesRoot?: string; context?: string }>, +): void; export function assertPublishPackageContents( entries: readonly NpmPackEntry[], options?: Readonly<{ + noticesRoot?: string; repoRoot?: string; requireSnapshotBridge?: boolean; macosHelperManifestSource?: string; }>, ): void; + +export function parseNpmPackOutput(stdout: string): Readonly<{ + filename: string; + size: number; + unpackedSize: number; + files: readonly NpmPackEntry[]; +}>; diff --git a/scripts/size-report-package.mjs b/scripts/size-report-package.mjs index 787f624012..fbf2628ead 100644 --- a/scripts/size-report-package.mjs +++ b/scripts/size-report-package.mjs @@ -1,6 +1,7 @@ import { execFileSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; +import { readLicenseNotices } from './size-report-package-notices.mjs'; export const SNAPSHOT_BRIDGE_ASSET_PATHS = Object.freeze([ 'apple/snapshot-bridge/SnapshotBridge.m', 'apple/snapshot-bridge/SnapshotBridgeRuntime.m', @@ -50,8 +51,9 @@ export function collectNpmPack(root) { { cwd: root, encoding: 'utf8' }, ); const pack = parseNpmPackOutput(stdout); - const entries = normalizeNpmPackEntries(pack); + const entries = pack.files; assertPublishPackageContents(entries, { + noticesRoot: root, repoRoot: root, requireSnapshotBridge: fs.existsSync(path.join(root, 'apple', 'snapshot-bridge')), }); @@ -65,7 +67,76 @@ export function collectNpmPack(root) { }; } +/** + * The notices asset a packed tree ships, and the build output it comes from. One path keeps the + * manifest check and the content check naming the same file. + */ +const BUNDLED_LICENSE_NOTICES_PATH = 'dist/src/THIRD_PARTY_NOTICES.txt'; + +function describeUnresolvedAttribution(attribution) { + return ( + `${attribution.label} -> ` + + (attribution.references.length > 0 + ? `[${attribution.references.join('], [')}]` + : 'no reference') + ); +} + +/** + * The published CLI bundles its runtime dependencies into `dist`, so the notices it ships must name + * them and carry the terms behind every name: each `name@version (license): [n]` attribution must + * point at a numbered section that holds text. The bundler's zero-dependency output — the generic + * "nothing bundled" line — names nobody, and a file of bare `[n]` headings has the right shape with + * no license terms in it, so a build that stopped bundling dependencies, or wrote headings without + * bodies, would otherwise keep clearing a gate that only counts the two kinds of line. + * + * This is the published CLI's policy, not the bundler's: a project that genuinely bundles nothing + * still gets the generic notice, and only this package is held to naming somebody. + */ +function assertLicenseNoticeContents(contents, context) { + const { attributions, licenseTextIds } = readLicenseNotices(contents); + const unresolved = attributions.filter( + (attribution) => + attribution.references.length === 0 || + attribution.references.some((reference) => !licenseTextIds.has(reference)), + ); + if (attributions.length > 0 && unresolved.length === 0) return; + throw new Error( + `${context} ships ${BUNDLED_LICENSE_NOTICES_PATH} with no bundled dependency notices: ` + + `expected every "name@version (license): [n]" attribution to reference a non-empty license ` + + `text, found ${attributions.length} attributions and ${licenseTextIds.size} license texts` + + (unresolved.length > 0 + ? `; unresolved: ${unresolved.map(describeUnresolvedAttribution).join('; ')}` + : '') + + '.', + ); +} + +/** + * Requires the notices asset the `npm pack` manifest declares, and — when `noticesRoot` names the tree + * that was packed or installed — that its contents attribute somebody. A manifest says only that a + * path exists and how many bytes it is, which the bundler's zero-dependency placeholder satisfies; + * reading the bytes is what lets the gate see the attribution a user actually receives. A tree without + * the file fails naming the path, rather than with a bare ENOENT from a read. + */ +export function assertBundledLicenseNotices(entries, options = {}) { + if (!entries.some((entry) => entry.path === BUNDLED_LICENSE_NOTICES_PATH && entry.size > 0)) { + throw new Error(`npm pack is missing non-empty ${BUNDLED_LICENSE_NOTICES_PATH}`); + } + if (!options.noticesRoot) return; + const noticesPath = path.join(options.noticesRoot, BUNDLED_LICENSE_NOTICES_PATH); + const context = options.context ?? options.noticesRoot; + if (!fs.existsSync(noticesPath)) { + throw new Error(`${context} has no ${BUNDLED_LICENSE_NOTICES_PATH}.`); + } + assertLicenseNoticeContents(fs.readFileSync(noticesPath, 'utf8'), context); +} + export function assertPublishPackageContents(entries, options = {}) { + assertBundledLicenseNotices(entries, { + noticesRoot: options.noticesRoot, + context: options.noticesRoot ? `npm pack of ${options.noticesRoot}` : undefined, + }); const paths = entries.map((entry) => entry.path); const requiredAssets = [ { directory: 'android/snapshot-helper/dist/', suffix: '.apk' }, @@ -157,9 +228,18 @@ export function assertPublishPackageContents(entries, options = {}) { } } -function parseNpmPackOutput(stdout) { +export function parseNpmPackOutput(stdout) { const parsed = JSON.parse(stdout); - return Array.isArray(parsed) ? parsed[0] : parsed; + const pack = Array.isArray(parsed) ? parsed[0] : parsed; + if (!pack || typeof pack.filename !== 'string' || pack.filename.length === 0) { + throw new Error('npm pack did not return a filename'); + } + for (const field of ['size', 'unpackedSize']) { + if (!Number.isSafeInteger(pack[field]) || pack[field] < 0) { + throw new Error(`npm pack returned an invalid ${field}: ${JSON.stringify(pack[field])}`); + } + } + return { ...pack, files: normalizeNpmPackEntries(pack) }; } function normalizeNpmPackEntries(pack) { diff --git a/tsdown.config.ts b/tsdown.config.ts index 26b29b5a65..4310dfdf72 100644 --- a/tsdown.config.ts +++ b/tsdown.config.ts @@ -2,6 +2,7 @@ import { buildEntries } from './scripts/build-entries.ts'; import fs from 'node:fs'; import { fileURLToPath } from 'node:url'; import { defineConfig, type TsdownPluginOption } from 'tsdown'; +import { bundledLicenseNotices } from './scripts/lib/bundled-license-notices.ts'; const typeScriptPackageJsonUrl = import.meta.resolve('typescript/package.json'); const { default: getTypeScript7ExePath } = await import( @@ -174,7 +175,7 @@ export default defineConfig({ }, outExtensions: () => ({ js: '.js', dts: '.d.ts' }), minify: true, - plugins: [dropAmbientDeclarationImports], + plugins: [dropAmbientDeclarationImports, bundledLicenseNotices(import.meta.dirname)], dts: { tsgo: { path: getTypeScript7ExePath(), diff --git a/vitest.config.ts b/vitest.config.ts index d3becdd644..06d7375b6f 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -184,6 +184,8 @@ export default defineConfig({ // Package attribution models npm-pack output. Keep it in the fast lane so every new // package path remains accounted for. 'scripts/__tests__/size-report-package.test.ts', + 'scripts/__tests__/bundled-license-notices.test.ts', + 'scripts/lib/cli-entrypoint.test.ts', // Publish preparation spawns only fixture-owned scripts and proves both Android // helper families are rebuilt through the shared release/size-report owner. 'scripts/__tests__/prepare-publish-assets.test.ts',