From 7a5a840dbf3e6c7157653504c5b0fdf7defb0ec9 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Wed, 4 Nov 2026 01:49:55 -0700 Subject: [PATCH 1/9] fix(release): supply the two license texts the managed CPython archive omits python-build-standalone 20260901's PYTHON.json declares licenses/LICENSE.zstd.txt (all three targets) and licenses/LICENSE.zlib-ng.txt (Linux, macOS), but the full archives do not contain them, so torch-native-e2e failed on every OS. The texts are taken byte-exact from the exact source archives that release pins (python/cpython-source-deps zstd-1.5.7 and zlib-ng-2.2.4, sha256 and size verified), kept under scripts/release/licenses/managed-python-supplemental/ with a provenance record each, and never presented as archive members. - The acceptance collector accepts a missing declared path only with a matching record for that target and release whose hash verifies. - generate-notices and check-attribution include and verify them; the 0.8.0 and 0.8.0-s3 notices and inventories are regenerated. - .gitattributes keeps the texts byte-exact on Windows checkouts. Co-Authored-By: Claude Opus 5.5 --- .gitattributes | 1 + .../README.md | 11 +- .../linux-x86_64-cpython-3.14.7/README.md | 6 +- .../missing-declared-license-references.md | 55 ++-- .../0.8.0-s3/THIRD-PARTY-NOTICES.txt | 172 ++++++++++++ .../0.8.0-s3/inventory.json | 109 +++++++- docs/release-attribution/0.8.0/README.md | 4 +- .../0.8.0/THIRD-PARTY-NOTICES.txt | 172 ++++++++++++ docs/release-attribution/0.8.0/inventory.json | 109 +++++++- scripts/release/check-attribution.cjs | 125 ++++++++- scripts/release/check-attribution.test.mjs | 151 ++++++++++- scripts/release/generate-notices.py | 48 ++++ .../zlib-ng-2.2.4-LICENSE.md | 19 ++ .../zlib-ng-2.2.4-LICENSE.md.provenance.json | 24 ++ .../zstd-1.5.7-LICENSE | 30 +++ .../zstd-1.5.7-LICENSE.provenance.json | 25 ++ .../release/managed_python_supplemental.py | 178 +++++++++++++ .../test_managed_python_supplemental.py | 163 ++++++++++++ .../torch-managed-python-acceptance.py | 246 +++++++++++++++++- 19 files changed, 1599 insertions(+), 49 deletions(-) create mode 100644 scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md create mode 100644 scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md.provenance.json create mode 100644 scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE create mode 100644 scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE.provenance.json create mode 100644 scripts/release/managed_python_supplemental.py create mode 100644 scripts/release/test_managed_python_supplemental.py diff --git a/.gitattributes b/.gitattributes index 3e4f7281a..6556a9eee 100644 --- a/.gitattributes +++ b/.gitattributes @@ -14,6 +14,7 @@ docs/plans/torch-cross-platform-runtime-management/reports/managed-python-licens # Preserve verbatim third-party legal bytes, including mixed line endings. scripts/release/licenses/* -text -whitespace +scripts/release/licenses/managed-python-supplemental/* -text -whitespace docs/release-attribution/**/THIRD-PARTY-NOTICES.txt -text -whitespace # Archive-captured CPython metadata and license texts are evidence and must remain verbatim. docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/**/managed-python-licenses/full-archive/** -text -whitespace diff --git a/docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/README.md b/docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/README.md index 354e23bdc..9d6bbda27 100644 --- a/docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/README.md +++ b/docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/README.md @@ -29,9 +29,10 @@ The Linux evidence README describes the accepted installation that produced its selected archive record. All per-file hashes can be checked directly against the manifests. -## Outstanding attribution evidence +## Declared files missing from the archives -The inventory includes the collected files for all three targets. It does not -certify completeness of declared references: the upstream Windows archive -omits a declared zstd license file. See the -[verified evidence gap and release gate](missing-declared-license-references.md). +Each `PYTHON.json` also declares zstd (all three targets) or zlib-ng (Linux and +macOS) license files that the upstream full archives omit. Release attribution +fills these from byte-verified supplemental texts taken from the source +archives that python-build-standalone pins, and labels them as such. See the +[evidence gap and its resolution](missing-declared-license-references.md). diff --git a/docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/linux-x86_64-cpython-3.14.7/README.md b/docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/linux-x86_64-cpython-3.14.7/README.md index abcf50170..0eab9b014 100644 --- a/docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/linux-x86_64-cpython-3.14.7/README.md +++ b/docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/linux-x86_64-cpython-3.14.7/README.md @@ -13,9 +13,9 @@ declared file references outside that directory. The files document the Linux archive only. Corresponding Windows x86_64 and macOS arm64 collections are retained beside this directory; all three are -included in the 0.7.0 attribution inventory. Collection does not establish -complete upstream license coverage: see the [missing declared license -report](../missing-declared-license-references.md) before release acceptance. +included in the 0.7.0 attribution inventory. The archive omits +two declared license files; see the [missing declared license +report](../missing-declared-license-references.md) for the supplemental texts. See [`full-archive-manifest.json`](managed-python-licenses/full-archive-manifest.json) for exact source identities and per-file hashes. The raw `PYTHON.json` and diff --git a/docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/missing-declared-license-references.md b/docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/missing-declared-license-references.md index 88611f81c..20da36857 100644 --- a/docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/missing-declared-license-references.md +++ b/docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/missing-declared-license-references.md @@ -1,28 +1,43 @@ # Missing declared upstream license references -**Status:** Open release attribution blocker. Owner: Pumas release/licensing review. +**Status:** Resolved for 0.8.0 by supplemental texts from the pinned source +archives. Owner: Pumas release/licensing review. -CodeRabbit identified a reference to `licenses/LICENSE.zstd.txt` in the Windows -CPython 3.14.7 `PYTHON.json` that is absent from the retained license files. -On 2026-09-29 the exact official Windows full archive was downloaded again: - -- Asset: `cpython-3.14.7+20260901-x86_64-pc-windows-msvc-pgo-full.tar.zst` -- SHA-256: `5363ec4aab59c24417f9877217aae95ca17f9ae6eb99c3bbfb25e4a76dcadafe` -- Size: 49,262,780 bytes; both size and digest match the retained manifest. -- The archive itself contains 19 `python/licenses/` files and omits the declared - zstd file. This is an upstream evidence gap, not an omitted extraction. - -The retained metadata and manifest remain byte-exact. Do not insert a guessed -license text or misrepresent an externally sourced file as an archive member. -The current inventory covers collected files; it does not certify every declared -reference. Audit of all three retained metadata records found: +The CPython 3.14.7 `PYTHON.json` files from python-build-standalone `20260901` +declare license files that the official full archives do not contain: - `windows-x86_64-cpython-3.14.7`: `licenses/LICENSE.zstd.txt`. - `macos-arm64-cpython-3.14.7`: `licenses/LICENSE.zlib-ng.txt`, `licenses/LICENSE.zstd.txt`. - `linux-x86_64-cpython-3.14.7`: `licenses/LICENSE.zlib-ng.txt`, `licenses/LICENSE.zstd.txt`. -Before release acceptance, obtain authoritative terms and exact source/version -mapping for the missing bundled component from the upstream provider, retain -separate provenance for any supplemental legal files, and update the notice -generator and inventory from that evidence. Native runtime test success does -not close this licensing gate. +The Windows archive was downloaded again on 2026-09-29 +(`cpython-3.14.7+20260901-x86_64-pc-windows-msvc-pgo-full.tar.zst`, SHA-256 +`5363ec4aab59c24417f9877217aae95ca17f9ae6eb99c3bbfb25e4a76dcadafe`, +49,262,780 bytes, matching the retained manifest). It holds 19 +`python/licenses/` files and no zstd file, so this is an upstream gap, not an +extraction error. The retained manifests and `PYTHON.json` files stay byte-exact. + +## Resolution + +`pythonbuild/downloads.py` at python-build-standalone tag `20260901` pins the +two components, and both source archives were downloaded and matched their pins: + +| Declared file | Component | SPDX | Source archive | Size | SHA-256 | Text taken from | +| --- | --- | --- | --- | ---: | --- | --- | +| `LICENSE.zstd.txt` | zstd 1.5.7 | BSD-3-Clause | [`zstd-1.5.7.tar.gz`](https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz) | 2,440,298 | `f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e` | `cpython-source-deps-zstd-1.5.7/LICENSE` | +| `LICENSE.zlib-ng.txt` | zlib-ng 2.2.4 | Zlib | [`zlib-ng-2.2.4.tar.gz`](https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.4.tar.gz) | 2,415,819 | `00bbd88709bc416cb96160ab61d3e1c8f76e106799af7328d0fe434dc7dd5004` | `cpython-source-deps-zlib-ng-2.2.4/LICENSE.md` | + +The texts are kept byte-exact, apart from the archive evidence, in +[`scripts/release/licenses/managed-python-supplemental/`](../../../../../scripts/release/licenses/managed-python-supplemental/). +Each has a `.provenance.json` record naming the declared path, release, targets, +source archive identity, path inside that archive and the text's SHA-256. They +are not archive members and are never listed as such. + +- The Torch acceptance collector accepts a declared path missing from the + archive only when a record for that exact path, target and release exists + and its text hash matches. Any other missing declared path still fails. Its + manifest lists archive files under `files` and supplemental texts under + `supplemental_files`. +- The notice generator and attribution checker require every declared path to + be an archive member or a verified supplemental text. Each supplemental text + gets its own notice, labelled with the source archive it came from. diff --git a/docs/release-attribution/0.8.0-s3/THIRD-PARTY-NOTICES.txt b/docs/release-attribution/0.8.0-s3/THIRD-PARTY-NOTICES.txt index a11efbef8..3d5a8e94e 100644 --- a/docs/release-attribution/0.8.0-s3/THIRD-PARTY-NOTICES.txt +++ b/docs/release-attribution/0.8.0-s3/THIRD-PARTY-NOTICES.txt @@ -10,6 +10,10 @@ is recorded in the release attribution report, not inferred from this list. The uv notice covers a runtime executable downloaded by Pumas, not a bundled binary. Managed CPython full-archive notices are target-scoped supersets for separately selected install-only distributions; each archive identity is recorded below. +Where a CPython PYTHON.json declares a license file that its full archive omits, +a separately labelled supplemental notice reproduces that component's license +file from the source archive pinned by python-build-standalone; it is not an +archive member. adler2 2.0.1 (Rust normal/build dependency, all desktop targets; conservative superset) Source: https://crates.io/api/v1/crates/adler2/2.0.1/download @@ -88870,6 +88874,71 @@ docs/license.html_lib. jloup@gzip.org madler@alumni.caltech.edu +============================================================================== + +zlib-ng 2.2.4 (Managed Python supplemental license text for aarch64-apple-darwin; PYTHON.json declares licenses/LICENSE.zlib-ng.txt but the CPython full archive omits it; supplied from cpython-source-deps-zlib-ng-2.2.4/LICENSE.md in the source archive pinned by python-build-standalone 20260901 (sha256 00bbd88709bc416cb96160ab61d3e1c8f76e106799af7328d0fe434dc7dd5004), not from the CPython archive) +Source: https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.4.tar.gz +Declared license: Zlib +--- managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md --- +(C) 1995-2024 Jean-loup Gailly and Mark Adler + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + +3. This notice may not be removed or altered from any source distribution. + + +============================================================================== + +zstd 1.5.7 (Managed Python supplemental license text for aarch64-apple-darwin; PYTHON.json declares licenses/LICENSE.zstd.txt but the CPython full archive omits it; supplied from cpython-source-deps-zstd-1.5.7/LICENSE in the source archive pinned by python-build-standalone 20260901 (sha256 f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e), not from the CPython archive) +Source: https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz +Declared license: BSD-3-Clause +--- managed-python-supplemental/zstd-1.5.7-LICENSE --- +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + ============================================================================== CPython full archive 3.14.7 (Managed Python full-archive notice superset for x86_64-pc-windows-msvc; selected install-only runtime tracked separately) @@ -91444,6 +91513,44 @@ docs/license.html_lib. jloup@gzip.org madler@alumni.caltech.edu +============================================================================== + +zstd 1.5.7 (Managed Python supplemental license text for x86_64-pc-windows-msvc; PYTHON.json declares licenses/LICENSE.zstd.txt but the CPython full archive omits it; supplied from cpython-source-deps-zstd-1.5.7/LICENSE in the source archive pinned by python-build-standalone 20260901 (sha256 f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e), not from the CPython archive) +Source: https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz +Declared license: BSD-3-Clause +--- managed-python-supplemental/zstd-1.5.7-LICENSE --- +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + ============================================================================== CPython full archive 3.14.7 (Managed Python full-archive notice superset for x86_64-unknown-linux-gnu; selected install-only runtime tracked separately) @@ -94018,6 +94125,71 @@ docs/license.html_lib. jloup@gzip.org madler@alumni.caltech.edu +============================================================================== + +zlib-ng 2.2.4 (Managed Python supplemental license text for x86_64-unknown-linux-gnu; PYTHON.json declares licenses/LICENSE.zlib-ng.txt but the CPython full archive omits it; supplied from cpython-source-deps-zlib-ng-2.2.4/LICENSE.md in the source archive pinned by python-build-standalone 20260901 (sha256 00bbd88709bc416cb96160ab61d3e1c8f76e106799af7328d0fe434dc7dd5004), not from the CPython archive) +Source: https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.4.tar.gz +Declared license: Zlib +--- managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md --- +(C) 1995-2024 Jean-loup Gailly and Mark Adler + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + +3. This notice may not be removed or altered from any source distribution. + + +============================================================================== + +zstd 1.5.7 (Managed Python supplemental license text for x86_64-unknown-linux-gnu; PYTHON.json declares licenses/LICENSE.zstd.txt but the CPython full archive omits it; supplied from cpython-source-deps-zstd-1.5.7/LICENSE in the source archive pinned by python-build-standalone 20260901 (sha256 f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e), not from the CPython archive) +Source: https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz +Declared license: BSD-3-Clause +--- managed-python-supplemental/zstd-1.5.7-LICENSE --- +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + ============================================================================== uv 0.12.18 (Managed Python provider runtime executable downloaded by Pumas; not bundled) diff --git a/docs/release-attribution/0.8.0-s3/inventory.json b/docs/release-attribution/0.8.0-s3/inventory.json index 3501583be..b41d3ab86 100644 --- a/docs/release-attribution/0.8.0-s3/inventory.json +++ b/docs/release-attribution/0.8.0-s3/inventory.json @@ -8715,6 +8715,32 @@ } ] }, + { + "name": "zlib-ng", + "version": "2.2.4", + "license_declared": "Zlib", + "source": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.4.tar.gz", + "scope": "Managed Python supplemental license text for aarch64-apple-darwin; PYTHON.json declares licenses/LICENSE.zlib-ng.txt but the CPython full archive omits it; supplied from cpython-source-deps-zlib-ng-2.2.4/LICENSE.md in the source archive pinned by python-build-standalone 20260901 (sha256 00bbd88709bc416cb96160ab61d3e1c8f76e106799af7328d0fe434dc7dd5004), not from the CPython archive", + "texts": [ + { + "path": "managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md", + "sha256": "6c9f0d975b41afaa34d22f55bb8986ce69e5cb7ad327cb2b28820cd425edf5ee" + } + ] + }, + { + "name": "zstd", + "version": "1.5.7", + "license_declared": "BSD-3-Clause", + "source": "https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz", + "scope": "Managed Python supplemental license text for aarch64-apple-darwin; PYTHON.json declares licenses/LICENSE.zstd.txt but the CPython full archive omits it; supplied from cpython-source-deps-zstd-1.5.7/LICENSE in the source archive pinned by python-build-standalone 20260901 (sha256 f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e), not from the CPython archive", + "texts": [ + { + "path": "managed-python-supplemental/zstd-1.5.7-LICENSE", + "sha256": "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8" + } + ] + }, { "name": "CPython full archive", "version": "3.14.7", @@ -8800,6 +8826,19 @@ } ] }, + { + "name": "zstd", + "version": "1.5.7", + "license_declared": "BSD-3-Clause", + "source": "https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz", + "scope": "Managed Python supplemental license text for x86_64-pc-windows-msvc; PYTHON.json declares licenses/LICENSE.zstd.txt but the CPython full archive omits it; supplied from cpython-source-deps-zstd-1.5.7/LICENSE in the source archive pinned by python-build-standalone 20260901 (sha256 f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e), not from the CPython archive", + "texts": [ + { + "path": "managed-python-supplemental/zstd-1.5.7-LICENSE", + "sha256": "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8" + } + ] + }, { "name": "CPython full archive", "version": "3.14.7", @@ -8885,6 +8924,32 @@ } ] }, + { + "name": "zlib-ng", + "version": "2.2.4", + "license_declared": "Zlib", + "source": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.4.tar.gz", + "scope": "Managed Python supplemental license text for x86_64-unknown-linux-gnu; PYTHON.json declares licenses/LICENSE.zlib-ng.txt but the CPython full archive omits it; supplied from cpython-source-deps-zlib-ng-2.2.4/LICENSE.md in the source archive pinned by python-build-standalone 20260901 (sha256 00bbd88709bc416cb96160ab61d3e1c8f76e106799af7328d0fe434dc7dd5004), not from the CPython archive", + "texts": [ + { + "path": "managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md", + "sha256": "6c9f0d975b41afaa34d22f55bb8986ce69e5cb7ad327cb2b28820cd425edf5ee" + } + ] + }, + { + "name": "zstd", + "version": "1.5.7", + "license_declared": "BSD-3-Clause", + "source": "https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz", + "scope": "Managed Python supplemental license text for x86_64-unknown-linux-gnu; PYTHON.json declares licenses/LICENSE.zstd.txt but the CPython full archive omits it; supplied from cpython-source-deps-zstd-1.5.7/LICENSE in the source archive pinned by python-build-standalone 20260901 (sha256 f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e), not from the CPython archive", + "texts": [ + { + "path": "managed-python-supplemental/zstd-1.5.7-LICENSE", + "sha256": "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8" + } + ] + }, { "name": "uv", "version": "0.12.18", @@ -9073,6 +9138,20 @@ "path": "docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/macos-arm64-cpython-3.14.7/managed-python-licenses/full-archive/licenses/LICENSE.zlib.txt", "sha256": "818922b2620f12801a12bf78e399644a30990e66824abd8ca8ec24d451d6f92c" } + ], + "supplemental_legal_files": [ + { + "declared_path": "licenses/LICENSE.zlib-ng.txt", + "path": "scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md", + "sha256": "6c9f0d975b41afaa34d22f55bb8986ce69e5cb7ad327cb2b28820cd425edf5ee", + "provenance": "scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md.provenance.json" + }, + { + "declared_path": "licenses/LICENSE.zstd.txt", + "path": "scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE", + "sha256": "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8", + "provenance": "scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE.provenance.json" + } ] }, { @@ -9164,6 +9243,14 @@ "path": "docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/windows-x86_64-cpython-3.14.7/managed-python-licenses/full-archive/licenses/LICENSE.zlib.txt", "sha256": "365e36720db366054863235aea5d6f440142ab968834bfaf81fa9d187c99b9b8" } + ], + "supplemental_legal_files": [ + { + "declared_path": "licenses/LICENSE.zstd.txt", + "path": "scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE", + "sha256": "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8", + "provenance": "scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE.provenance.json" + } ] }, { @@ -9255,6 +9342,20 @@ "path": "docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/linux-x86_64-cpython-3.14.7/managed-python-licenses/full-archive/licenses/LICENSE.zlib.txt", "sha256": "818922b2620f12801a12bf78e399644a30990e66824abd8ca8ec24d451d6f92c" } + ], + "supplemental_legal_files": [ + { + "declared_path": "licenses/LICENSE.zlib-ng.txt", + "path": "scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md", + "sha256": "6c9f0d975b41afaa34d22f55bb8986ce69e5cb7ad327cb2b28820cd425edf5ee", + "provenance": "scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md.provenance.json" + }, + { + "declared_path": "licenses/LICENSE.zstd.txt", + "path": "scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE", + "sha256": "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8", + "provenance": "scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE.provenance.json" + } ] } ], @@ -9336,7 +9437,11 @@ "rust/crates/pumas-rpc/Cargo.toml": "92b67b71767f6f495895c6d35a7c72107cfccc1cb60dedc400dd7f301a3a97f8", "scripts/release/artifact-plan.json": "66348a507f00ffdc74741fc7d70eab02f4a0eed4d3655354784cea2971662064", "scripts/release/licenses/managed-python-sources.json": "eabda4e2a25a92cfa8610865e79d11a50836930f53098efe89dd90778c4212ca", - "scripts/release/torch-managed-python-acceptance.py": "b0486387627d6ae98b4b6533adcceed93e93ebf868385626f3b13f1fb949893b" + "scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md": "6c9f0d975b41afaa34d22f55bb8986ce69e5cb7ad327cb2b28820cd425edf5ee", + "scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md.provenance.json": "3abbd4c93221c44bd33aa7012b5cd12bd1bce0a569768d076e820ee24959c10d", + "scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE": "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8", + "scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE.provenance.json": "7b15e355b4839ed30f39267b47c3048902765f21805c9caa5099a7ab5301431d", + "scripts/release/torch-managed-python-acceptance.py": "2fe0e7a7727026690c67485a0aa072c07d728836b3ad0f8db4eb5ab3edb66198" }, - "notices_sha256": "d8d7ca82bbf0ae6d37b590d364bd94493fff7315f948d3d2ab904dc749ae5f2a" + "notices_sha256": "b0d1d248fc093fa50447549a5a7105f5aa458ca6e7e5e6844fad6f3a2e386c5d" } diff --git a/docs/release-attribution/0.8.0/README.md b/docs/release-attribution/0.8.0/README.md index 30f3a66f3..a8eed4dba 100644 --- a/docs/release-attribution/0.8.0/README.md +++ b/docs/release-attribution/0.8.0/README.md @@ -9,7 +9,9 @@ text. Validate with `node scripts/release/check-attribution.cjs`. The bundled JPEG codec retains both its license and incorporated IJG terms; the wrapper MIT text is pinned to the published crate's upstream VCS revision. Managed CPython archive notices remain a conservative superset, bound to the -checked provider/catalog evidence. These records establish attribution inputs, +checked provider/catalog evidence. Declared CPython license files that the +upstream archives omit come from separately labelled, hash-checked supplemental +texts in `scripts/release/licenses/managed-python-supplemental/`. These records establish attribution inputs, not native execution, signing, model quality or permission to publish a release. An S3-enabled build must use the [S3 profile](../0.8.0-s3/README.md). diff --git a/docs/release-attribution/0.8.0/THIRD-PARTY-NOTICES.txt b/docs/release-attribution/0.8.0/THIRD-PARTY-NOTICES.txt index c4dda3ff8..5f0595f4e 100644 --- a/docs/release-attribution/0.8.0/THIRD-PARTY-NOTICES.txt +++ b/docs/release-attribution/0.8.0/THIRD-PARTY-NOTICES.txt @@ -10,6 +10,10 @@ is recorded in the release attribution report, not inferred from this list. The uv notice covers a runtime executable downloaded by Pumas, not a bundled binary. Managed CPython full-archive notices are target-scoped supersets for separately selected install-only distributions; each archive identity is recorded below. +Where a CPython PYTHON.json declares a license file that its full archive omits, +a separately labelled supplemental notice reproduces that component's license +file from the source archive pinned by python-build-standalone; it is not an +archive member. adler2 2.0.1 (Rust normal/build dependency, all desktop targets; conservative superset) Source: https://crates.io/api/v1/crates/adler2/2.0.1/download @@ -79740,6 +79744,71 @@ docs/license.html_lib. jloup@gzip.org madler@alumni.caltech.edu +============================================================================== + +zlib-ng 2.2.4 (Managed Python supplemental license text for aarch64-apple-darwin; PYTHON.json declares licenses/LICENSE.zlib-ng.txt but the CPython full archive omits it; supplied from cpython-source-deps-zlib-ng-2.2.4/LICENSE.md in the source archive pinned by python-build-standalone 20260901 (sha256 00bbd88709bc416cb96160ab61d3e1c8f76e106799af7328d0fe434dc7dd5004), not from the CPython archive) +Source: https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.4.tar.gz +Declared license: Zlib +--- managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md --- +(C) 1995-2024 Jean-loup Gailly and Mark Adler + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + +3. This notice may not be removed or altered from any source distribution. + + +============================================================================== + +zstd 1.5.7 (Managed Python supplemental license text for aarch64-apple-darwin; PYTHON.json declares licenses/LICENSE.zstd.txt but the CPython full archive omits it; supplied from cpython-source-deps-zstd-1.5.7/LICENSE in the source archive pinned by python-build-standalone 20260901 (sha256 f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e), not from the CPython archive) +Source: https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz +Declared license: BSD-3-Clause +--- managed-python-supplemental/zstd-1.5.7-LICENSE --- +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + ============================================================================== CPython full archive 3.14.7 (Managed Python full-archive notice superset for x86_64-pc-windows-msvc; selected install-only runtime tracked separately) @@ -82314,6 +82383,44 @@ docs/license.html_lib. jloup@gzip.org madler@alumni.caltech.edu +============================================================================== + +zstd 1.5.7 (Managed Python supplemental license text for x86_64-pc-windows-msvc; PYTHON.json declares licenses/LICENSE.zstd.txt but the CPython full archive omits it; supplied from cpython-source-deps-zstd-1.5.7/LICENSE in the source archive pinned by python-build-standalone 20260901 (sha256 f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e), not from the CPython archive) +Source: https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz +Declared license: BSD-3-Clause +--- managed-python-supplemental/zstd-1.5.7-LICENSE --- +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + ============================================================================== CPython full archive 3.14.7 (Managed Python full-archive notice superset for x86_64-unknown-linux-gnu; selected install-only runtime tracked separately) @@ -84888,6 +84995,71 @@ docs/license.html_lib. jloup@gzip.org madler@alumni.caltech.edu +============================================================================== + +zlib-ng 2.2.4 (Managed Python supplemental license text for x86_64-unknown-linux-gnu; PYTHON.json declares licenses/LICENSE.zlib-ng.txt but the CPython full archive omits it; supplied from cpython-source-deps-zlib-ng-2.2.4/LICENSE.md in the source archive pinned by python-build-standalone 20260901 (sha256 00bbd88709bc416cb96160ab61d3e1c8f76e106799af7328d0fe434dc7dd5004), not from the CPython archive) +Source: https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.4.tar.gz +Declared license: Zlib +--- managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md --- +(C) 1995-2024 Jean-loup Gailly and Mark Adler + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + +3. This notice may not be removed or altered from any source distribution. + + +============================================================================== + +zstd 1.5.7 (Managed Python supplemental license text for x86_64-unknown-linux-gnu; PYTHON.json declares licenses/LICENSE.zstd.txt but the CPython full archive omits it; supplied from cpython-source-deps-zstd-1.5.7/LICENSE in the source archive pinned by python-build-standalone 20260901 (sha256 f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e), not from the CPython archive) +Source: https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz +Declared license: BSD-3-Clause +--- managed-python-supplemental/zstd-1.5.7-LICENSE --- +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + ============================================================================== uv 0.12.18 (Managed Python provider runtime executable downloaded by Pumas; not bundled) diff --git a/docs/release-attribution/0.8.0/inventory.json b/docs/release-attribution/0.8.0/inventory.json index 37b5d6f57..68e2089a3 100644 --- a/docs/release-attribution/0.8.0/inventory.json +++ b/docs/release-attribution/0.8.0/inventory.json @@ -7776,6 +7776,32 @@ } ] }, + { + "name": "zlib-ng", + "version": "2.2.4", + "license_declared": "Zlib", + "source": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.4.tar.gz", + "scope": "Managed Python supplemental license text for aarch64-apple-darwin; PYTHON.json declares licenses/LICENSE.zlib-ng.txt but the CPython full archive omits it; supplied from cpython-source-deps-zlib-ng-2.2.4/LICENSE.md in the source archive pinned by python-build-standalone 20260901 (sha256 00bbd88709bc416cb96160ab61d3e1c8f76e106799af7328d0fe434dc7dd5004), not from the CPython archive", + "texts": [ + { + "path": "managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md", + "sha256": "6c9f0d975b41afaa34d22f55bb8986ce69e5cb7ad327cb2b28820cd425edf5ee" + } + ] + }, + { + "name": "zstd", + "version": "1.5.7", + "license_declared": "BSD-3-Clause", + "source": "https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz", + "scope": "Managed Python supplemental license text for aarch64-apple-darwin; PYTHON.json declares licenses/LICENSE.zstd.txt but the CPython full archive omits it; supplied from cpython-source-deps-zstd-1.5.7/LICENSE in the source archive pinned by python-build-standalone 20260901 (sha256 f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e), not from the CPython archive", + "texts": [ + { + "path": "managed-python-supplemental/zstd-1.5.7-LICENSE", + "sha256": "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8" + } + ] + }, { "name": "CPython full archive", "version": "3.14.7", @@ -7861,6 +7887,19 @@ } ] }, + { + "name": "zstd", + "version": "1.5.7", + "license_declared": "BSD-3-Clause", + "source": "https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz", + "scope": "Managed Python supplemental license text for x86_64-pc-windows-msvc; PYTHON.json declares licenses/LICENSE.zstd.txt but the CPython full archive omits it; supplied from cpython-source-deps-zstd-1.5.7/LICENSE in the source archive pinned by python-build-standalone 20260901 (sha256 f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e), not from the CPython archive", + "texts": [ + { + "path": "managed-python-supplemental/zstd-1.5.7-LICENSE", + "sha256": "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8" + } + ] + }, { "name": "CPython full archive", "version": "3.14.7", @@ -7946,6 +7985,32 @@ } ] }, + { + "name": "zlib-ng", + "version": "2.2.4", + "license_declared": "Zlib", + "source": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.4.tar.gz", + "scope": "Managed Python supplemental license text for x86_64-unknown-linux-gnu; PYTHON.json declares licenses/LICENSE.zlib-ng.txt but the CPython full archive omits it; supplied from cpython-source-deps-zlib-ng-2.2.4/LICENSE.md in the source archive pinned by python-build-standalone 20260901 (sha256 00bbd88709bc416cb96160ab61d3e1c8f76e106799af7328d0fe434dc7dd5004), not from the CPython archive", + "texts": [ + { + "path": "managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md", + "sha256": "6c9f0d975b41afaa34d22f55bb8986ce69e5cb7ad327cb2b28820cd425edf5ee" + } + ] + }, + { + "name": "zstd", + "version": "1.5.7", + "license_declared": "BSD-3-Clause", + "source": "https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz", + "scope": "Managed Python supplemental license text for x86_64-unknown-linux-gnu; PYTHON.json declares licenses/LICENSE.zstd.txt but the CPython full archive omits it; supplied from cpython-source-deps-zstd-1.5.7/LICENSE in the source archive pinned by python-build-standalone 20260901 (sha256 f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e), not from the CPython archive", + "texts": [ + { + "path": "managed-python-supplemental/zstd-1.5.7-LICENSE", + "sha256": "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8" + } + ] + }, { "name": "uv", "version": "0.12.18", @@ -8134,6 +8199,20 @@ "path": "docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/macos-arm64-cpython-3.14.7/managed-python-licenses/full-archive/licenses/LICENSE.zlib.txt", "sha256": "818922b2620f12801a12bf78e399644a30990e66824abd8ca8ec24d451d6f92c" } + ], + "supplemental_legal_files": [ + { + "declared_path": "licenses/LICENSE.zlib-ng.txt", + "path": "scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md", + "sha256": "6c9f0d975b41afaa34d22f55bb8986ce69e5cb7ad327cb2b28820cd425edf5ee", + "provenance": "scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md.provenance.json" + }, + { + "declared_path": "licenses/LICENSE.zstd.txt", + "path": "scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE", + "sha256": "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8", + "provenance": "scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE.provenance.json" + } ] }, { @@ -8225,6 +8304,14 @@ "path": "docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/windows-x86_64-cpython-3.14.7/managed-python-licenses/full-archive/licenses/LICENSE.zlib.txt", "sha256": "365e36720db366054863235aea5d6f440142ab968834bfaf81fa9d187c99b9b8" } + ], + "supplemental_legal_files": [ + { + "declared_path": "licenses/LICENSE.zstd.txt", + "path": "scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE", + "sha256": "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8", + "provenance": "scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE.provenance.json" + } ] }, { @@ -8316,6 +8403,20 @@ "path": "docs/plans/torch-cross-platform-runtime-management/reports/managed-python-license-collection/linux-x86_64-cpython-3.14.7/managed-python-licenses/full-archive/licenses/LICENSE.zlib.txt", "sha256": "818922b2620f12801a12bf78e399644a30990e66824abd8ca8ec24d451d6f92c" } + ], + "supplemental_legal_files": [ + { + "declared_path": "licenses/LICENSE.zlib-ng.txt", + "path": "scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md", + "sha256": "6c9f0d975b41afaa34d22f55bb8986ce69e5cb7ad327cb2b28820cd425edf5ee", + "provenance": "scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md.provenance.json" + }, + { + "declared_path": "licenses/LICENSE.zstd.txt", + "path": "scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE", + "sha256": "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8", + "provenance": "scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE.provenance.json" + } ] } ], @@ -8397,7 +8498,11 @@ "rust/crates/pumas-rpc/Cargo.toml": "92b67b71767f6f495895c6d35a7c72107cfccc1cb60dedc400dd7f301a3a97f8", "scripts/release/artifact-plan.json": "66348a507f00ffdc74741fc7d70eab02f4a0eed4d3655354784cea2971662064", "scripts/release/licenses/managed-python-sources.json": "eabda4e2a25a92cfa8610865e79d11a50836930f53098efe89dd90778c4212ca", - "scripts/release/torch-managed-python-acceptance.py": "b0486387627d6ae98b4b6533adcceed93e93ebf868385626f3b13f1fb949893b" + "scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md": "6c9f0d975b41afaa34d22f55bb8986ce69e5cb7ad327cb2b28820cd425edf5ee", + "scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md.provenance.json": "3abbd4c93221c44bd33aa7012b5cd12bd1bce0a569768d076e820ee24959c10d", + "scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE": "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8", + "scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE.provenance.json": "7b15e355b4839ed30f39267b47c3048902765f21805c9caa5099a7ab5301431d", + "scripts/release/torch-managed-python-acceptance.py": "2fe0e7a7727026690c67485a0aa072c07d728836b3ad0f8db4eb5ab3edb66198" }, - "notices_sha256": "9d050ff595057df42bcc67809a457e9b362dd286ed5b02ed218062d6ab0a570f" + "notices_sha256": "2d72871a68f29d50bbcee55091b624379b9e1c9b76a26b35116f5f309d533c39" } diff --git a/scripts/release/check-attribution.cjs b/scripts/release/check-attribution.cjs index 9e0991ab1..564b81ac4 100644 --- a/scripts/release/check-attribution.cjs +++ b/scripts/release/check-attribution.cjs @@ -7,6 +7,14 @@ const catalogPath = 'scripts/release/licenses/managed-python-sources.json'; const planPath = 'scripts/release/artifact-plan.json'; const pinsPath = 'rust/crates/pumas-app-manager/src/version_manager/managed_python.rs'; const mappingPath = 'scripts/release/torch-managed-python-acceptance.py'; +const supplementalPath = 'scripts/release/licenses/managed-python-supplemental'; +const supplementalSuffix = '.provenance.json'; +const supplementalReason = 'declared_by_python_json_but_absent_from_upstream_full_archive'; +const supplementalKeys = [ + 'schema_version', 'declared_path', 'supplemental_reason', 'note', + 'python_build_standalone_release', 'pin_authority', 'targets', 'component', + 'version', 'license', 'source_archive', 'path_in_source_archive', 'file', 'size', 'sha256', +].sort(); const uvArmTargets = { LinuxX8664: 'x86_64-unknown-linux-gnu', WindowsX8664: 'x86_64-pc-windows-msvc', @@ -45,6 +53,82 @@ function retainedFullArchiveMapping(source, expectedTargets) { return { release: release[0], flavors: Object.fromEntries(flavors.map(item => [item[1], item[2]])) }; } +function declaredLicensePaths(metadata) { + const paths = new Set(); + const visit = value => { + if (Array.isArray(value)) value.forEach(visit); + else if (value && typeof value === 'object') { + for (const [key, item] of Object.entries(value)) { + if ((key === 'license_path' || key === 'license_paths') && item !== null && item !== undefined) { + const values = typeof item === 'string' ? [item] : item; + if (!Array.isArray(values) || !values.every(entry => typeof entry === 'string' && entry)) { + throw new Error(`Invalid PYTHON.json license reference: ${JSON.stringify(item)}`); + } + values.forEach(entry => paths.add(entry)); + } else visit(item); + } + } + }; + visit(metadata); + return paths; +} + +// Supplemental texts satisfy declared CPython license paths that the upstream +// full archive omits. They are byte-verified against their provenance records +// and must never be presented as archive members. +function supplementalRecords(relative, hash, release, expectedTargets) { + const directory = relative(supplementalPath); + if (!fs.existsSync(directory)) return []; + const entries = fs.readdirSync(directory, { withFileTypes: true }); + if (entries.some(item => !item.isFile())) throw new Error('Supplemental license directory must hold regular files'); + const sha = value => typeof value === 'string' && /^[0-9a-f]{64}$/.test(value); + const positive = value => Number.isInteger(value) && value > 0; + const text = value => typeof value === 'string' && value.trim() !== ''; + const records = entries.filter(item => item.name.endsWith(supplementalSuffix)).map(item => { + const recordPath = `${supplementalPath}/${item.name}`; + const record = JSON.parse(fs.readFileSync(relative(recordPath))); + const source = record?.source_archive; + const targets = record?.targets; + if (!record || typeof record !== 'object' || Array.isArray(record) + || JSON.stringify(Object.keys(record).sort()) !== JSON.stringify(supplementalKeys) + || record.schema_version !== 1 || record.supplemental_reason !== supplementalReason + || record.python_build_standalone_release !== release + || record.pin_authority !== `https://github.com/astral-sh/python-build-standalone/blob/${release}/pythonbuild/downloads.py` + || typeof record.declared_path !== 'string' || !/^licenses\/[A-Za-z0-9._-]+$/.test(record.declared_path) + || !Array.isArray(targets) || !targets.length || !targets.every(text) + || JSON.stringify(targets) !== JSON.stringify([...new Set(targets)].sort()) + || !targets.every(target => expectedTargets.includes(target)) + || !['note', 'component', 'version', 'license', 'path_in_source_archive'].every(key => text(record[key])) + || !source || typeof source !== 'object' + || JSON.stringify(Object.keys(source).sort()) !== JSON.stringify(['sha256', 'size', 'url']) + || typeof source.url !== 'string' || !source.url.startsWith('https://') + || !positive(source.size) || !sha(source.sha256) + || record.file !== item.name.slice(0, -supplementalSuffix.length) + || !/^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(record.file) + || !positive(record.size) || !sha(record.sha256)) { + throw new Error(`Invalid managed CPython supplemental license record: ${item.name}`); + } + const textPath = `${supplementalPath}/${record.file}`; + const data = fs.readFileSync(relative(textPath)); + if (hash(relative(textPath)) !== record.sha256 || data.length !== record.size) { + throw new Error(`Changed managed CPython supplemental license: ${record.file}`); + } + return { ...record, recordPath, textPath }; + }); + const referenced = new Set(records.map(record => record.file)); + const orphans = entries.filter(item => !item.name.endsWith(supplementalSuffix) && !referenced.has(item.name)); + if (orphans.length) throw new Error(`Supplemental license texts without provenance: ${orphans.map(item => item.name)}`); + const claimed = new Set(); + for (const record of records) { + for (const target of record.targets) { + const key = `${target}\0${record.declared_path}`; + if (claimed.has(key)) throw new Error(`Duplicate supplemental license for ${target}: ${record.declared_path}`); + claimed.add(key); + } + } + return records; +} + function checkAttribution(directory, features = []) { if (!Array.isArray(features) || (features.length && (features.length !== 1 || features[0] !== 's3'))) { throw new Error('Unsupported release attribution feature profile'); @@ -100,6 +184,7 @@ function checkAttribution(directory, features = []) { const catalog = JSON.parse(fs.readFileSync(relative(catalogPath))); const entries = catalog.targets; const managed = inventory.managed_python; + const supplemental = supplementalRecords(relative, hash, pbs.release, expectedTargets); const coversTargets = values => Array.isArray(values) && values.length === 3 && expectedTargets.length === 3 && JSON.stringify(values.map(value => value.target).sort()) === JSON.stringify(expectedTargets); @@ -129,15 +214,45 @@ function checkAttribution(directory, features = []) { const pythonBytes = fs.readFileSync(relative(pythonPath)); const python = JSON.parse(pythonBytes); const files = manifest.files; - const declared = Array.isArray(files) ? files.map(item => item.path) : []; + const archived = Array.isArray(files) ? files.map(item => item.path) : []; const actual = fs.readdirSync(relative(`${archiveDir}/licenses`), { withFileTypes: true }) .filter(item => item.isFile()).map(item => `licenses/${item.name}`).sort(); - if (!declared.length || new Set(declared).size !== declared.length - || declared.some(name => !/^licenses\/[A-Za-z0-9._-]+$/.test(name)) - || JSON.stringify(declared.slice().sort()) !== JSON.stringify(actual) - || !declared.includes(python.license_path)) { + if (!archived.length || new Set(archived).size !== archived.length + || archived.some(name => !/^licenses\/[A-Za-z0-9._-]+$/.test(name)) + || JSON.stringify(archived.slice().sort()) !== JSON.stringify(actual) + || !archived.includes(python.license_path)) { throw new Error(`Incomplete managed CPython legal files: ${entry.target}`); } + // Every declared reference must be an archive member or a verified supplemental text. + const missing = [...declaredLicensePaths(python)].filter(name => !archived.includes(name)).sort(); + const applicable = supplemental.filter(item => item.targets.includes(entry.target)); + const supplied = applicable.filter(item => missing.includes(item.declared_path)) + .sort((a, b) => a.declared_path.localeCompare(b.declared_path)); + const unsatisfied = missing.filter(name => !supplied.some(item => item.declared_path === name)); + if (unsatisfied.length) { + throw new Error(`Declared managed CPython licenses absent from the full archive and without a verified supplemental text for ${entry.target}: ${unsatisfied}`); + } + const stale = applicable.filter(item => !supplied.includes(item)).map(item => item.declared_path); + if (stale.length) throw new Error(`Supplemental managed CPython licenses not needed for ${entry.target}: ${stale}`); + const supplementalFiles = supplied.map(item => ({ + declared_path: item.declared_path, path: item.textPath, sha256: item.sha256, provenance: item.recordPath, + })); + if (JSON.stringify(record.supplemental_legal_files) !== JSON.stringify(supplementalFiles)) { + throw new Error(`Managed CPython supplemental inventory mismatch: ${entry.target}`); + } + for (const item of supplied) { + input(item.textPath); + input(item.recordPath); + if (inventory.input_sha256[item.textPath] !== item.sha256) throw new Error(`Changed managed CPython supplemental license: ${item.textPath}`); + const label = item.textPath.slice('scripts/release/licenses/'.length); + const notice = inventory.packages.find(pkg => pkg.name === item.component && pkg.version === item.version + && pkg.scope?.startsWith(`Managed Python supplemental license text for ${entry.target};`)); + if (!notice || notice.source !== item.source_archive.url || notice.license_declared !== item.license + || !notice.scope.includes(item.declared_path) || !notice.scope.includes(item.source_archive.sha256) + || JSON.stringify(notice.texts) !== JSON.stringify([{ path: label, sha256: item.sha256 }])) { + throw new Error(`Missing managed CPython supplemental notice: ${entry.target} ${item.declared_path}`); + } + } const legalFiles = files.map(item => ({ path: `${archiveDir}/${item.path}`, sha256: item.sha256 })); const byPath = (a, b) => a.path.localeCompare(b.path); if (JSON.stringify(legalFiles.slice().sort(byPath)) !== JSON.stringify(record.legal_files?.slice().sort(byPath))) { diff --git a/scripts/release/check-attribution.test.mjs b/scripts/release/check-attribution.test.mjs index e9fb95f4d..c0eadff91 100644 --- a/scripts/release/check-attribution.test.mjs +++ b/scripts/release/check-attribution.test.mjs @@ -15,6 +15,9 @@ const planPath = 'scripts/release/artifact-plan.json'; const pinsPath = 'rust/crates/pumas-app-manager/src/version_manager/managed_python.rs'; const mappingPath = 'scripts/release/torch-managed-python-acceptance.py'; const directory = 'docs/release-attribution/0.7.0'; +const supplementalDir = 'scripts/release/licenses/managed-python-supplemental'; +const supplementalReason = 'declared_by_python_json_but_absent_from_upstream_full_archive'; +const python = process.env.PYTHON || (process.platform === 'win32' ? 'python' : 'python3'); const hash = data => createHash('sha256').update(data).digest('hex'); test('bundled JPEG attribution retains IJG terms and exact wrapper provenance', () => { @@ -44,6 +47,28 @@ function fixture(t) { write(pinsPath, `const UV_VERSION: &str = "0.12.18";\nconst UV_BASE_URL: &str = "https://releases.astral.sh/github/uv/releases/download/0.12.18/";\nfn pin(self) -> UvPin {\n${targets.map((target, index) => ` Self::${arms[index]} => UvPin {\n archive: "uv-${target}.tar.gz",\n sha256: "${uvHashes[target]}",`).join('\n')}\n fn accepts_observed`, true); write(mappingPath, `REVIEWED_PBS_RELEASE = "20260901"\nFULL_FLAVORS = {\n${targets.map(target => ` "${target}": "${flavors[target]}",`).join('\n')}\n}\n`, true); write('scripts/release/licenses/source.txt', 'upstream'); + // Mirrors PBS 20260901: PYTHON.json declares texts its full archive omits. + const supplemental = [ + { declared_path: 'licenses/LICENSE.zlib-ng.txt', component: 'zlib-ng', version: '2.2.4', license: 'Zlib', + file: 'zlib-ng-2.2.4-LICENSE.md', text: 'zlib-ng terms\n', targets: targets.filter(target => !target.includes('windows')).sort() }, + { declared_path: 'licenses/LICENSE.zstd.txt', component: 'zstd', version: '1.5.7', license: 'BSD-3-Clause', + file: 'zstd-1.5.7-LICENSE', text: 'zstd terms\n', targets: targets.slice().sort() }, + ].map(item => { + const record = { + schema_version: 1, declared_path: item.declared_path, supplemental_reason: supplementalReason, + note: 'Fixture text supplied outside the archive.', python_build_standalone_release: '20260901', + pin_authority: 'https://github.com/astral-sh/python-build-standalone/blob/20260901/pythonbuild/downloads.py', + targets: item.targets, component: item.component, version: item.version, license: item.license, + source_archive: { url: `https://example.invalid/${item.component}.tar.gz`, size: 10, sha256: 'b'.repeat(64) }, + path_in_source_archive: `${item.component}/LICENSE`, file: item.file, + size: Buffer.byteLength(item.text), sha256: hash(item.text), + }; + const textPath = `${supplementalDir}/${item.file}`; + const recordPath = `${textPath}.provenance.json`; + write(textPath, item.text, true); + json(recordPath, record, true); + return { record, textPath, recordPath }; + }); write(`${directory}/THIRD-PARTY-NOTICES.txt`, 'license text'); const entries = []; const managed = []; @@ -60,7 +85,9 @@ function fixture(t) { const fullName = `cpython-3.14.7+20260901-${target}-${flavors[target]}-full.tar.zst`; const full = `https://github.com/astral-sh/python-build-standalone/releases/download/20260901/${encodeURIComponent(fullName)}`; const legal = target.includes('windows') ? 'Windows legal\r\nsecond line\r\n' : 'Legal text\n'; - const python = { target_triple: target, python_version: '3.14.7', build_options: flavors[target], license_path: 'licenses/LICENSE.cpython.txt', licenses: ['Python-2.0'] }; + const supplied = supplemental.filter(item => item.record.targets.includes(target)); + const python = { target_triple: target, python_version: '3.14.7', build_options: flavors[target], license_path: 'licenses/LICENSE.cpython.txt', licenses: ['Python-2.0'], + build_info: { extensions: { _zstd: [{ license_paths: supplied.map(item => item.record.declared_path) }] } } }; const pythonBytes = Buffer.from(`${JSON.stringify(python)}\n`); write(legalPath, legal, true); write(pythonPath, pythonBytes, true); @@ -80,10 +107,20 @@ function fixture(t) { provider_archive_sha256: uvHashes[target], full_archive_url: full, full_archive_sha256: 'd'.repeat(64), python_json: pythonPath, legal_files: [{ path: legalPath, sha256: hash(legal) }], + supplemental_legal_files: supplied.map(item => ({ declared_path: item.record.declared_path, + path: item.textPath, sha256: item.record.sha256, provenance: item.recordPath })), }); - packages.push({ name: 'CPython full archive', version: '3.14.7', scope: `full archive ${target}`, - source: full, texts: [{ path: 'licenses/LICENSE.cpython.txt', sha256: hash(legal) }], + packages.push({ name: 'CPython full archive', version: '3.14.7', license_declared: 'Python-2.0', source: full, + scope: `Managed Python full-archive notice superset for ${target}; selected install-only runtime tracked separately`, + texts: [{ path: 'licenses/LICENSE.cpython.txt', sha256: hash(legal) }], }); + for (const { record } of supplied) { + packages.push({ name: record.component, version: record.version, license_declared: record.license, + source: record.source_archive.url, + scope: `Managed Python supplemental license text for ${target}; PYTHON.json declares ${record.declared_path} but the CPython full archive omits it; supplied from ${record.path_in_source_archive} in the source archive pinned by python-build-standalone 20260901 (sha256 ${record.source_archive.sha256}), not from the CPython archive`, + texts: [{ path: `managed-python-supplemental/${record.file}`, sha256: record.sha256 }], + }); + } paths[target] = { reportPath, manifestPath, pythonPath, legalPath }; } json(catalogPath, { schema_version: 1, targets: entries }, true); @@ -95,11 +132,36 @@ function fixture(t) { fs.copyFileSync(path.join(source, '.gitattributes'), path.join(root, '.gitattributes')); const run = (features = []) => spawnSync(process.execPath, [path.join(root, 'scripts/release/check-attribution.cjs'), '--directory', path.join(root, directory), ...(features.length ? ['--features', ...features] : [])], { encoding: 'utf8' }); - return { root, run, write, json, inventory, paths }; + const save = () => json(`${directory}/inventory.json`, inventory); + return { root, run, write, json, inventory, paths, supplemental, save }; +} + +// Run the real generator's managed-CPython collection against a fixture root. +function generateManagedPython(root) { + const probe = `import importlib.util, json, pathlib, sys +spec = importlib.util.spec_from_file_location('notices', sys.argv[1]) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) +module.ROOT = pathlib.Path(sys.argv[2]) +module.LICENSES = module.ROOT / 'scripts/release/licenses' +packages = [] +def add(name, version, license_id, source, root, files, scope): + packages.append({'name': name, 'version': version, 'license_declared': license_id, 'source': source, 'scope': scope, + 'texts': [{'path': file.relative_to(root if file.is_relative_to(root) else module.LICENSES).as_posix(), + 'sha256': module.digest(file.read_bytes())} for file in files]}) +try: + inventory, inputs, _ = module.collect_managed_python(add) +except ValueError as error: + print(json.dumps({'error': str(error)})) +else: + print(json.dumps({'inventory': inventory, 'inputs': sorted(inputs), 'packages': packages}))`; + const result = spawnSync(python, ['-c', probe, path.join(source, 'scripts/release/generate-notices.py'), root], { encoding: 'utf8' }); + assert.equal(result.status, 0, result.stderr); + return JSON.parse(result.stdout); } test('attribution survives a Windows-style checkout with original legal bytes', t => { - const { root, run, inventory, paths } = fixture(t); + const { root, run, inventory, paths, supplemental } = fixture(t); assert.equal(run().status, 0); const checkout = path.join(path.dirname(root), `${path.basename(root)}-checkout`); t.after(() => fs.rmSync(checkout, { recursive: true, force: true })); @@ -114,7 +176,10 @@ test('attribution survives a Windows-style checkout with original legal bytes', assert.equal(checked.status, 0, checked.stderr); const windowsLegal = paths['x86_64-pc-windows-msvc'].legalPath; assert.deepEqual(fs.readFileSync(path.join(checkout, windowsLegal)), fs.readFileSync(path.join(root, windowsLegal))); - assert.equal(Object.keys(inventory.input_sha256).length, 12 + targets.length * 4); + for (const { textPath } of supplemental) { + assert.deepEqual(fs.readFileSync(path.join(checkout, textPath)), fs.readFileSync(path.join(root, textPath))); + } + assert.equal(Object.keys(inventory.input_sha256).length, 12 + targets.length * 4 + supplemental.length * 2); }); test('S3 notices require the explicit feature profile and complete Rust closure', t => { @@ -140,7 +205,6 @@ test('S3 notices require the explicit feature profile and complete Rust closure' test('generator refuses selected dependencies absent from metadata instead of omitting notices', t => { const data = fixture(t); - const python = process.env.PYTHON || (process.platform === 'win32' ? 'python' : 'python3'); const probe = `import importlib.util, pathlib, sys\nspec = importlib.util.spec_from_file_location('notices', sys.argv[1])\nmodule = importlib.util.module_from_spec(spec)\nspec.loader.exec_module(module)\nmodule.LICENSES = pathlib.Path(sys.argv[2])\n(module.LICENSES / 'sources.json').write_text('[]')\nseen = []\ndef run(*args):\n seen.append(args)\n if args[1] == 'metadata':\n assert '--features' in args and 'pumas-rpc/s3' in args\n return '{"packages": []}'\n return 'aws-sdk-s3 v1.137.0\\n'\nmodule.run = run\ntry:\n module.collect(('s3',))\nexcept ValueError as error:\n assert 'Incomplete Rust attribution closure' in str(error), str(error)\nelse:\n raise AssertionError('selected SDK was silently omitted')`; const result = spawnSync(python, ['-c', probe, path.join(source, 'scripts/release/generate-notices.py'), path.join(data.root, 'scripts/release/licenses')], { encoding: 'utf8' }); assert.equal(result.status, 0, result.stderr); @@ -246,7 +310,6 @@ test('checker binds reports to shipped uv pins and the reviewed PBS build', t => test('swapping complete Rust target pin arms is rejected after rehashing', t => { const data = fixture(t); - const python = process.env.PYTHON || (process.platform === 'win32' ? 'python' : 'python3'); const probe = `import importlib.util, pathlib, sys\nspec = importlib.util.spec_from_file_location('notices', sys.argv[1])\nmodule = importlib.util.module_from_spec(spec)\nspec.loader.exec_module(module)\nmodule.ROOT = pathlib.Path(sys.argv[2])\nmodule.retained_provider_pins(set(sys.argv[3].split(',')))`; const validateGeneratorPins = () => spawnSync(python, ['-c', probe, path.join(source, 'scripts/release/generate-notices.py'), data.root, targets.join(',')], { encoding: 'utf8' }); assert.equal(validateGeneratorPins().status, 0); @@ -316,3 +379,75 @@ test('checker requires each legal notice text exactly once', t => { data.json(`${directory}/inventory.json`, data.inventory); assert.match(data.run().stderr, /Missing managed CPython full-archive notice/); }); + +test('supplemental license rules pass their Python unit tests', () => { + const result = spawnSync(python, ['-m', 'unittest', '-v', 'test_managed_python_supplemental'], + { cwd: path.join(source, 'scripts/release'), encoding: 'utf8', timeout: 30_000 }); + assert.equal(result.status, 0, result.stderr || String(result.error)); +}); + +test('generator and checker agree on verified supplemental CPython notices', t => { + const data = fixture(t); + assert.equal(data.run().status, 0, data.run().stderr); + const generated = generateManagedPython(data.root); + assert.equal(generated.error, undefined, generated.error); + const byTarget = (a, b) => a.target.localeCompare(b.target); + assert.deepEqual(generated.inventory, data.inventory.managed_python.slice().sort(byTarget)); + const byNotice = (a, b) => `${a.scope}\0${a.name}`.localeCompare(`${b.scope}\0${b.name}`); + assert.deepEqual(generated.packages.slice().sort(byNotice), data.inventory.packages.slice().sort(byNotice)); + const base = new Set(['rust/Cargo.lock', 'rust/Cargo.toml', 'rust/crates/pumas-core/Cargo.toml', 'rust/crates/pumas-rpc/Cargo.toml', 'rust/crates/pumas-app-manager/Cargo.toml', 'pnpm-lock.yaml', 'electron/package.json', 'frontend/package.json']); + assert.deepEqual(generated.inputs, Object.keys(data.inventory.input_sha256).filter(name => !base.has(name)).sort()); + const windows = generated.inventory.find(item => item.target === 'x86_64-pc-windows-msvc'); + assert.deepEqual(windows.supplemental_legal_files.map(item => item.declared_path), ['licenses/LICENSE.zstd.txt']); + for (const item of generated.inventory) { + assert.ok(item.legal_files.every(file => !file.path.startsWith(supplementalDir)), 'supplemental text listed as archive member'); + } +}); + +test('supplemental text whose bytes differ from its provenance is refused', t => { + const data = fixture(t); + const { textPath } = data.supplemental[1]; + data.write(textPath, 'altered zstd terms\n'); + assert.match(data.run().stderr, /Stale release attribution/); + data.inventory.input_sha256[textPath] = hash('altered zstd terms\n'); + data.save(); + assert.match(data.run().stderr, /Changed managed CPython supplemental license/); + assert.match(generateManagedPython(data.root).error, /Changed managed CPython supplemental license/); +}); + +test('declared licenses absent from the archive need an exact verified supplemental text', t => { + const unsupplied = fixture(t); + const zlib = unsupplied.supplemental[0]; + for (const name of [zlib.textPath, zlib.recordPath]) { + fs.unlinkSync(path.join(unsupplied.root, name)); + delete unsupplied.inventory.input_sha256[name]; + } + for (const item of unsupplied.inventory.managed_python) { + item.supplemental_legal_files = item.supplemental_legal_files.filter(file => file.path !== zlib.textPath); + } + unsupplied.save(); + assert.match(unsupplied.run().stderr, /absent from the full archive and without a verified supplemental text .*LICENSE\.zlib-ng\.txt/); + assert.match(generateManagedPython(unsupplied.root).error, /absent from the full archive.*LICENSE\.zlib-ng\.txt/); + + const undeclared = fixture(t); + const extra = { ...undeclared.supplemental[1].record, declared_path: 'licenses/LICENSE.other.txt', file: 'other-LICENSE' }; + undeclared.write(`${supplementalDir}/other-LICENSE`, 'zstd terms\n', false); + undeclared.json(`${supplementalDir}/other-LICENSE.provenance.json`, extra); + assert.match(undeclared.run().stderr, /Supplemental managed CPython licenses not needed .*LICENSE\.other\.txt/); + assert.match(generateManagedPython(undeclared.root).error, /not needed .*LICENSE\.other\.txt/); + + const orphan = fixture(t); + orphan.write(`${supplementalDir}/unexplained-LICENSE`, 'terms\n'); + assert.match(orphan.run().stderr, /Supplemental license texts without provenance/); + + const presented = fixture(t); + const notice = presented.inventory.packages.find(item => item.name === 'zstd'); + notice.texts = [{ path: 'licenses/LICENSE.zstd.txt', sha256: notice.texts[0].sha256 }]; + presented.save(); + assert.match(presented.run().stderr, /Missing managed CPython supplemental notice/); + + const omitted = fixture(t); + omitted.inventory.managed_python[0].supplemental_legal_files = []; + omitted.save(); + assert.match(omitted.run().stderr, /Managed CPython supplemental inventory mismatch/); +}); diff --git a/scripts/release/generate-notices.py b/scripts/release/generate-notices.py index b30aeffeb..7def63324 100644 --- a/scripts/release/generate-notices.py +++ b/scripts/release/generate-notices.py @@ -3,6 +3,7 @@ import argparse import hashlib +import importlib.util import json from pathlib import Path import re @@ -17,6 +18,11 @@ ARTIFACT_PLAN = "scripts/release/artifact-plan.json" MANAGED_PYTHON_PINS = "rust/crates/pumas-app-manager/src/version_manager/managed_python.rs" PBS_MAPPING = "scripts/release/torch-managed-python-acceptance.py" +_SUPPLEMENTAL_SPEC = importlib.util.spec_from_file_location( + "managed_python_supplemental", Path(__file__).with_name("managed_python_supplemental.py") +) +SUPPLEMENTAL = importlib.util.module_from_spec(_SUPPLEMENTAL_SPEC) +_SUPPLEMENTAL_SPEC.loader.exec_module(SUPPLEMENTAL) UV_ARM_TARGETS = { "LinuxX8664": "x86_64-unknown-linux-gnu", "WindowsX8664": "x86_64-pc-windows-msvc", @@ -144,6 +150,9 @@ def collect_managed_python(add): [], {ARTIFACT_PLAN, MANAGED_PYTHON_CATALOG, MANAGED_PYTHON_PINS, PBS_MAPPING}, ) + supplemental_records = SUPPLEMENTAL.load_records( + repository_file(SUPPLEMENTAL.DIRECTORY), pbs_release, expected_targets + ) for item in sorted(entries, key=lambda item: item["target"]): target = item["target"] report_name = item["runtime_report"] @@ -224,6 +233,13 @@ def collect_managed_python(add): raise ValueError(f"Changed managed CPython legal file: {name}") legal_files.append({"path": name, "sha256": entry["sha256"]}) inputs.add(name) + supplemental = SUPPLEMENTAL.select( + supplemental_records, + target, + SUPPLEMENTAL.declared_license_paths(python), + set(names), + strict=True, + ) add( "CPython full archive", version, @@ -236,6 +252,33 @@ def collect_managed_python(add): ], f"Managed Python full-archive notice superset for {target}; selected install-only runtime tracked separately", ) + supplemental_files = [] + for record in supplemental: + text_name = record["text_path"].relative_to(ROOT).as_posix() + record_name = record["record_path"].relative_to(ROOT).as_posix() + source = record["source_archive"] + add( + record["component"], + record["version"], + record["license"], + source["url"], + LICENSES, + [record["text_path"]], + f"Managed Python supplemental license text for {target}; PYTHON.json declares " + f"{record['declared_path']} but the CPython full archive omits it; supplied " + f"from {record['path_in_source_archive']} in the source archive pinned by " + f"python-build-standalone {record['python_build_standalone_release']} " + f"(sha256 {source['sha256']}), not from the CPython archive", + ) + supplemental_files.append( + { + "declared_path": record["declared_path"], + "path": text_name, + "sha256": record["sha256"], + "provenance": record_name, + } + ) + inputs.update((text_name, record_name)) inventory.append( { "target": target, @@ -250,6 +293,7 @@ def collect_managed_python(add): "full_archive_sha256": manifest["archive_sha256"], "python_json": python_name, "legal_files": legal_files, + "supplemental_legal_files": supplemental_files, } ) return inventory, inputs, uv_version @@ -476,6 +520,10 @@ def add(name, version, license_id, source, root, files, scope): The uv notice covers a runtime executable downloaded by Pumas, not a bundled binary. Managed CPython full-archive notices are target-scoped supersets for separately selected install-only distributions; each archive identity is recorded below. +Where a CPython PYTHON.json declares a license file that its full archive omits, +a separately labelled supplemental notice reproduces that component's license +file from the source archive pinned by python-build-standalone; it is not an +archive member. """ text = header + ("\n\n" + "=" * 78 + "\n\n").join(sections) + "\n" diff --git a/scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md b/scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md new file mode 100644 index 000000000..e866d7ac1 --- /dev/null +++ b/scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md @@ -0,0 +1,19 @@ +(C) 1995-2024 Jean-loup Gailly and Mark Adler + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + +3. This notice may not be removed or altered from any source distribution. diff --git a/scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md.provenance.json b/scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md.provenance.json new file mode 100644 index 000000000..edd635e0e --- /dev/null +++ b/scripts/release/licenses/managed-python-supplemental/zlib-ng-2.2.4-LICENSE.md.provenance.json @@ -0,0 +1,24 @@ +{ + "schema_version": 1, + "declared_path": "licenses/LICENSE.zlib-ng.txt", + "supplemental_reason": "declared_by_python_json_but_absent_from_upstream_full_archive", + "note": "Not a member of any python-build-standalone full archive. PYTHON.json declares this path but the upstream archive omits it; this is the license file of the source archive that python-build-standalone pins for the component.", + "python_build_standalone_release": "20260901", + "pin_authority": "https://github.com/astral-sh/python-build-standalone/blob/20260901/pythonbuild/downloads.py", + "targets": [ + "aarch64-apple-darwin", + "x86_64-unknown-linux-gnu" + ], + "component": "zlib-ng", + "version": "2.2.4", + "license": "Zlib", + "source_archive": { + "url": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.4.tar.gz", + "size": 2415819, + "sha256": "00bbd88709bc416cb96160ab61d3e1c8f76e106799af7328d0fe434dc7dd5004" + }, + "path_in_source_archive": "cpython-source-deps-zlib-ng-2.2.4/LICENSE.md", + "file": "zlib-ng-2.2.4-LICENSE.md", + "size": 867, + "sha256": "6c9f0d975b41afaa34d22f55bb8986ce69e5cb7ad327cb2b28820cd425edf5ee" +} diff --git a/scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE b/scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE new file mode 100644 index 000000000..75800288c --- /dev/null +++ b/scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE @@ -0,0 +1,30 @@ +BSD License + +For Zstandard software + +Copyright (c) Meta Platforms, Inc. and affiliates. All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the name Facebook, nor Meta, nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE.provenance.json b/scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE.provenance.json new file mode 100644 index 000000000..25e0fc1c6 --- /dev/null +++ b/scripts/release/licenses/managed-python-supplemental/zstd-1.5.7-LICENSE.provenance.json @@ -0,0 +1,25 @@ +{ + "schema_version": 1, + "declared_path": "licenses/LICENSE.zstd.txt", + "supplemental_reason": "declared_by_python_json_but_absent_from_upstream_full_archive", + "note": "Not a member of any python-build-standalone full archive. PYTHON.json declares this path but the upstream archive omits it; this is the license file of the source archive that python-build-standalone pins for the component.", + "python_build_standalone_release": "20260901", + "pin_authority": "https://github.com/astral-sh/python-build-standalone/blob/20260901/pythonbuild/downloads.py", + "targets": [ + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc", + "x86_64-unknown-linux-gnu" + ], + "component": "zstd", + "version": "1.5.7", + "license": "BSD-3-Clause", + "source_archive": { + "url": "https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz", + "size": 2440298, + "sha256": "f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e" + }, + "path_in_source_archive": "cpython-source-deps-zstd-1.5.7/LICENSE", + "file": "zstd-1.5.7-LICENSE", + "size": 1549, + "sha256": "7055266497633c9025b777c78eb7235af13922117480ed5c674677adc381c9d8" +} diff --git a/scripts/release/managed_python_supplemental.py b/scripts/release/managed_python_supplemental.py new file mode 100644 index 000000000..c479b125c --- /dev/null +++ b/scripts/release/managed_python_supplemental.py @@ -0,0 +1,178 @@ +"""Supplemental managed-CPython legal texts for declared files an upstream archive omits. + +A python-build-standalone ``PYTHON.json`` can declare a license file that the +published full archive does not contain. Such a path is satisfied only by a +checked-in text whose provenance record names exactly that path, the reviewed +build release, the target, and the pinned source archive the text came from, +and whose bytes match the recorded hash. Supplemental texts are never archive +members and are reported separately from them. +""" + +import hashlib +import json +from pathlib import Path +import re + +DIRECTORY = "scripts/release/licenses/managed-python-supplemental" +RECORD_SUFFIX = ".provenance.json" +REASON = "declared_by_python_json_but_absent_from_upstream_full_archive" +RECORD_KEYS = { + "schema_version", + "declared_path", + "supplemental_reason", + "note", + "python_build_standalone_release", + "pin_authority", + "targets", + "component", + "version", + "license", + "source_archive", + "path_in_source_archive", + "file", + "size", + "sha256", +} +DECLARED_PATH = re.compile(r"licenses/[A-Za-z0-9._-]+") +FILE_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9._-]*") +SHA256 = re.compile(r"[0-9a-f]{64}") + + +def declared_license_paths(metadata) -> set[str]: + """Return every ``license_path``/``license_paths`` reference in PYTHON.json.""" + paths = set() + + def visit(value) -> None: + if isinstance(value, dict): + for key, item in value.items(): + if key in ("license_path", "license_paths") and item is not None: + values = [item] if isinstance(item, str) else item + if not isinstance(values, list) or not all( + isinstance(path, str) and path for path in values + ): + raise ValueError(f"Invalid PYTHON.json license reference: {item!r}") + paths.update(values) + else: + visit(item) + elif isinstance(value, list): + for item in value: + visit(item) + + visit(metadata) + return paths + + +def _positive_int(value) -> bool: + return isinstance(value, int) and not isinstance(value, bool) and value > 0 + + +def _text(value) -> bool: + return isinstance(value, str) and bool(value.strip()) + + +def _validate(record, name: str, release: str, targets: set[str] | None) -> None: + source = record.get("source_archive") if isinstance(record, dict) else None + listed = record.get("targets") if isinstance(record, dict) else None + if ( + not isinstance(record, dict) + or set(record) != RECORD_KEYS + or record["schema_version"] != 1 + or record["supplemental_reason"] != REASON + or record["python_build_standalone_release"] != release + or record["pin_authority"] + != f"https://github.com/astral-sh/python-build-standalone/blob/{release}/pythonbuild/downloads.py" + or not isinstance(record["declared_path"], str) + or not DECLARED_PATH.fullmatch(record["declared_path"]) + or not isinstance(listed, list) + or not listed + or not all(_text(target) for target in listed) + or listed != sorted(set(listed)) + or (targets is not None and not set(listed) <= targets) + or not all( + _text(record[key]) + for key in ("note", "component", "version", "license", "path_in_source_archive") + ) + or not isinstance(source, dict) + or set(source) != {"url", "size", "sha256"} + or not isinstance(source["url"], str) + or not source["url"].startswith("https://") + or not _positive_int(source["size"]) + or not isinstance(source["sha256"], str) + or not SHA256.fullmatch(source["sha256"]) + or record["file"] != name.removesuffix(RECORD_SUFFIX) + or not FILE_NAME.fullmatch(record["file"]) + or not _positive_int(record["size"]) + or not isinstance(record["sha256"], str) + or not SHA256.fullmatch(record["sha256"]) + ): + raise ValueError(f"Invalid managed CPython supplemental license record: {name}") + + +def load_records(directory: Path, release: str, targets: set[str] | None = None) -> list[dict]: + """Load and byte-verify every supplemental record; refuse orphans and overlaps.""" + if not directory.exists(): + return [] + entries = sorted(directory.iterdir(), key=lambda path: path.name) + if any(not path.is_file() or path.is_symlink() for path in entries): + raise ValueError(f"Supplemental license directory must hold regular files: {directory}") + records = [] + for path in entries: + if not path.name.endswith(RECORD_SUFFIX): + continue + record = json.loads(path.read_bytes()) + _validate(record, path.name, release, targets) + text = directory / record["file"] + if not text.is_file() or text.is_symlink(): + raise ValueError(f"Missing managed CPython supplemental license: {record['file']}") + data = text.read_bytes() + if hashlib.sha256(data).hexdigest() != record["sha256"] or len(data) != record["size"]: + raise ValueError(f"Changed managed CPython supplemental license: {record['file']}") + records.append({**record, "record_path": path, "text_path": text}) + referenced = {record["file"] for record in records} + orphans = sorted( + path.name + for path in entries + if not path.name.endswith(RECORD_SUFFIX) and path.name not in referenced + ) + if orphans: + raise ValueError(f"Supplemental license texts without provenance: {orphans}") + claimed = set() + for record in records: + for target in record["targets"]: + key = (target, record["declared_path"]) + if key in claimed: + raise ValueError(f"Duplicate supplemental license for {target}: {key[1]}") + claimed.add(key) + return records + + +def select( + records: list[dict], + target: str, + declared: set[str], + archive_paths: set[str], + *, + strict: bool, +) -> list[dict]: + """Return the records that satisfy declared paths missing from the archive. + + Every declared path absent from ``archive_paths`` needs exactly one record + for ``target``. With ``strict``, a record for ``target`` whose path is not + a declared, missing path is refused as stale. + """ + missing = declared - archive_paths + applicable = [record for record in records if target in record["targets"]] + selected = [record for record in applicable if record["declared_path"] in missing] + unsatisfied = sorted(missing - {record["declared_path"] for record in selected}) + if unsatisfied: + raise ValueError( + "Declared managed CPython licenses absent from the full archive and without " + f"a verified supplemental text for {target}: {unsatisfied}" + ) + if strict: + stale = sorted(record["declared_path"] for record in applicable if record not in selected) + if stale: + raise ValueError( + f"Supplemental managed CPython licenses not needed for {target}: {stale}" + ) + return sorted(selected, key=lambda record: record["declared_path"]) diff --git a/scripts/release/test_managed_python_supplemental.py b/scripts/release/test_managed_python_supplemental.py new file mode 100644 index 000000000..3816b2f54 --- /dev/null +++ b/scripts/release/test_managed_python_supplemental.py @@ -0,0 +1,163 @@ +"""Supplemental managed-CPython license texts satisfy only verified, declared gaps.""" + +import hashlib +import importlib.util +import json +from pathlib import Path +import tempfile +import unittest + +spec = importlib.util.spec_from_file_location( + "managed_python_supplemental", Path(__file__).with_name("managed_python_supplemental.py") +) +supplemental = importlib.util.module_from_spec(spec) +spec.loader.exec_module(supplemental) + +ROOT = Path(__file__).resolve().parents[2] +RELEASE = "20260901" +LINUX = "x86_64-unknown-linux-gnu" +WINDOWS = "x86_64-pc-windows-msvc" +MACOS = "aarch64-apple-darwin" +TARGETS = {LINUX, WINDOWS, MACOS} +CPYTHON = "licenses/LICENSE.cpython.txt" +ZSTD = "licenses/LICENSE.zstd.txt" +ZLIB_NG = "licenses/LICENSE.zlib-ng.txt" + + +def write_record( + directory, declared_path, content, *, name=None, targets=(LINUX,), signed=None, overrides=None +): + directory.mkdir(parents=True, exist_ok=True) + if name is None: + name = declared_path.removeprefix("licenses/LICENSE.").removesuffix(".txt") + "-LICENSE" + (directory / name).write_bytes(content) + signed = content if signed is None else signed + record = { + "schema_version": 1, + "declared_path": declared_path, + "supplemental_reason": supplemental.REASON, + "note": "Fixture text supplied outside the archive.", + "python_build_standalone_release": RELEASE, + "pin_authority": ( + f"https://github.com/astral-sh/python-build-standalone/blob/{RELEASE}" + "/pythonbuild/downloads.py" + ), + "targets": sorted(targets), + "component": name, + "version": "1.0", + "license": "BSD-3-Clause", + "source_archive": { + "url": "https://example.invalid/source.tar.gz", + "size": 10, + "sha256": "b" * 64, + }, + "path_in_source_archive": "source/LICENSE", + "file": name, + "size": len(signed), + "sha256": hashlib.sha256(signed).hexdigest(), + **(overrides or {}), + } + (directory / (name + supplemental.RECORD_SUFFIX)).write_text(json.dumps(record)) + return record + + +class SupplementalLicenseTests(unittest.TestCase): + def load(self, directory): + return supplemental.load_records(directory, RELEASE, TARGETS) + + def test_verified_text_satisfies_its_absent_declared_path(self): + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + write_record(directory, ZSTD, b"zstd terms\n", targets=(LINUX, WINDOWS)) + records = self.load(directory) + for target in (LINUX, WINDOWS): + selected = supplemental.select( + records, target, {CPYTHON, ZSTD}, {CPYTHON}, strict=True + ) + self.assertEqual([item["declared_path"] for item in selected], [ZSTD]) + self.assertEqual(selected[0]["text_path"].read_bytes(), b"zstd terms\n") + + def test_mismatched_hash_or_size_is_refused(self): + for changes in ({"signed": b"original\n"}, {"overrides": {"size": 99}}): + with self.subTest(changes=changes), tempfile.TemporaryDirectory() as tmp: + write_record(Path(tmp), ZSTD, b"altered\n", **changes) + with self.assertRaisesRegex(ValueError, "Changed managed CPython supplemental"): + self.load(Path(tmp)) + + def test_unsupplied_or_mismatched_absent_path_still_fails(self): + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + write_record(directory, ZSTD, b"zstd terms\n", targets=(LINUX,)) + records = self.load(directory) + for target, declared in ( + (LINUX, {CPYTHON, ZSTD, ZLIB_NG}), + (WINDOWS, {CPYTHON, ZSTD}), + (LINUX, {CPYTHON, "licenses/LICENSE.other.txt"}), + ): + for strict in (False, True): + with self.subTest(target=target, declared=declared, strict=strict): + with self.assertRaisesRegex(ValueError, "absent from the full archive"): + supplemental.select(records, target, declared, {CPYTHON}, strict=strict) + + def test_strict_selection_refuses_undeclared_or_archived_supplemental_paths(self): + with tempfile.TemporaryDirectory() as tmp: + directory = Path(tmp) + write_record(directory, ZSTD, b"zstd terms\n") + records = self.load(directory) + for declared, archived in (({CPYTHON}, {CPYTHON}), ({CPYTHON, ZSTD}, {CPYTHON, ZSTD})): + with self.assertRaisesRegex(ValueError, "not needed"): + supplemental.select(records, LINUX, declared, archived, strict=True) + self.assertEqual( + supplemental.select(records, LINUX, declared, archived, strict=False), [] + ) + + def test_records_bind_release_target_shape_and_reject_orphans_or_overlaps(self): + invalid = ( + {"python_build_standalone_release": "20261001"}, + {"targets": ["riscv64-unknown-linux-gnu"]}, + {"targets": [LINUX, WINDOWS]}, + {"declared_path": "../LICENSE.zstd.txt"}, + {"supplemental_reason": "guess"}, + {"source_archive": {"url": "http://example.invalid/a", "size": 1, "sha256": "b" * 64}}, + {"extra": True}, + ) + for changes in invalid: + with self.subTest(changes=changes), tempfile.TemporaryDirectory() as tmp: + write_record(Path(tmp), ZSTD, b"terms\n", overrides=changes) + with self.assertRaisesRegex(ValueError, "Invalid managed CPython supplemental"): + self.load(Path(tmp)) + with tempfile.TemporaryDirectory() as tmp: + write_record(Path(tmp), ZSTD, b"terms\n") + (Path(tmp) / "unexplained-LICENSE").write_bytes(b"terms\n") + with self.assertRaisesRegex(ValueError, "without provenance"): + self.load(Path(tmp)) + with tempfile.TemporaryDirectory() as tmp: + write_record(Path(tmp), ZSTD, b"terms\n", name="zstd-a-LICENSE") + write_record(Path(tmp), ZSTD, b"terms\n", name="zstd-b-LICENSE") + with self.assertRaisesRegex(ValueError, "Duplicate supplemental license"): + self.load(Path(tmp)) + + def test_checked_in_records_cover_the_retained_archive_gaps(self): + records = self.load(ROOT / supplemental.DIRECTORY) + evidence = ( + ROOT / "docs/plans/torch-cross-platform-runtime-management/reports" + "/managed-python-license-collection" + ) + for folder, target, absent in ( + ("linux-x86_64-cpython-3.14.7", LINUX, [ZLIB_NG, ZSTD]), + ("macos-arm64-cpython-3.14.7", MACOS, [ZLIB_NG, ZSTD]), + ("windows-x86_64-cpython-3.14.7", WINDOWS, [ZSTD]), + ): + with self.subTest(target=target): + retained = evidence / folder / "managed-python-licenses" + manifest = json.loads((retained / "full-archive-manifest.json").read_bytes()) + python = json.loads((retained / "full-archive/PYTHON.json").read_bytes()) + declared = supplemental.declared_license_paths(python) + archived = {item["path"] for item in manifest["files"]} + self.assertEqual(sorted(declared - archived), absent) + selected = supplemental.select(records, target, declared, archived, strict=True) + self.assertEqual([item["declared_path"] for item in selected], absent) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/release/torch-managed-python-acceptance.py b/scripts/release/torch-managed-python-acceptance.py index 1160d0d8c..af12c700b 100644 --- a/scripts/release/torch-managed-python-acceptance.py +++ b/scripts/release/torch-managed-python-acceptance.py @@ -9,6 +9,7 @@ from contextlib import contextmanager import errno import hashlib +import importlib.util import io import json import ntpath @@ -43,6 +44,7 @@ "aarch64-apple-darwin": "pgo+lto", "x86_64-pc-windows-msvc": "pgo", } +SUPPLEMENTAL_LICENSE_DIR = Path(__file__).resolve().parent / "licenses/managed-python-supplemental" MAX_RELEASE_JSON_BYTES = 8 * 1024 * 1024 MAX_FULL_ARCHIVE_BYTES = 200 * 1024 * 1024 MAX_LICENSE_BYTES = 16 * 1024 * 1024 @@ -64,6 +66,18 @@ def redirect_request(self, request, fp, code, msg, headers, newurl): DOWNLOAD_HTTP = urllib.request.build_opener(HTTPSOnlyRedirectHandler()) +def load_supplemental_module(): + """Load the shared supplemental-license rules without relying on sys.path.""" + path = Path(__file__).resolve().with_name("managed_python_supplemental.py") + spec = importlib.util.spec_from_file_location("managed_python_supplemental", path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +SUPPLEMENTAL = load_supplemental_module() + + class GitHubApiRedirectHandler(HTTPSOnlyRedirectHandler): def redirect_request(self, request, fp, code, msg, headers, newurl): target = urlsplit(newurl) @@ -1570,6 +1584,7 @@ def archive( pax_bytes: int = 0, zstd: bool = False, metadata_overrides: dict | None = None, + declared_absent: tuple[str, ...] = (), ) -> None: metadata = { "version": "8", @@ -1580,6 +1595,11 @@ def archive( "python_implementation_name": "cpython", } metadata.update(metadata_overrides or {}) + if declared_absent: + # Mirrors PBS 20260901: PYTHON.json names files the archive omits. + metadata["build_info"] = { + "extensions": {"_zstd": [{"license_paths": list(declared_absent)}]} + } entries = [ ("python/PYTHON.json", json.dumps(metadata).encode()), ("python/licenses/LICENSE.cpython.txt", b"CPython license"), @@ -1931,6 +1951,152 @@ def fake_download(_asset: dict, destination: Path) -> None: export_full_archive_licenses(root, Path(temporary) / "evidence") self.assertFalse((root / "tmp/selected-managed-python-full.tar.zst").exists()) + @staticmethod + def supplemental( + directory: Path, + declared_path: str, + content: bytes, + *, + targets: tuple[str, ...] = ("x86_64-unknown-linux-gnu",), + recorded: bytes | None = None, + ) -> dict: + directory.mkdir(parents=True, exist_ok=True) + component = declared_path.removeprefix("licenses/LICENSE.").removesuffix(".txt") + name = f"{component}-1.0-LICENSE" + (directory / name).write_bytes(content) + signed = content if recorded is None else recorded + record = { + "schema_version": 1, + "declared_path": declared_path, + "supplemental_reason": SUPPLEMENTAL.REASON, + "note": "Fixture text supplied outside the archive.", + "python_build_standalone_release": REVIEWED_PBS_RELEASE, + "pin_authority": ( + "https://github.com/astral-sh/python-build-standalone/blob/" + f"{REVIEWED_PBS_RELEASE}/pythonbuild/downloads.py" + ), + "targets": sorted(targets), + "component": component, + "version": "1.0", + "license": "BSD-3-Clause", + "source_archive": { + "url": f"https://example.invalid/{component}-1.0.tar.gz", + "size": 10, + "sha256": "b" * 64, + }, + "path_in_source_archive": f"{component}-1.0/LICENSE", + "file": name, + "size": len(signed), + "sha256": hashlib.sha256(signed).hexdigest(), + } + (directory / f"{name}{SUPPLEMENTAL.RECORD_SUFFIX}").write_text( + json.dumps(record), encoding="utf-8" + ) + return record + + def test_absent_declared_licenses_use_verified_supplemental_texts(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + base = Path(temporary) + archive = base / "archive.tar" + absent = ("licenses/LICENSE.zlib-ng.txt", "licenses/LICENSE.zstd.txt") + self.archive(archive, declared_absent=absent) + supplemental = base / "supplemental" + for path in absent: + self.supplemental(supplemental, path, f"text for {path}\n".encode()) + extract_full_archive_licenses( + archive, + base / "evidence", + self.collector_asset(), + mode="r|", + supplemental_dir=supplemental, + ) + evidence = base / "evidence/managed-python-licenses" + manifest = json.loads((evidence / "full-archive-manifest.json").read_text()) + self.assertEqual( + [item["path"] for item in manifest["files"]], ["licenses/LICENSE.cpython.txt"] + ) + self.assertEqual( + manifest["declared_license_sources"], + {"archive": ["licenses/LICENSE.cpython.txt"], "supplemental": list(absent)}, + ) + self.assertEqual( + [item["path"] for item in manifest["supplemental_files"]], list(absent) + ) + for item in manifest["supplemental_files"]: + self.assertEqual(item["supplemental_reason"], SUPPLEMENTAL.REASON) + self.assertTrue(item["provenance"].startswith(SUPPLEMENTAL.DIRECTORY + "/")) + self.assertEqual( + hashlib.sha256((evidence / item["evidence_path"]).read_bytes()).hexdigest(), + item["sha256"], + ) + self.assertEqual( + sorted(path.name for path in (evidence / "full-archive/licenses").iterdir()), + ["LICENSE.cpython.txt"], + ) + + def test_supplemental_text_with_mismatched_hash_is_refused(self) -> None: + with tempfile.TemporaryDirectory() as temporary: + base = Path(temporary) + archive = base / "archive.tar" + self.archive(archive, declared_absent=("licenses/LICENSE.zstd.txt",)) + supplemental = base / "supplemental" + self.supplemental( + supplemental, "licenses/LICENSE.zstd.txt", b"altered\n", recorded=b"original\n" + ) + with self.assertRaisesRegex(RuntimeError, "Changed managed CPython supplemental"): + extract_full_archive_licenses( + archive, + base / "evidence", + self.collector_asset(), + mode="r|", + supplemental_dir=supplemental, + ) + self.assertFalse((base / "evidence").exists()) + + def test_absent_declared_license_without_matching_supplemental_still_fails(self) -> None: + cases = { + "no supplemental texts": (), + "other declared path only": ( + ("licenses/LICENSE.zlib-ng.txt", ("x86_64-unknown-linux-gnu",)), + ), + "other target only": (("licenses/LICENSE.zstd.txt", ("x86_64-pc-windows-msvc",)),), + } + for case, records in cases.items(): + with self.subTest(case=case), tempfile.TemporaryDirectory() as temporary: + base = Path(temporary) + archive = base / "archive.tar" + self.archive(archive, declared_absent=("licenses/LICENSE.zstd.txt",)) + supplemental = base / "supplemental" + for path, targets in records: + self.supplemental(supplemental, path, b"text\n", targets=targets) + with self.assertRaisesRegex(RuntimeError, "LICENSE.zstd.txt"): + extract_full_archive_licenses( + archive, + base / "evidence", + self.collector_asset(), + mode="r|", + supplemental_dir=supplemental, + ) + self.assertFalse((base / "evidence").exists()) + + def test_checked_in_supplemental_texts_cover_reviewed_targets(self) -> None: + records = SUPPLEMENTAL.load_records( + SUPPLEMENTAL_LICENSE_DIR, REVIEWED_PBS_RELEASE, set(FULL_FLAVORS) + ) + cpython = "licenses/LICENSE.cpython.txt" + zstd = "licenses/LICENSE.zstd.txt" + zlib_ng = "licenses/LICENSE.zlib-ng.txt" + expected = { + "x86_64-pc-windows-msvc": [zstd], + "x86_64-unknown-linux-gnu": [zlib_ng, zstd], + "aarch64-apple-darwin": [zlib_ng, zstd], + } + for target, absent in expected.items(): + selected = SUPPLEMENTAL.select( + records, target, {cpython, *absent}, {cpython}, strict=True + ) + self.assertEqual([record["declared_path"] for record in selected], absent) + INSTALL_REPORTS = ( "runtime.json", @@ -2277,8 +2443,41 @@ def open_bounded_tar(archive: Path, mode: str): yield tar +def supplemental_licenses( + target: str, declared: set[str], members: set[str], directory: Path +) -> list[dict]: + """Verified supplemental texts for declared paths that the archive omits. + + Any declared path that is missing and lacks a byte-verified record for this + exact path, target and reviewed release still fails acceptance. + """ + missing = declared - members + if not missing: + return [] + relative = {path.removeprefix("python/") for path in members if path.startswith("python/")} + try: + records = SUPPLEMENTAL.load_records(directory, REVIEWED_PBS_RELEASE, set(FULL_FLAVORS)) + return SUPPLEMENTAL.select( + records, + target, + {path.removeprefix("python/") for path in declared}, + relative, + strict=False, + ) + except (OSError, ValueError) as error: + detail = {"missing": sorted(missing), "supplemental": str(error)} + raise RuntimeError( + f"declared full-archive license members failed: {json.dumps(detail)[:1200]}" + ) from error + + def extract_full_archive_licenses( - archive: Path, output: Path, asset: dict, *, mode: str = "r|zst" + archive: Path, + output: Path, + asset: dict, + *, + mode: str = "r|zst", + supplemental_dir: Path | None = None, ) -> None: members = set() regular_sizes = {} @@ -2333,10 +2532,19 @@ def extract_full_archive_licenses( ) validate_full_archive_metadata(metadata, asset) declared = declared_license_paths(metadata) - wanted = candidates | declared require(bool(candidates), "full-archive license directory", asset["name"]) + supplemental = supplemental_licenses( + asset["target"], + declared, + members, + SUPPLEMENTAL_LICENSE_DIR if supplemental_dir is None else supplemental_dir, + ) + archive_declared = declared - {"python/" + record["declared_path"] for record in supplemental} + wanted = candidates | archive_declared require( - declared <= members, "declared full-archive license members", sorted(declared - members) + archive_declared <= members, + "declared full-archive license members", + sorted(archive_declared - members), ) require(len(wanted) <= MAX_ARCHIVE_MEMBERS, "bounded selected licenses", len(wanted)) selected_bytes = 0 @@ -2384,6 +2592,33 @@ def extract_full_archive_licenses( ) found.add(name) require(found == wanted, "all selected full-archive licenses", sorted(wanted - found)) + supplemental_files = [] + for record in supplemental: + # Kept apart from full-archive/: these texts are not archive members. + content = record["text_path"].read_bytes() + require( + hashlib.sha256(content).hexdigest() == record["sha256"], + "verified supplemental license text", + record["file"], + ) + copied = output / "managed-python-licenses" / "supplemental" / record["file"] + copied.parent.mkdir(parents=True, exist_ok=True) + copied.write_bytes(content) + supplemental_files.append( + { + "path": record["declared_path"], + "sha256": record["sha256"], + "size": record["size"], + "evidence_path": f"supplemental/{record['file']}", + "provenance": f"{SUPPLEMENTAL.DIRECTORY}/{record['record_path'].name}", + "supplemental_reason": record["supplemental_reason"], + "component": record["component"], + "version": record["version"], + "license": record["license"], + "source_archive": record["source_archive"], + "path_in_source_archive": record["path_in_source_archive"], + } + ) (destination / "PYTHON.json").write_bytes(metadata_bytes) (output / "managed-python-licenses" / "full-archive-manifest.json").write_text( json.dumps( @@ -2404,6 +2639,11 @@ def extract_full_archive_licenses( "size": len(metadata_bytes), }, "files": sorted(files, key=lambda item: item["path"]), + "supplemental_files": supplemental_files, + "declared_license_sources": { + "archive": sorted(path.removeprefix("python/") for path in archive_declared), + "supplemental": [item["path"] for item in supplemental_files], + }, }, indent=2, sort_keys=True, From 184c64338c93565658e165656e9fb7c640021f5f Mon Sep 17 00:00:00 2001 From: MrScripty Date: Wed, 4 Nov 2026 01:50:17 -0700 Subject: [PATCH 2/9] fix(versions): install the chosen llama.cpp CUDA build and surface early failures - Pumas lists llama.cpp CUDA 12/13 builds for Linux, but the installer could not select them: its Linux GPU preference had no CUDA entry and its CPU matcher excludes CUDA names. A runtime variant (`b12099+cuda-12`) carries only the archive the user chose, so the installer now installs that one. - A failure before an install started tracking progress (release lookup, asset selection) was never logged or shown: progress stayed empty. It is now logged and visible in installation progress. - Each installer starts tracking before its first effect (llama.cpp now starts it before preparing its workspace). A failure before that point created nothing, so it no longer fails shutdown; a failed shutdown retains library ownership, which had blocked reopening the library. Co-Authored-By: Claude Opus 5.5 --- .../src/version_manager/installer.rs | 47 ++++++-- .../src/version_manager/mod.rs | 109 +++++++++++++++++- 2 files changed, 145 insertions(+), 11 deletions(-) diff --git a/rust/crates/pumas-app-manager/src/version_manager/installer.rs b/rust/crates/pumas-app-manager/src/version_manager/installer.rs index 4f95d08c4..adc94cff4 100644 --- a/rust/crates/pumas-app-manager/src/version_manager/installer.rs +++ b/rust/crates/pumas-app-manager/src/version_manager/installer.rs @@ -1967,6 +1967,18 @@ impl VersionInstaller { Utc::now().format("%Y%m%d-%H%M%S") )); + // Tracking starts before the first effect: a failure before this point + // created nothing, and the version manager treats it as an outcome. + { + let mut tracker = self.progress_tracker.write().await; + tracker.start_installation( + tag, + total_size, + None, + Some(log_path.to_string_lossy().as_ref()), + ); + } + let versions = self.versions_dir(); let workspace_versions = versions.clone(); let workspace_tag = tag.to_owned(); @@ -1979,16 +1991,6 @@ impl VersionInstaller { }) .await?; - { - let mut tracker = self.progress_tracker.write().await; - tracker.start_installation( - tag, - total_size, - None, - Some(log_path.to_string_lossy().as_ref()), - ); - } - let demand = AcquisitionDemand { consumer: consumer.owner().to_owned(), operation: format!( @@ -2729,6 +2731,14 @@ impl VersionInstaller { "windows" => "win", _ => os, }; + // A runtime variant (`b1234+cuda-12`) carries only the archive the user + // chose; install exactly that one, whatever its flavor. + let mut platform_archives = assets.iter().filter(|asset| { + Self::is_llama_cpp_platform_archive(&asset.name.to_ascii_lowercase(), platform, arch) + }); + if let (Some(only), None) = (platform_archives.next(), platform_archives.next()) { + return Ok(only); + } if prefer_gpu_asset { if let Some(asset) = Self::find_llama_cpp_gpu_asset(assets, os, platform, arch) { return Ok(asset); @@ -4646,6 +4656,23 @@ mod tests { assert_eq!(selected.name, "llama-b9082-bin-ubuntu-vulkan-x64.tar.gz"); } + #[test] + fn llama_cpp_asset_selection_installs_the_chosen_linux_cuda_variant() { + for name in [ + "llama-b12099-bin-ubuntu-cuda-12.8-x64.tar.gz", + "llama-b12099-bin-ubuntu-cuda-13.4-x64.tar.gz", + ] { + let assets = vec![github_asset(name)]; + for prefer_gpu in [true, false] { + let selected = VersionInstaller::select_llama_cpp_asset_for_platform( + &assets, "linux", "x64", prefer_gpu, + ) + .unwrap(); + assert_eq!(selected.name, name); + } + } + } + fn github_asset(name: &str) -> GitHubAsset { GitHubAsset { name: name.to_string(), diff --git a/rust/crates/pumas-app-manager/src/version_manager/mod.rs b/rust/crates/pumas-app-manager/src/version_manager/mod.rs index bea91e2a9..a7b290f95 100644 --- a/rust/crates/pumas-app-manager/src/version_manager/mod.rs +++ b/rust/crates/pumas-app-manager/src/version_manager/mod.rs @@ -1276,8 +1276,19 @@ impl VersionManager { } // Send final status + // Each installer starts tracking before its first effect. A failure + // before that (release lookup, asset selection) created nothing to + // settle: it is an outcome reported to the caller, not a shutdown + // failure that must retain library ownership. + let mut failed_before_effects = false; if let Err(error) = &result { + warn!("Installation of {} failed: {}", tag, error); let mut tracker = progress_tracker.write().await; + let tracked_tag = tracker.get_current_state().and_then(|state| state.tag); + if tracked_tag.as_deref() != Some(tag.as_str()) { + failed_before_effects = true; + tracker.start_installation(&tag, None, None, None); + } tracker.set_error(&error.to_string()); tracker.complete_installation(false); } @@ -1287,7 +1298,11 @@ impl VersionManager { .get_current_state() .as_ref() .and_then(CompletedProgressIdentity::capture); - let terminal = result.as_ref().map(|_| ()).map_err(ToString::to_string); + let terminal = if failed_before_effects { + Ok(()) + } else { + result.as_ref().map(|_| ()).map_err(ToString::to_string) + }; tokio::select! { _ = tx.send(match result { Ok(_) => ProgressUpdate::Completed { success: true }, @@ -5256,6 +5271,98 @@ mod tests { } } + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] + #[tokio::test] + async fn install_failure_before_progress_tracking_is_still_reported() { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let root = TempDir::new().unwrap(); + pumas_library::network::ReleasesCache::new( + root.path().join("launcher-data/cache"), + Duration::from_secs(3600), + ) + .set_disk( + AppId::LlamaCpp.github_repo(), + &[pumas_library::network::GitHubRelease { + tag_name: "b1234".into(), + name: "Native fixture".into(), + published_at: "2026-09-29T00:00:00Z".into(), + body: None, + tarball_url: None, + zipball_url: None, + prerelease: false, + assets: vec![pumas_library::network::GitHubAsset { + name: "llama-b1234-bin-ubuntu-x64.tar.gz".into(), + size: 4, + download_url: "http://127.0.0.1:9/archive".into(), + content_type: Some("application/gzip".into()), + }], + html_url: "https://github.com/ggml-org/llama.cpp/releases/tag/b1234".into(), + total_size: Some(4), + archive_size: Some(4), + dependencies_size: None, + }], + ) + .unwrap(); + // The publisher lookup fails before the installer starts tracking. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}", listener.local_addr().unwrap()); + let server = tokio::spawn(async move { + while let Ok((mut stream, _)) = listener.accept().await { + let mut request = [0; 4096]; + let _ = stream.read(&mut request).await; + let _ = stream + .write_all(b"HTTP/1.1 404 Not Found\r\ncontent-length: 2\r\nconnection: close\r\n\r\n{}") + .await; + } + }); + let api = pumas_library::PumasApi::builder(root.path()) + .with_hf_client(false) + .with_process_manager(false) + .build() + .await + .unwrap(); + let mut manager = VersionManager::new_with_acquisition( + root.path(), + AppId::LlamaCpp, + api.acquisition().clone(), + ) + .await + .unwrap(); + manager.github_client = Arc::new( + GitHubClient::with_loopback_api( + manager.cache_dir(), + Duration::from_secs(3600), + base_url, + ) + .unwrap(), + ); + let mut updates = manager.install_version("b1234+cpu").await.unwrap(); + let mut failure = None; + while let Some(update) = tokio::time::timeout(Duration::from_secs(30), updates.recv()) + .await + .unwrap() + { + if let ProgressUpdate::Error { message } = update { + failure = Some(message); + break; + } + } + let failure = failure.expect("the caller receives the failure"); + let progress = manager + .get_installation_progress() + .await + .expect("an early failure is visible in progress"); + assert_eq!(progress.tag.as_deref(), Some("b1234+cpu")); + assert_eq!(progress.success, Some(false)); + assert_eq!(progress.error.as_deref(), Some(failure.as_str())); + // Nothing was created, so the failure does not fail shutdown (which + // would retain library ownership and block the next open). + manager.shutdown_installations().await.unwrap(); + server.abort(); + api.shutdown_intent().await.unwrap(); + api.shutdown_acquisition().await.unwrap(); + } + #[cfg(all(target_os = "linux", target_arch = "x86_64"))] #[tokio::test] async fn native_shutdown_cancels_stalled_transfer_and_withdraws_owned_workspace() { From 3998a410f9dac2ae6443267038030964ce892c85 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Wed, 4 Nov 2026 01:50:17 -0700 Subject: [PATCH 3/9] fix(serving): serve long library IDs and projector models on dedicated profiles - Served-model aliases were capped at 128 characters while llama.cpp serving defaults the alias to the library ID, and downloaded IDs often run past that (EmbeddingGemma 2's is 147), so those models could not be served. The bound is now the 256 bytes the model-operations contract accepts for `model`, one shared constant. - A model with a projector launches with `--alias `, but the dedicated readiness check required llama.cpp to report the file path, so every projector model (EmbeddingGemma 2 image embeddings, vision models) failed to become ready on a dedicated profile. Readiness accepts either identity the launch owns. Co-Authored-By: Claude Opus 5.5 --- .../pumas-core/src/serving/gateway_alias.rs | 9 ++-- rust/crates/pumas-core/src/serving/mod.rs | 25 ++++++++++- .../handlers/model_operations/projection.rs | 2 +- .../src/handlers/serving_llama_cpp.rs | 12 ++--- rust/crates/pumas-rpc/src/provider_clients.rs | 44 ++++++++++++++----- 5 files changed, 69 insertions(+), 23 deletions(-) diff --git a/rust/crates/pumas-core/src/serving/gateway_alias.rs b/rust/crates/pumas-core/src/serving/gateway_alias.rs index 98e3e70c2..a4af2d15e 100644 --- a/rust/crates/pumas-core/src/serving/gateway_alias.rs +++ b/rust/crates/pumas-core/src/serving/gateway_alias.rs @@ -5,7 +5,10 @@ use crate::models::{ use super::{served_model_reserves_gateway_alias, ServingValidationContext}; -const MAX_GATEWAY_ALIAS_LEN: usize = 128; +/// The longest served-model alias, in bytes: the bound the model-operations +/// contract accepts for `model`, so every servable alias stays addressable. +/// Library IDs are the default llama.cpp alias and can exceed 128 bytes. +pub const MAX_GATEWAY_ALIAS_BYTES: usize = 256; pub(super) fn validate_gateway_alias_contract( model_id: &str, @@ -22,9 +25,9 @@ pub(super) fn validate_gateway_alias_contract( } let mut message = None; - if alias.len() > MAX_GATEWAY_ALIAS_LEN { + if alias.len() > MAX_GATEWAY_ALIAS_BYTES { message = Some(format!( - "model_alias must be {MAX_GATEWAY_ALIAS_LEN} characters or fewer" + "model_alias must be {MAX_GATEWAY_ALIAS_BYTES} bytes or fewer; choose a shorter alias for this model" )); } else if alias.starts_with('/') || alias.ends_with('/') || alias.contains("//") { message = Some("model_alias cannot begin or end with '/', or contain '//'".to_string()); diff --git a/rust/crates/pumas-core/src/serving/mod.rs b/rust/crates/pumas-core/src/serving/mod.rs index 3451de415..399913c70 100644 --- a/rust/crates/pumas-core/src/serving/mod.rs +++ b/rust/crates/pumas-core/src/serving/mod.rs @@ -17,7 +17,7 @@ use std::sync::{Arc, Mutex}; use tokio::sync::broadcast; -pub use gateway_alias::effective_gateway_model_alias; +pub use gateway_alias::{effective_gateway_model_alias, MAX_GATEWAY_ALIAS_BYTES}; use gateway_alias::{validate_gateway_alias_contract, validate_gateway_alias_is_unique}; use placement::validate_provider_placement; @@ -1321,6 +1321,29 @@ mod tests { && error.message.contains("model_alias"))); } + #[test] + fn validation_accepts_a_long_library_id_alias_up_to_the_operations_bound() { + // A real downloaded EmbeddingGemma 2 ID (147 bytes), which llama.cpp + // serving uses as its default alias. + let library_id = "embedding/gemma-embedding2/unsloth--embeddinggemma-2-gguf__files_555e3f992f30__555e3f992f30__revision_031f0d4b35536f69ab3509d4893c923264fcf253"; + for (alias, accepted) in [ + (library_id.to_string(), true), + ("a".repeat(MAX_GATEWAY_ALIAS_BYTES), true), + ("a".repeat(MAX_GATEWAY_ALIAS_BYTES + 1), false), + ] { + let mut request = request(); + request.config.provider = RuntimeProviderId::LlamaCpp; + request.config.profile_id = RuntimeProfileId::parse("llama-gpu").unwrap(); + request.config.model_alias = Some(alias); + let response = validate(&request, &valid_context()); + let alias_error = response + .errors + .iter() + .any(|error| error.message.contains("bytes or fewer")); + assert_eq!(alias_error, !accepted); + } + } + #[test] fn validation_allows_revalidating_same_served_instance_alias() { let mut request = request(); diff --git a/rust/crates/pumas-rpc/src/handlers/model_operations/projection.rs b/rust/crates/pumas-rpc/src/handlers/model_operations/projection.rs index c1595de05..9553d7022 100644 --- a/rust/crates/pumas-rpc/src/handlers/model_operations/projection.rs +++ b/rust/crates/pumas-rpc/src/handlers/model_operations/projection.rs @@ -304,7 +304,7 @@ fn options_kind(options: &OperationOptions) -> &'static str { } pub fn valid_model(value: &str) -> bool { !value.trim().is_empty() - && value.len() <= 256 + && value.len() <= pumas_library::serving::MAX_GATEWAY_ALIAS_BYTES && value.trim() == value && !value.chars().any(char::is_control) } diff --git a/rust/crates/pumas-rpc/src/handlers/serving_llama_cpp.rs b/rust/crates/pumas-rpc/src/handlers/serving_llama_cpp.rs index a55b9ccda..7406771b3 100644 --- a/rust/crates/pumas-rpc/src/handlers/serving_llama_cpp.rs +++ b/rust/crates/pumas-rpc/src/handlers/serving_llama_cpp.rs @@ -157,7 +157,7 @@ pub(super) async fn serve_llama_cpp_model( wait_for_dedicated_readiness( &state.llama_cpp_router_client, owned.endpoint_url.as_str(), - model_path, + &[model_path, request.model_id.as_str()], Duration::from_secs(60), || { state @@ -240,7 +240,7 @@ pub(super) async fn serve_llama_cpp_model( async fn wait_for_dedicated_readiness( client: &crate::provider_clients::LlamaCppRouterClient, endpoint: &str, - model_path: &str, + identities: &[&str], budget: Duration, mut ownership_matches: impl FnMut() -> Result, mut owns_listener: impl FnMut() -> Result, @@ -256,7 +256,7 @@ async fn wait_for_dedicated_readiness( continue; } let ready = client - .dedicated_model_ready(endpoint, model_path, deadline) + .dedicated_model_ready(endpoint, identities, deadline) .await?; // HTTP success is valid only while the same child generation remains owned. if !ownership_matches()? { @@ -381,7 +381,7 @@ mod tests { let result = wait_for_dedicated_readiness( &client, &endpoint, - "/test.gguf", + &["/test.gguf"], if slow_health { Duration::from_millis(50) } else { @@ -450,7 +450,7 @@ mod tests { let result = wait_for_dedicated_readiness( &client, "http://127.0.0.1:1", - "/test.gguf", + &["/test.gguf"], Duration::from_secs(1), || Ok(false), || Ok(true), @@ -468,7 +468,7 @@ mod tests { let check = wait_for_dedicated_readiness( &client, &endpoint, - "/test.gguf", + &["/test.gguf"], Duration::from_millis(30), || Ok(true), || listener_result.clone(), diff --git a/rust/crates/pumas-rpc/src/provider_clients.rs b/rust/crates/pumas-rpc/src/provider_clients.rs index 67e0f7a21..5ef0e4966 100644 --- a/rust/crates/pumas-rpc/src/provider_clients.rs +++ b/rust/crates/pumas-rpc/src/provider_clients.rs @@ -127,11 +127,13 @@ impl LlamaCppRouterClient { } /// Prove readiness of one dedicated server against its owned launch model. - /// The caller owns the overall startup deadline and process identity checks. + /// `identities` are the IDs that launch can report: the model file path, + /// or the library model ID when the launch set `--alias` (models with a + /// projector). The caller owns the startup deadline and process checks. pub(crate) async fn dedicated_model_ready( &self, endpoint: &str, - model_path: &str, + identities: &[&str], deadline: tokio::time::Instant, ) -> Result { let health = self @@ -175,7 +177,7 @@ impl LlamaCppRouterClient { .json() .await .map_err(|_| "llama.cpp model endpoint returned invalid JSON")?; - dedicated_model_identity_ready(&models, model_path) + dedicated_model_identity_ready(&models, identities) } pub async fn load_model( @@ -357,14 +359,17 @@ fn router_catalog_status(status: &serde_json::Value, require_loaded: bool) -> Re fn dedicated_model_identity_ready( models: &serde_json::Value, - model_path: &str, + identities: &[&str], ) -> Result { let rows = models .get("data") .and_then(serde_json::Value::as_array) .ok_or("llama.cpp model endpoint omitted model identity")?; - if rows.len() != 1 || rows[0].get("id").and_then(serde_json::Value::as_str) != Some(model_path) - { + let reported = rows + .first() + .and_then(|row| row.get("id")) + .and_then(serde_json::Value::as_str); + if rows.len() != 1 || !reported.is_some_and(|id| identities.contains(&id)) { return Err("llama.cpp model endpoint does not match the owned launch model".into()); } match rows[0].get("meta") { @@ -480,22 +485,37 @@ mod tests { serde_json::json!({"data": [{"id": expected, "meta": false}]}), serde_json::json!({"data": [{"id": expected, "meta": {}}, {"id": expected, "meta": {}}]}), ] { - assert!(dedicated_model_identity_ready(&payload, expected).is_err()); + assert!(dedicated_model_identity_ready(&payload, &[expected]).is_err()); } assert_eq!( dedicated_model_identity_ready( &serde_json::json!({"data": [{"id": expected, "meta": null}]}), - expected + &[expected] ), Ok(false) ); assert_eq!( dedicated_model_identity_ready( &serde_json::json!({"data": [{"id": expected, "meta": {}}]}), - expected + &[expected] ), Ok(true) ); + // A launch with a projector sets `--alias `; llama.cpp + // then reports the alias, which the launch also owns. + let library_id = "embedding/gemma-embedding2/embeddinggemma-2-gguf"; + assert_eq!( + dedicated_model_identity_ready( + &serde_json::json!({"data": [{"id": library_id, "meta": {}}]}), + &[expected, library_id] + ), + Ok(true) + ); + assert!(dedicated_model_identity_ready( + &serde_json::json!({"data": [{"id": library_id, "meta": {}}]}), + &[expected] + ) + .is_err()); } #[tokio::test] @@ -545,18 +565,18 @@ mod tests { let deadline = tokio::time::Instant::now() + Duration::from_secs(2); assert_eq!( client - .dedicated_model_ready(&endpoint, "/models/test.gguf", deadline) + .dedicated_model_ready(&endpoint, &["/models/test.gguf"], deadline) .await, Ok(false) ); assert_eq!( client - .dedicated_model_ready(&endpoint, "/models/test.gguf", deadline) + .dedicated_model_ready(&endpoint, &["/models/test.gguf"], deadline) .await, Ok(true) ); assert!(client - .dedicated_model_ready(&endpoint, "/models/wrong.gguf", deadline) + .dedicated_model_ready(&endpoint, &["/models/wrong.gguf"], deadline) .await .is_err()); stop.send(()).unwrap(); From 53f3094700d3f6958b8359b7c7f81c92eaac0d9f Mon Sep 17 00:00:00 2001 From: MrScripty Date: Wed, 4 Nov 2026 01:50:18 -0700 Subject: [PATCH 4/9] fix(library): stop guessed startup downloads, silent import refusals and partial rebuilds Found re-checking the standards remediation findings against 0.8.0: - Startup recovery guessed a Hugging Face repository (`family/folder`) for any folder with partial files and no download record, then started a network download into it with no user action. Only a repository recorded in the `.pumas_download` marker now resumes; others are logged and left. Inspection failures are logged instead of dropped. - import_batch, classify_model_import_paths and detect_sharded_sets turned a missing or malformed list into an empty one and reported success. They now refuse with the reason. - Rebuilding the model-type rule tables (run when categories are added, as in 0.8.0) was four separate statements; an interruption after the rename stranded user rules. Each rebuild now runs in one savepoint. Co-Authored-By: Claude Opus 5.5 --- rust/crates/pumas-core/src/api/builder.rs | 39 ++++++---- .../src/index/model_index/governance.rs | 75 ++++++++++++++++++- .../pumas-rpc/src/handlers/models/imports.rs | 36 +++++---- 3 files changed, 121 insertions(+), 29 deletions(-) diff --git a/rust/crates/pumas-core/src/api/builder.rs b/rust/crates/pumas-core/src/api/builder.rs index 25f9801b8..1dc12cd8b 100644 --- a/rust/crates/pumas-core/src/api/builder.rs +++ b/rust/crates/pumas-core/src/api/builder.rs @@ -148,11 +148,22 @@ fn start_primary_background_work( }, ); runtime_tasks.spawn(async move { - let Ok(scan) = scan else { - return; - }; - let Ok(Ok(interrupted)) = scan.await else { - return; + let interrupted = match scan { + Err(error) => { + tracing::warn!("Could not inspect interrupted downloads: {error}"); + return; + } + Ok(scan) => match scan.await { + Ok(Ok(interrupted)) => interrupted, + Ok(Err(error)) => { + tracing::warn!("Could not inspect interrupted downloads: {error}"); + return; + } + Err(error) => { + tracing::warn!("Interrupted download inspection did not finish: {error}"); + return; + } + }, }; if interrupted.is_empty() { return; @@ -166,14 +177,16 @@ fn start_primary_background_work( return; }; for item in interrupted { - let repo_id = item.repo_id.unwrap_or_else(|| { - let dir_name = item - .model_dir - .file_name() - .and_then(|n| n.to_str()) - .unwrap_or(&item.inferred_name); - format!("{}/{}", item.family, dir_name) - }); + // Only a repository recorded by Pumas when the download + // started authorizes resuming it. A name guessed from the + // folder could fetch an unrelated repository into it. + let Some(repo_id) = item.repo_id else { + tracing::warn!( + "Not resuming partial files in {}: no download record names their repository", + crate::platform::platform_display_path(&item.model_dir), + ); + continue; + }; let request = model_library::DownloadRequest { repo_id: repo_id.clone(), family: item.family, diff --git a/rust/crates/pumas-core/src/index/model_index/governance.rs b/rust/crates/pumas-core/src/index/model_index/governance.rs index 0396ad755..06ca940f8 100644 --- a/rust/crates/pumas-core/src/index/model_index/governance.rs +++ b/rust/crates/pumas-core/src/index/model_index/governance.rs @@ -246,8 +246,35 @@ impl ModelIndex { Ok(()) } + /// Run a multi-statement schema change as one unit. A savepoint nests + /// inside a caller's transaction and, at the outermost level, is atomic + /// across a crash: an interrupted rebuild never leaves a renamed table + /// behind with an empty replacement. + fn all_or_nothing(conn: &Connection, name: &str, sql: &str) -> Result<()> { + conn.execute_batch(&format!("SAVEPOINT {name};"))?; + match conn.execute_batch(sql) { + Ok(()) => { + conn.execute_batch(&format!("RELEASE {name};"))?; + Ok(()) + } + Err(error) => { + if let Err(rollback) = + conn.execute_batch(&format!("ROLLBACK TO {name}; RELEASE {name};")) + { + return Err(PumasError::Database { + message: format!("{error}; rolling back {name} also failed: {rollback}"), + source: None, + }); + } + Err(error.into()) + } + } + } + fn rebuild_model_type_arch_rules_table(conn: &Connection) -> Result<()> { - conn.execute_batch( + Self::all_or_nothing( + conn, + "rebuild_model_type_arch_rules", " ALTER TABLE model_type_arch_rules RENAME TO model_type_arch_rules_legacy; @@ -297,7 +324,9 @@ impl ModelIndex { } fn rebuild_model_type_config_rules_table(conn: &Connection) -> Result<()> { - conn.execute_batch( + Self::all_or_nothing( + conn, + "rebuild_model_type_config_rules", " ALTER TABLE model_type_config_rules RENAME TO model_type_config_rules_legacy; @@ -911,3 +940,45 @@ impl ModelIndex { Ok(mapped) } } + +#[cfg(test)] +mod rebuild_tests { + use super::*; + + #[test] + fn a_failed_rule_table_rebuild_leaves_the_original_table_and_rows() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE model_type_arch_rules (id INTEGER PRIMARY KEY, pattern TEXT); + INSERT INTO model_type_arch_rules (pattern) VALUES ('UserRuleForASR');", + ) + .unwrap(); + + // Fails after the rename, as an interrupted rebuild would. + let result = ModelIndex::all_or_nothing( + &conn, + "rebuild_model_type_arch_rules", + "ALTER TABLE model_type_arch_rules RENAME TO model_type_arch_rules_legacy; + CREATE TABLE model_type_arch_rules (id INTEGER PRIMARY KEY, pattern TEXT); + INSERT INTO model_type_arch_rules SELECT * FROM missing_table;", + ); + + assert!(result.is_err()); + let pattern: String = conn + .query_row("SELECT pattern FROM model_type_arch_rules", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(pattern, "UserRuleForASR"); + let legacy: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE name = 'model_type_arch_rules_legacy'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(legacy, 0); + // The connection is usable afterwards: no savepoint is left open. + assert!(conn.is_autocommit()); + } +} diff --git a/rust/crates/pumas-rpc/src/handlers/models/imports.rs b/rust/crates/pumas-rpc/src/handlers/models/imports.rs index 259be1489..bd0bcc6a4 100644 --- a/rust/crates/pumas-rpc/src/handlers/models/imports.rs +++ b/rust/crates/pumas-rpc/src/handlers/models/imports.rs @@ -169,12 +169,27 @@ pub async fn import_local_cohere(state: &AppState, params: &Value) -> pumas_libr )?) } +/// A required list parameter. A missing or malformed list is refused with its +/// reason; it never becomes an empty list that reports success. +fn required_list( + params: &Value, + key: &str, +) -> pumas_library::Result> { + let value = params + .get(key) + .ok_or_else(|| pumas_library::PumasError::InvalidParams { + message: format!("`{key}` is required"), + })?; + serde_json::from_value(value.clone()).map_err(|error| { + pumas_library::PumasError::InvalidParams { + message: format!("`{key}` is invalid: {error}"), + } + }) +} + pub async fn import_batch(state: &AppState, params: &Value) -> pumas_library::Result { - // Parse the imports array from params - let imports: Vec = params - .get("imports") - .and_then(|v| serde_json::from_value(v.clone()).ok()) - .unwrap_or_default(); + let imports: Vec = + required_list(params, "imports")?; let results = state.api.import_models_batch(imports).await; let imported = results.iter().filter(|r| r.success).count(); @@ -213,10 +228,7 @@ pub async fn classify_model_import_paths( state: &AppState, params: &Value, ) -> pumas_library::Result { - let paths: Vec = params - .get("paths") - .and_then(|value| serde_json::from_value(value.clone()).ok()) - .unwrap_or_default(); + let paths: Vec = required_list(params, "paths")?; let result = state.api.classify_model_import_paths(&paths).await?; Ok(serde_json::to_value(result)?) @@ -292,11 +304,7 @@ pub async fn detect_sharded_sets( _state: &AppState, params: &Value, ) -> pumas_library::Result { - // Get files array from params - let files: Vec = params - .get("files") - .and_then(|v| serde_json::from_value(v.clone()).ok()) - .unwrap_or_default(); + let files: Vec = required_list(params, "files")?; let mut paths = Vec::with_capacity(files.len()); for file in files { From 7d0284680f3708968bc0edddf8b49efba8fd0d99 Mon Sep 17 00:00:00 2001 From: MrScripty Date: Wed, 4 Nov 2026 01:50:18 -0700 Subject: [PATCH 5/9] fix(desktop): save a chosen category on Apply and re-read what the backend saved - Picking an option no longer saves it: keyboard users stepping through the options saved a category on every step, and the select was disabled (and lost focus) while saving. An Apply button saves; the select stays usable. - The metadata window re-reads the category, its source, task and modalities from the backend after a save instead of guessing them. - "Detected: " misreported detection; the option is now "Detect automatically", and the hint names the current state and is the select's description. Saves are announced. - The choosable category list is checked at compile time against the backend's generated category type. Co-Authored-By: Claude Opus 5.5 --- .../components/ModelCategoryControl.test.tsx | 29 ++++++--- .../src/components/ModelCategoryControl.tsx | 61 ++++++++++++++----- .../src/components/ModelMetadataModal.tsx | 28 +++++++-- frontend/src/utils/modelCategories.ts | 15 ++++- 4 files changed, 102 insertions(+), 31 deletions(-) diff --git a/frontend/src/components/ModelCategoryControl.test.tsx b/frontend/src/components/ModelCategoryControl.test.tsx index 5538fb04d..d7f8d8d01 100644 --- a/frontend/src/components/ModelCategoryControl.test.tsx +++ b/frontend/src/components/ModelCategoryControl.test.tsx @@ -15,17 +15,27 @@ vi.mock('../api/models', () => ({ describe('ModelCategoryControl', () => { beforeEach(() => setModelCategoryMock.mockReset()); - it('records a chosen category and reports it as the person\'s choice', async () => { + it('saves only on Apply, so stepping through options records nothing', async () => { const onChanged = vi.fn(); setModelCategoryMock.mockResolvedValue({ success: true, model_id: 'llm/x/kev', category: 'decision' }); render(); - expect(screen.getByRole('combobox')).toHaveValue(''); - expect(screen.getByText('Detected: Language models')).toBeInTheDocument(); - fireEvent.change(screen.getByRole('combobox'), { target: { value: 'decision' } }); + const select = screen.getByRole('combobox', { name: 'Used as' }); + expect(select).toHaveValue(''); + expect(select).toHaveAccessibleDescription('Detected as Language models from the model files and source.'); + const apply = screen.getByRole('button', { name: 'Apply' }); + expect(apply).toBeDisabled(); - await waitFor(() => expect(onChanged).toHaveBeenCalledWith('decision', 'user')); + fireEvent.change(select, { target: { value: 'vision' } }); + fireEvent.change(select, { target: { value: 'decision' } }); + expect(setModelCategoryMock).not.toHaveBeenCalled(); + expect(select).not.toBeDisabled(); + + fireEvent.click(apply); + await waitFor(() => expect(onChanged).toHaveBeenCalledTimes(1)); + expect(setModelCategoryMock).toHaveBeenCalledTimes(1); expect(setModelCategoryMock).toHaveBeenCalledWith('llm/x/kev', 'decision'); + expect(screen.getByRole('status')).toHaveTextContent('Saved as Decision and evaluation.'); }); it('clears a choice to re-detect, and shows refusal reasons', async () => { @@ -33,9 +43,12 @@ describe('ModelCategoryControl', () => { setModelCategoryMock.mockRejectedValueOnce(new Error('category must be one of llm, vlm')); render(); - expect(screen.getByRole('combobox')).toHaveValue('vision'); - expect(screen.getByText('Chosen by you; re-scans keep it.')).toBeInTheDocument(); - fireEvent.change(screen.getByRole('combobox'), { target: { value: '' } }); + const select = screen.getByRole('combobox', { name: 'Used as' }); + expect(select).toHaveValue('vision'); + expect(screen.getByText('Chosen by you: Vision. Re-scans keep it.')).toBeInTheDocument(); + expect(screen.getByRole('option', { name: 'Detect automatically' })).toBeInTheDocument(); + fireEvent.change(select, { target: { value: '' } }); + fireEvent.click(screen.getByRole('button', { name: 'Apply' })); expect(await screen.findByRole('alert')).toHaveTextContent('category must be one of llm, vlm'); expect(setModelCategoryMock).toHaveBeenCalledWith('llm/x/kev', null); expect(onChanged).not.toHaveBeenCalled(); diff --git a/frontend/src/components/ModelCategoryControl.tsx b/frontend/src/components/ModelCategoryControl.tsx index f7325b00b..ed3ac86ea 100644 --- a/frontend/src/components/ModelCategoryControl.tsx +++ b/frontend/src/components/ModelCategoryControl.tsx @@ -2,9 +2,13 @@ * Choose what a model is used for. The choice is library metadata: the model * keeps its folder and ID, and later re-detection keeps the choice until it * is cleared. + * + * Picking an option does not save it; **Apply** does, so stepping through the + * options with the keyboard records nothing. The backend owns the result: after + * a save the parent re-reads the model's metadata. */ -import { useState } from 'react'; +import { useEffect, useState } from 'react'; import { modelsAPI } from '../api/models'; import { CHOOSABLE_MODEL_CATEGORIES, @@ -19,7 +23,8 @@ interface ModelCategoryControlProps { modelId: string; category: string; resolutionSource: string | null; - onChanged: (category: string, resolutionSource: string | null) => void; + /** Called after the backend saved a change; re-read the model's metadata. */ + onChanged: () => void; } export function ModelCategoryControl({ @@ -28,16 +33,27 @@ export function ModelCategoryControl({ resolutionSource, onChanged, }: ModelCategoryControlProps) { + const chosen = resolutionSource === USER_CATEGORY_SOURCE; + const current = chosen ? category : DETECT; + const [pending, setPending] = useState(current); const [saving, setSaving] = useState(false); const [error, setError] = useState(null); - const chosen = resolutionSource === USER_CATEGORY_SOURCE; + const [saved, setSaved] = useState(null); + + useEffect(() => setPending(current), [current]); - const choose = async (value: string) => { + const apply = async () => { setSaving(true); setError(null); + setSaved(null); try { - const result = await modelsAPI.setModelCategory(modelId, value === DETECT ? null : value); - onChanged(result.category, value === DETECT ? null : USER_CATEGORY_SOURCE); + const result = await modelsAPI.setModelCategory(modelId, pending === DETECT ? null : pending); + setSaved( + pending === DETECT + ? `Detected as ${modelCategoryLabel(result.category)}.` + : `Saved as ${modelCategoryLabel(result.category)}.`, + ); + onChanged(); } catch (cause) { setError(cause instanceof Error && cause.message ? cause.message : 'The category could not be saved.'); } finally { @@ -45,29 +61,42 @@ export function ModelCategoryControl({ } }; + const selectId = `model-category-${modelId}`; + const hintId = `${selectId}-hint`; return (
-