From 2b8d0642bce3339a4fc5abeef16a6f94c6efb1cb Mon Sep 17 00:00:00 2001 From: dervoeti Date: Fri, 21 Aug 2026 07:30:41 +0000 Subject: [PATCH 1/9] feat: Add a shared generator for vendored JavaScript SBOMs --- shared/sbom/vendored_js.py | 189 +++++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100755 shared/sbom/vendored_js.py diff --git a/shared/sbom/vendored_js.py b/shared/sbom/vendored_js.py new file mode 100755 index 000000000..6b7f9e5e0 --- /dev/null +++ b/shared/sbom/vendored_js.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Generates a CycloneDX SBOM for third-party JavaScript that is checked into a product's source +tree as pre-built (usually minified) files. + +Such files carry no package manifest and no lockfile, so cdxgen, syft and trivy are all blind to +them: the libraries are shipped in our images but appear in no SBOM. We found no tool that +identifies a minified bundle reliably. retire.js recognises only about half of the libraries we +ship and gets some versions wrong. So the components have to be recorded by hand, in a manifest +per product version: + + /stackable/vendored-js/.json + +An entry may also declare the libraries that a bundle inlines, which is how a library that is +shipped without a file of its own still ends up in the SBOM. Trino's vendored vis bundle for +example inlines a copy of moment that nothing else would report. + +A hand-written manifest goes stale the moment a product version is bumped, so every entry pins the +SHA-256 of the file it describes. Both commands fail on a changed file, on a file that is listed +nowhere and on an entry whose file has disappeared. Recording a version is therefore a one-time +cost per file, and the build tells us when to revisit it. + +shared/sbom/identify_js.py helps with writing and updating a manifest. +""" + +import argparse +import hashlib +import json +import re +import sys +from pathlib import Path +from urllib.parse import unquote + +# A purl always carries the name and the version, so they are not repeated in the manifest. +PURL = re.compile(r"^pkg:[^/]+/(?P.+)@(?P[^@?#]+)$") + + +def scan(manifest, source_root): + """Every JavaScript file below the scanned directories, keyed by its path relative to the + source root. The manifest uses those relative paths because they stay unambiguous even when a + product has several scanned directories.""" + return { + str(path.relative_to(source_root)): path + for directory in manifest["scan-dirs"] + for path in sorted((source_root / directory).rglob("*.js")) + } + + +def verify(manifest, source_root): + """Every disagreement between the manifest and the source tree that would make the generated SBOM wrong.""" + own = set(manifest.get("own", [])) + listed = {} + problems = [] + + for library in manifest["libraries"]: + if library["file"] in listed or library["file"] in own: + problems.append(f"DUPLICATE {library['file']}\n Listed more than once in the manifest.") + listed[library["file"]] = library + + found = scan(manifest, source_root) + for file, path in found.items(): + library = listed.get(file) + if library is None: + if file not in own: + problems.append( + f'UNLISTED {file}\n Add it to "libraries" with a purl if it is third-party,' + ' or to "own" if the product wrote it.' + ) + continue + + actual = hashlib.sha256(path.read_bytes()).hexdigest() + if library["sha256"] != actual: + problems.append( + f"CHANGED {file}\n manifest {library['sha256']}\n actual {actual}\n" + " The file was updated upstream, so re-check the version it records." + ) + + for file in listed: + if file not in found: + problems.append(f"GONE {file}\n Listed in the manifest but no longer in the source tree.") + for file in sorted(own - found.keys()): + problems.append(f'GONE {file}\n Listed in "own" but no longer in the source tree.') + + return found, problems + + +def identity(entry): + """A component's name and version, plus its purl if it has one. A purl is the preferred + identity because that is what vulnerability scanners match on, but libraries that were never + published to a package registry cannot have one and are recorded by name only. The "note" + field of such an entry says why.""" + purl = entry.get("purl") + if not purl: + if not entry.get("name"): + raise SystemExit(f"Entry without a purl and without a name: {entry}") + return None, entry["name"], entry.get("version") + + match = PURL.match(purl) + if not match: + raise SystemExit(f"Cannot parse the purl {purl}") + return purl, unquote(match["name"]), unquote(match["version"]) + + +def build_bom(manifest, component_version, spec_version): + components = {} + + def add(entry, location, sha256): + purl, name, version = identity(entry) + # Some libraries carry no version anywhere, so the name alone identifies them. + key = purl or (f"{name}@{version}" if version else name) + if key not in components: + component = {"type": "library", "name": name} + if version: + component["version"] = version + component["bom-ref"] = key + if purl: + component["purl"] = purl + if entry.get("license"): + component["licenses"] = [{"expression": entry["license"]}] + if sha256: + component["hashes"] = [] + component["evidence"] = {"occurrences": []} + components[key] = component + + component = components[key] + if sha256: + component.setdefault("hashes", []).append({"alg": "SHA-256", "content": sha256}) + component["evidence"]["occurrences"].append({"location": location}) + + for library in manifest["libraries"]: + # A library can be shipped as several files, for example a minified and a plain build, so + # the files are collapsed into one component that records each of them as evidence. + add(library, library["file"], library["sha256"]) + # Bundles inline their own dependencies, which are shipped without a file of their own. + # Their hash would be the hash of the bundle, so it is deliberately not recorded. + for bundled in library.get("bundles", []): + add(bundled, library["file"], None) + + # No timestamp and no serial number, so that repeated runs produce the same file. + return { + "bomFormat": "CycloneDX", + "specVersion": spec_version, + "version": 1, + "metadata": { + "component": { + "type": "application", + "bom-ref": f"{manifest['name']}@{component_version}", + "name": manifest["name"], + "version": component_version, + }, + "tools": {"components": [{"type": "application", "name": "vendored_js.py", "group": "tech.stackable"}]}, + }, + "components": list(components.values()), + } + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + commands = parser.add_subparsers(dest="command", required=True) + + check_command = commands.add_parser("check", help="report every mismatch between the manifest and the source tree") + bom_command = commands.add_parser("bom", help="write the SBOM the manifest describes") + for command in (check_command, bom_command): + command.add_argument("manifest", type=Path) + command.add_argument("source_root", type=Path) + bom_command.add_argument("output", type=Path) + bom_command.add_argument("component_version") + bom_command.add_argument("spec_version") + + arguments = parser.parse_args() + manifest = json.loads(arguments.manifest.read_text()) + + # Never generate an SBOM that we know to be wrong, so this also gates "bom". + found, problems = verify(manifest, arguments.source_root) + if problems: + print(f"{arguments.manifest} does not match the source tree ({len(problems)} problem(s)):\n", file=sys.stderr) + print("\n".join(problems), file=sys.stderr) + raise SystemExit(1) + + if arguments.command == "check": + print(f"{arguments.manifest}: {len(found)} JavaScript files, all accounted for") + return + + bom = build_bom(manifest, arguments.component_version, arguments.spec_version) + arguments.output.write_text(json.dumps(bom, indent=2) + "\n") + print(f"Wrote {arguments.output} with {len(bom['components'])} components") + + +if __name__ == "__main__": + main() From 2c53f4f115e836b85c2cdb179086e67046febc6a Mon Sep 17 00:00:00 2001 From: dervoeti Date: Fri, 21 Aug 2026 07:30:50 +0000 Subject: [PATCH 2/9] feat(hbase): Add an SBOM for the web UI dependencies --- CHANGELOG.md | 2 + boil.toml | 6 ++ hbase/hbase/Dockerfile | 47 ++++++++++- hbase/hbase/boil-config.toml | 8 ++ hbase/hbase/stackable/hbase_webapps_deps.py | 91 +++++++++++++++++++++ 5 files changed, 153 insertions(+), 1 deletion(-) create mode 100755 hbase/hbase/stackable/hbase_webapps_deps.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ea959918..65d557bcd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ All notable changes to this project will be documented in this file. - airflow, superset, druid, nifi: Add SBOMs for the frontend (npm) dependencies ([#1600]). - nifi: Backport NIFI-15958 to log periodic progress while waiting for the content archive scan and provenance re-index, for `2.6.0`, `2.7.2`, and `2.9.0` ([#1611]). +- hbase: Add an SBOM for the web UI (npm) dependencies, which are unpacked from webjars and therefore not covered by the CycloneDX Maven plugin ([#1620]). ### Changed @@ -29,6 +30,7 @@ All notable changes to this project will be documented in this file. [#1600]: https://github.com/stackabletech/docker-images/pull/1600 [#1611]: https://github.com/stackabletech/docker-images/pull/1611 [#1616]: https://github.com/stackabletech/docker-images/pull/1616 +[#1620]: https://github.com/stackabletech/docker-images/pull/1620 ## [26.7.0] - 2026-07-21 diff --git a/boil.toml b/boil.toml index c7ec34d11..f5ac03bb1 100644 --- a/boil.toml +++ b/boil.toml @@ -6,6 +6,12 @@ DELETE_CACHES = "true" # CycloneDX specification version used for the SBOMs generated by cdxgen. # 1.6 is the lowest version cdxgen 13 accepts as a generation target. CDXGEN_SPEC_VERSION = "1.6" +# Node version used to run cdxgen in the builders that need it. It is unrelated to any product +# and to the Node version that a product uses to build its frontend, so it is configured once +# here instead of per product version. +# Find the latest release here: https://github.com/nodejs/node/releases +# renovate: datasource=node-version packageName=node +SBOM_NODEJS_VERSION = "24.19.0" [metadata] documentation = "https://docs.stackable.tech/home/stable/" diff --git a/hbase/hbase/Dockerfile b/hbase/hbase/Dockerfile index 84ed95311..fe8045efa 100644 --- a/hbase/hbase/Dockerfile +++ b/hbase/hbase/Dockerfile @@ -11,6 +11,9 @@ ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION} ARG TARGETARCH ARG TARGETOS ARG STACKABLE_USER_UID +ARG SBOM_NODEJS_VERSION +ARG CDXGEN_SPEC_VERSION +ARG CDXGEN_VERSION # Setting this to anything other than "true" will keep the cache folders around (e.g. for Maven, NPM etc.) # This can be used to speed up builds when disk space is of no concern. @@ -18,11 +21,31 @@ ARG DELETE_CACHES="true" COPY hbase/licenses /licenses +RUN <= 24, so it gets its own Node installation in /opt/node-sbom and +# is invoked with that prepended to PATH. +# -fsSL is not needed: the shared /root/.curlrc sets location, fail, silent and show-error. +ARCH="${TARGETARCH/amd64/x64}" +mkdir -p /opt/node-sbom +curl "https://repo.stackable.tech/repository/packages/node/node-v${SBOM_NODEJS_VERSION}-linux-${ARCH}.tar.xz" | \ + tar --extract --xz --directory=/opt/node-sbom --strip-components=1 +PATH="/opt/node-sbom/bin:$PATH" npm install --global "@cdxgen/cdxgen@${CDXGEN_VERSION}" + +microdnf update +microdnf install python3 +microdnf clean all +rm -rf /var/cache/yum +EOF + USER ${STACKABLE_USER_UID} WORKDIR /stackable COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase/stackable/patches/patchable.toml /stackable/src/hbase/hbase/stackable/patches/patchable.toml COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase/stackable/patches/${PRODUCT_VERSION} /stackable/src/hbase/hbase/stackable/patches/${PRODUCT_VERSION} +COPY --chown=${STACKABLE_USER_UID}:0 hbase/hbase/stackable/hbase_webapps_deps.py /stackable/hbase_webapps_deps.py COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 /stackable/patched-libs /stackable/patched-libs # Cache mounts are owned by root by default @@ -36,7 +59,9 @@ COPY --from=hadoop-builder --chown=${STACKABLE_USER_UID}:0 /stackable/patched-li # builder containers will share the same cache and the `rm -rf` commands will fail # with a "directory not empty" error on the first builder to finish, as other builders # are still working in the cache directory. -RUN --mount=type=cache,id=maven-hbase-${PRODUCT_VERSION},uid=${STACKABLE_USER_UID},target=/stackable/.m2/repository <s of a plugin and not as project dependencies, the CycloneDX Maven plugin does not +pick them up, so they are missing from the HBase SBOM. + +The generated package.json is only an intermediate artifact: cdxgen turns it into the actual +CycloneDX SBOM. npm coordinates are used rather than the Maven ones, because vulnerability scanners +match advisories against pkg:npm and largely fail to match pkg:maven/org.webjars purls. +""" + +import argparse +import json +import re +from pathlib import Path +from xml.etree import ElementTree + +PROPERTY = re.compile(r"\$\{([\w.-]+)\}") + + +def tag(element): + """The tag of an element without the Maven POM namespace.""" + # ElementTree keeps the namespace in the tag itself, so the root element of a pom is called + # "{http://maven.apache.org/POM/4.0.0}project". + return element.tag.rpartition("}")[2] + + +def properties(pom): + """Every entry of a pom, for example 3.7.1.""" + # The whole tree is walked because is not only a top-level element: HBase declares + # most of its properties inside profiles. + entries = {} + for block in pom.iter(): + if tag(block) == "properties": + for entry in block: + entries[tag(entry)] = (entry.text or "").strip() + return entries + + +def webjars(server_pom, versions): + """The org.webjars artifacts that the maven-dependency-plugin unpacks, as npm dependencies. + The webjar artifact IDs match their npm package names, so they can be used verbatim.""" + dependencies = {} + for item in server_pom.iter(): + if tag(item) != "artifactItem": + continue + + fields = {tag(field): (field.text or "").strip() for field in item} + if fields.get("groupId") != "org.webjars": + continue + if not fields.get("artifactId") or not fields.get("version"): + raise SystemExit(f" without an artifactId or version: {fields}") + + version = PROPERTY.sub(lambda match: versions.get(match[1], match[0]), fields["version"]) + if "${" in version: + raise SystemExit(f"Cannot resolve the version of {fields['artifactId']} from the root pom: {version}") + + # bootstrap is unpacked twice, once for its JavaScript and once for its CSS. + dependencies[fields["artifactId"]] = version + + # Guard against upstream restructuring the pom, which would otherwise silently produce an SBOM + # without any components. + if not dependencies: + raise SystemExit("No org.webjars found in hbase-server/pom.xml, did the pom layout change?") + return dependencies + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("source_root", type=Path, help="the HBase source tree") + parser.add_argument("version", help="the HBase version, used as the version of the generated package") + parser.add_argument("output", type=Path, help="the package.json to write") + arguments = parser.parse_args() + + dependencies = webjars( + ElementTree.parse(arguments.source_root / "hbase-server/pom.xml"), + properties(ElementTree.parse(arguments.source_root / "pom.xml")), + ) + package = {"name": "hbase-webapps", "version": arguments.version, "private": True, "dependencies": dependencies} + arguments.output.write_text(json.dumps(package, indent=2) + "\n") + + summary = ", ".join(f"{name}@{version}" for name, version in dependencies.items()) + print(f"Wrote {arguments.output}: {summary}") + + +if __name__ == "__main__": + main() From 077c0a53f0817f68b6898bb3ea35b525fc1b023c Mon Sep 17 00:00:00 2001 From: dervoeti Date: Fri, 21 Aug 2026 07:30:57 +0000 Subject: [PATCH 3/9] feat(hadoop, spark, trino): Add SBOMs for the web UI dependencies --- CHANGELOG.md | 2 + hadoop/hadoop/Dockerfile | 20 ++- .../hadoop/stackable/vendored-js/3.3.6.json | 121 ++++++++++++++++ .../hadoop/stackable/vendored-js/3.4.2.json | 122 ++++++++++++++++ .../hadoop/stackable/vendored-js/3.4.3.json | 122 ++++++++++++++++ .../hadoop/stackable/vendored-js/3.5.0.json | 123 ++++++++++++++++ spark-k8s/Dockerfile.3 | 31 +++++ spark-k8s/Dockerfile.4 | 31 +++++ spark-k8s/stackable/vendored-js/3.5.8.json | 122 ++++++++++++++++ spark-k8s/stackable/vendored-js/4.1.1.json | 131 ++++++++++++++++++ spark-k8s/stackable/vendored-js/4.1.2.json | 131 ++++++++++++++++++ trino/trino/Dockerfile | 83 +++++++++++ trino/trino/boil-config.toml | 18 +++ trino/trino/stackable/vendored-js/477.json | 112 +++++++++++++++ trino/trino/stackable/vendored-js/479.json | 112 +++++++++++++++ trino/trino/stackable/vendored-js/481.json | 112 +++++++++++++++ 16 files changed, 1391 insertions(+), 2 deletions(-) create mode 100644 hadoop/hadoop/stackable/vendored-js/3.3.6.json create mode 100644 hadoop/hadoop/stackable/vendored-js/3.4.2.json create mode 100644 hadoop/hadoop/stackable/vendored-js/3.4.3.json create mode 100644 hadoop/hadoop/stackable/vendored-js/3.5.0.json create mode 100644 spark-k8s/stackable/vendored-js/3.5.8.json create mode 100644 spark-k8s/stackable/vendored-js/4.1.1.json create mode 100644 spark-k8s/stackable/vendored-js/4.1.2.json create mode 100644 trino/trino/stackable/vendored-js/477.json create mode 100644 trino/trino/stackable/vendored-js/479.json create mode 100644 trino/trino/stackable/vendored-js/481.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 65d557bcd..6597ec766 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,8 @@ All notable changes to this project will be documented in this file. - airflow, superset, druid, nifi: Add SBOMs for the frontend (npm) dependencies ([#1600]). - nifi: Backport NIFI-15958 to log periodic progress while waiting for the content archive scan and provenance re-index, for `2.6.0`, `2.7.2`, and `2.9.0` ([#1611]). - hbase: Add an SBOM for the web UI (npm) dependencies, which are unpacked from webjars and therefore not covered by the CycloneDX Maven plugin ([#1620]). +- trino: Add SBOMs for the web UI, both for the two npm projects behind it and for the pre-built JavaScript vendored into the source tree ([#1620]). +- hadoop, spark: Add SBOMs for the pre-built JavaScript that is vendored into the source tree for the HDFS and Spark web UIs ([#1620]). ### Changed diff --git a/hadoop/hadoop/Dockerfile b/hadoop/hadoop/Dockerfile index b8387410e..a336c1858 100644 --- a/hadoop/hadoop/Dockerfile +++ b/hadoop/hadoop/Dockerfile @@ -12,6 +12,7 @@ ARG AZURE_STORAGE_VERSION ARG AZURE_KEYVAULT_CORE_VERSION ARG ANALYTICSACCELERATOR_S3_VERSION ARG STACKABLE_USER_UID +ARG CDXGEN_SPEC_VERSION WORKDIR /stackable @@ -21,8 +22,9 @@ COPY --chown=${STACKABLE_USER_UID}:0 shared/protobuf/stackable/patches/${PROTOBU RUN <>> Build spark RUN <>> Build spark RUN <= 24, which is unrelated to the Node version that the +# frontend-maven-plugin downloads for the actual build, so it gets its own Node installation in +# /opt/node-sbom and is invoked with that prepended to PATH. +# -fsSL is not needed: the shared /root/.curlrc sets location, fail, silent and show-error. +ARCH="${TARGETARCH/amd64/x64}" +mkdir -p /opt/node-sbom +curl "https://repo.stackable.tech/repository/packages/node/node-v${SBOM_NODEJS_VERSION}-linux-${ARCH}.tar.xz" | \ + tar --extract --xz --directory=/opt/node-sbom --strip-components=1 +PATH="/opt/node-sbom/bin:$PATH" npm install --global "@cdxgen/cdxgen@${CDXGEN_VERSION}" + +microdnf update +microdnf install python3 +microdnf clean all +rm -rf /var/cache/yum +EOF + +COPY --chown=${STACKABLE_USER_UID}:0 shared/sbom/vendored_js.py /stackable/vendored_js.py +COPY --chown=${STACKABLE_USER_UID}:0 trino/trino/stackable/vendored-js/${PRODUCT_VERSION}.json /stackable/vendored-js.json + # adding a hadolint ignore for SC2215, due to https://github.com/hadolint/hadolint/issues/980 # hadolint ignore=SC2215 RUN --mount=type=cache,id=maven-${PRODUCT_VERSION},target=/root/.m2/repository < Date: Fri, 21 Aug 2026 07:31:00 +0000 Subject: [PATCH 4/9] feat: Add a tool for identifying vendored JavaScript --- shared/sbom/identify_js.py | 189 +++++++++++++++++++++++++++++++++++++ 1 file changed, 189 insertions(+) create mode 100755 shared/sbom/identify_js.py diff --git a/shared/sbom/identify_js.py b/shared/sbom/identify_js.py new file mode 100755 index 000000000..e0d5b87ec --- /dev/null +++ b/shared/sbom/identify_js.py @@ -0,0 +1,189 @@ +#!/usr/bin/env python3 +"""Authoring aid for the manifests that shared/sbom/vendored_js.py consumes. Not used by any build. + +Writing such a manifest means naming a library and a version for a pre-built, usually minified +JavaScript file. The file name is not evidence: Hadoop ships d3 4.1.0 as "d3-v4.1.1.min.js" and +mustache.js as "jquery.mustache.js". Neither is a header comment always present. + +This tool provides the two things that turn writing a manifest into reading off facts: + + inspect lists every .js file under the given directories with its SHA-256 and any version + string found near the top of the file, which is the starting point for a new + manifest and for seeing what a version bump changed. + + identify downloads every published version of the given npm packages and hashes every file in + them, looking for one that is byte-identical to ours. A match is proof, and it is + what established that Hadoop's "d3-v4.1.1.min.js" really is d3 4.1.0. + +A file is also compared with its surrounding whitespace stripped, because vendoring a file through +an editor or a shell redirection commonly appends a trailing newline. Trino's clipboard.min.js is +the published clipboard 2.0.11 plus exactly one such byte. That is the same code and the same +advisories apply, so it counts as a match, and the manifest entry says which kind it was. + +When several releases match, the file was shipped unchanged across them and hashing cannot tell +them apart. Record the lowest one: it is the earliest release the code appeared in, and it keeps +the widest set of advisories applicable, which is the safe direction. Note the ambiguity in the +manifest entry. + +When nothing matches, the library either predates its npm releases or the product modified or +rebuilt it. Fall back to the version the file states and say so in the entry. + +Examples: + identify_js.py inspect . hadoop-hdfs-project/hadoop-hdfs/src/main/webapps + identify_js.py identify webapps/static/d3-v4.1.1.min.js d3 --prefix 4. + +Tarballs are cached, override the location with IDENTIFY_JS_CACHE. +""" + +import argparse +import hashlib +import json +import os +import re +import tarfile +import tempfile +from pathlib import Path +from urllib.error import URLError +from urllib.request import urlopen + +CACHE = Path(os.environ.get("IDENTIFY_JS_CACHE", Path(tempfile.gettempdir()) / "stackable-identify-js-cache")) + +# Version strings are written in every conceivable way, so cast a wide net over the top of the file +# and let the caller judge. Matching on bytes keeps minified files with odd encodings readable. +HINTS = [ + re.compile(rb"@version\s+v?([0-9]+\.[0-9][\w.-]*)"), + re.compile(rb"\bversion\s*[:=]\s*['\"]?v?([0-9]+\.[0-9][\w.-]*)", re.IGNORECASE), + # Banners such as "// https://d3js.org Version 4.1.0." separate with a space. All three + # components are required here, otherwise every "Apache License, Version 2.0" header matches. + re.compile(rb"\bversion\s+v?([0-9]+\.[0-9]+\.[0-9][\w.-]*)", re.IGNORECASE), + re.compile(rb"^/\*!?\s*([A-Za-z][\w.\- ]*?)\s+v?([0-9]+\.[0-9][\w.-]*)", re.MULTILINE), + re.compile(rb"\bv([0-9]+\.[0-9]+\.[0-9][\w.-]*)"), +] + + +def sha256(contents): + return hashlib.sha256(contents).hexdigest() + + +def digests(contents): + """The SHA-256 of the file and of the same file without its surrounding whitespace. The second + one identifies a copy that a vendoring step gave a trailing newline, which happens often enough + that comparing only the first would report the library as modified.""" + return sha256(contents), sha256(contents.strip()) + + +def version_hints(contents): + hints = [] + for pattern in HINTS: + for match in pattern.finditer(contents[:3000]): + # Rstrip because a version at the end of a sentence swallows the full stop. + hint = b" ".join(group for group in match.groups() if group).decode("latin1").rstrip(".-") + if hint not in hints: + hints.append(hint) + return hints + + +def published_versions(package, prefix): + """Every non-prerelease version of a package, ascending. The registry lists them in publication + order, which is not always ascending, and the advice to record the lowest match depends on the + order being right.""" + # The scope separator has to stay encoded, otherwise the registry sees two path segments. + with urlopen(f"https://registry.npmjs.org/{package.replace('/', '%2f')}") as response: + metadata = json.load(response) + + releases = [ + (version, release["dist"]["tarball"]) + for version, release in metadata.get("versions", {}).items() + # A prerelease is never what a product vendored. + if "-" not in version and version.startswith(prefix) and release.get("dist", {}).get("tarball") + ] + return sorted(releases, key=lambda release: [int(part) if part.isdigit() else 0 for part in release[0].split(".")]) + + +def tarball_hashes(package, version, url): + """The SHA-256 of every file in a release and of its stripped contents, keyed by its path inside + the package.""" + archive_path = CACHE / f"{package.replace('/', '_')}-{version}.tgz" + if not archive_path.exists(): + CACHE.mkdir(parents=True, exist_ok=True) + with urlopen(url) as response: + archive_path.write_bytes(response.read()) + + try: + with tarfile.open(archive_path) as archive: + # Every member is below a "package/" directory that is of no interest here. + return { + member.name.split("/", 1)[-1]: digests(archive.extractfile(member).read()) + for member in archive + if member.isfile() + } + except tarfile.TarError: + # A handful of very old releases have broken tarballs, skip them. + return {} + + +def inspect(source_root, directories): + for directory in directories: + for path in sorted((source_root / directory).rglob("*.js")): + contents = path.read_bytes() + hints = " | ".join(version_hints(contents)[:3]) or "-" + print("\t".join([str(path.relative_to(source_root)), sha256(contents), hints])) + + +def identify(target, packages, prefix): + wanted, wanted_stripped = digests(target.read_bytes()) + print(f"{target}\n sha256 {wanted}\n") + + matches = [] + for package in packages: + try: + releases = published_versions(package, prefix) + except URLError as error: + print(f"{package}: {error}") + continue + print(f"{package}: checking {len(releases)} version(s)") + for version, url in releases: + for name, (digest, stripped) in tarball_hashes(package, version, url).items(): + if digest == wanted: + matches.append((package, version, "identical")) + print(f" MATCH {package}@{version} {name}") + elif stripped == wanted_stripped: + matches.append((package, version, "whitespace")) + print(f" MATCH {package}@{version} {name} (differs only in surrounding whitespace)") + + if not matches: + print("\nNo match. Fall back to the version the file states and note that in the manifest.") + return + + if any(kind == "whitespace" for _, _, kind in matches): + print("\nThe code is identical, only the surrounding whitespace differs, so the release applies.") + print("Record it and say in the manifest that the copy carries extra whitespace.") + + if len(matches) > 1: + package, version, _ = matches[0] + print(f"\nThe file is the same in {len(matches)} releases, so record the lowest,") + print(f"{package}@{version}, and note the ambiguity in the manifest.") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + commands = parser.add_subparsers(dest="command", required=True) + + inspect_command = commands.add_parser("inspect", help="list the JavaScript files below the given directories") + inspect_command.add_argument("source_root", type=Path) + inspect_command.add_argument("directories", nargs="+") + + identify_command = commands.add_parser("identify", help="find the npm release a file came from") + identify_command.add_argument("file", type=Path) + identify_command.add_argument("packages", nargs="+", help="npm packages the file might come from") + identify_command.add_argument("--prefix", default="", help='only check versions starting with this, e.g. "4."') + + arguments = parser.parse_args() + if arguments.command == "inspect": + inspect(arguments.source_root, arguments.directories) + else: + identify(arguments.file, arguments.packages, arguments.prefix) + + +if __name__ == "__main__": + main() From 2f942680bc405e6daa07821a8847bb2d0de3c0ee Mon Sep 17 00:00:00 2001 From: dervoeti Date: Mon, 24 Aug 2026 14:53:04 +0000 Subject: [PATCH 5/9] chore: Unify the name of the Node version used to run cdxgen --- boil.toml | 5 +++-- hbase/hbase/Dockerfile | 14 +++++++------- trino/trino/Dockerfile | 23 +++++++++-------------- 3 files changed, 19 insertions(+), 23 deletions(-) diff --git a/boil.toml b/boil.toml index f5ac03bb1..881adcdfc 100644 --- a/boil.toml +++ b/boil.toml @@ -8,10 +8,11 @@ DELETE_CACHES = "true" CDXGEN_SPEC_VERSION = "1.6" # Node version used to run cdxgen in the builders that need it. It is unrelated to any product # and to the Node version that a product uses to build its frontend, so it is configured once -# here instead of per product version. +# here instead of per product version. Products that pin `cdxgen-nodejs-version` in their own +# boil-config.toml override this value. # Find the latest release here: https://github.com/nodejs/node/releases # renovate: datasource=node-version packageName=node -SBOM_NODEJS_VERSION = "24.19.0" +CDXGEN_NODEJS_VERSION = "24.19.0" [metadata] documentation = "https://docs.stackable.tech/home/stable/" diff --git a/hbase/hbase/Dockerfile b/hbase/hbase/Dockerfile index fe8045efa..ffe6ff214 100644 --- a/hbase/hbase/Dockerfile +++ b/hbase/hbase/Dockerfile @@ -11,7 +11,7 @@ ENV HADOOP_VERSION=${HADOOP_HADOOP_VERSION} ARG TARGETARCH ARG TARGETOS ARG STACKABLE_USER_UID -ARG SBOM_NODEJS_VERSION +ARG CDXGEN_NODEJS_VERSION ARG CDXGEN_SPEC_VERSION ARG CDXGEN_VERSION @@ -25,14 +25,14 @@ RUN <= 24, so it gets its own Node installation in /opt/node-sbom and +# cdxgen requires Node >= 24, so it gets its own Node installation in /opt/node-cdxgen and # is invoked with that prepended to PATH. # -fsSL is not needed: the shared /root/.curlrc sets location, fail, silent and show-error. ARCH="${TARGETARCH/amd64/x64}" -mkdir -p /opt/node-sbom -curl "https://repo.stackable.tech/repository/packages/node/node-v${SBOM_NODEJS_VERSION}-linux-${ARCH}.tar.xz" | \ - tar --extract --xz --directory=/opt/node-sbom --strip-components=1 -PATH="/opt/node-sbom/bin:$PATH" npm install --global "@cdxgen/cdxgen@${CDXGEN_VERSION}" +mkdir -p /opt/node-cdxgen +curl "https://repo.stackable.tech/repository/packages/node/node-v${CDXGEN_NODEJS_VERSION}-linux-${ARCH}.tar.xz" | \ + tar --extract --xz --directory=/opt/node-cdxgen --strip-components=1 +PATH="/opt/node-cdxgen/bin:$PATH" npm install --global "@cdxgen/cdxgen@${CDXGEN_VERSION}" microdnf update microdnf install python3 @@ -100,7 +100,7 @@ mv hbase-assembly/target/bom.json /stackable/hbase-${NEW_VERSION}/hbase-${NEW_VE # by the maven-dependency-plugin instead of being declared as project dependencies, so the # CycloneDX Maven plugin does not cover them, see hbase_webapps_deps.py. ( - export PATH="/opt/node-sbom/bin:$PATH" + export PATH="/opt/node-cdxgen/bin:$PATH" WEBAPPS_SBOM_DIR="$(mktemp --directory)" python3 /stackable/hbase_webapps_deps.py . "${ORIGINAL_VERSION}" "${WEBAPPS_SBOM_DIR}/package.json" cd "${WEBAPPS_SBOM_DIR}" diff --git a/trino/trino/Dockerfile b/trino/trino/Dockerfile index 6acf55b15..1e234adc3 100644 --- a/trino/trino/Dockerfile +++ b/trino/trino/Dockerfile @@ -9,7 +9,7 @@ ARG RELEASE_VERSION ARG STACKABLE_USER_UID ARG TRINO_AIRLIFT_VERSION ARG TARGETARCH -ARG SBOM_NODEJS_VERSION +ARG CDXGEN_NODEJS_VERSION ARG CDXGEN_SPEC_VERSION ARG CDXGEN_VERSION @@ -24,13 +24,13 @@ RUN <= 24, which is unrelated to the Node version that the # frontend-maven-plugin downloads for the actual build, so it gets its own Node installation in -# /opt/node-sbom and is invoked with that prepended to PATH. +# /opt/node-cdxgen and is invoked with that prepended to PATH. # -fsSL is not needed: the shared /root/.curlrc sets location, fail, silent and show-error. ARCH="${TARGETARCH/amd64/x64}" -mkdir -p /opt/node-sbom -curl "https://repo.stackable.tech/repository/packages/node/node-v${SBOM_NODEJS_VERSION}-linux-${ARCH}.tar.xz" | \ - tar --extract --xz --directory=/opt/node-sbom --strip-components=1 -PATH="/opt/node-sbom/bin:$PATH" npm install --global "@cdxgen/cdxgen@${CDXGEN_VERSION}" +mkdir -p /opt/node-cdxgen +curl "https://repo.stackable.tech/repository/packages/node/node-v${CDXGEN_NODEJS_VERSION}-linux-${ARCH}.tar.xz" | \ + tar --extract --xz --directory=/opt/node-cdxgen --strip-components=1 +PATH="/opt/node-cdxgen/bin:$PATH" npm install --global "@cdxgen/cdxgen@${CDXGEN_VERSION}" microdnf update microdnf install python3 @@ -105,20 +105,15 @@ mv core/trino-server/target/bom.json /stackable/trino-server-${NEW_VERSION}/trin # so this needs revisiting when such a version is added. The paths below fail the build rather # than silently producing nothing. ( - export PATH="/opt/node-sbom/bin:$PATH" + export PATH="/opt/node-cdxgen/bin:$PATH" WEB_UI="core/trino-web-ui/src/main/resources" # cdxgen resolves the dependency tree from the lockfile, and returns nothing at all when a # project has both a package-lock.json and a yarn.lock, which webapp/src does. Copying just the # manifest and the npm lockfile into a scratch directory avoids that. - # --required-only keeps what the lockfile marks as non-dev. Note that this is wider than what - # ends up in the bundle: upstream declares packages such as happy-dom and js-yaml as runtime - # dependencies even though the bundler drops them, so they are reported here as well. That is - # the same trade-off the other frontend SBOMs make, and over-reporting is preferred over - # missing a component. + # --required-only keeps what the lockfile marks as non-dev. # --no-babel disables the usage analysis, which would otherwise mark every package that is not - # imported directly as optional and thereby drop genuine transitive runtime dependencies. It - # has nothing to look at here anyway, because only the manifest and the lockfile are copied. + # imported directly as optional and thereby drop genuine transitive runtime dependencies. # --project-version is passed because the frontends declare a placeholder version upstream, and # --project-name because cdxgen otherwise names the root component after the scratch directory. for FRONTEND in "webapp/src:trino-web-ui" "webapp-preview:trino-web-ui-preview"; do From b9be75ec6bc74bbee6685f1b23086ffc009370a6 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Mon, 24 Aug 2026 15:24:15 +0000 Subject: [PATCH 6/9] fix(hbase): Correct the comment explaining the webjar SBOM --- hbase/hbase/Dockerfile | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/hbase/hbase/Dockerfile b/hbase/hbase/Dockerfile index ffe6ff214..8bdc0f3fb 100644 --- a/hbase/hbase/Dockerfile +++ b/hbase/hbase/Dockerfile @@ -104,9 +104,11 @@ mv hbase-assembly/target/bom.json /stackable/hbase-${NEW_VERSION}/hbase-${NEW_VE WEBAPPS_SBOM_DIR="$(mktemp --directory)" python3 /stackable/hbase_webapps_deps.py . "${ORIGINAL_VERSION}" "${WEBAPPS_SBOM_DIR}/package.json" cd "${WEBAPPS_SBOM_DIR}" - # cdxgen needs a lockfile to resolve the dependency tree. The webjars are pre-built browser - # bundles that inline their dependencies, so the transitive packages are shipped as well and - # belong in the SBOM. + # cdxgen reads the components from a lockfile, so one is generated for the package.json above. + # Usually a lockfile also resolves the npm dependencies of the listed packages, while a + # webjar only ever ships the files of the library itself. Such a dependency would therefore + # show up in the SBOM without being shipped. In this case it is not a problem, because the libraries + # HBase unpacks (jquery, moment and bootstrap) have no npm dependencies. npm install --package-lock-only --no-audit --no-fund cdxgen \ --type js \ From bc710f9e6940365b077e32f9c11ff2ee221f6eda Mon Sep 17 00:00:00 2001 From: dervoeti Date: Mon, 24 Aug 2026 15:24:16 +0000 Subject: [PATCH 7/9] fix(hadoop): Name the vendored JavaScript SBOM after its component --- hadoop/hadoop/Dockerfile | 2 +- hadoop/hadoop/stackable/vendored-js/3.3.6.json | 2 +- hadoop/hadoop/stackable/vendored-js/3.4.2.json | 2 +- hadoop/hadoop/stackable/vendored-js/3.4.3.json | 2 +- hadoop/hadoop/stackable/vendored-js/3.5.0.json | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/hadoop/hadoop/Dockerfile b/hadoop/hadoop/Dockerfile index a336c1858..de701628a 100644 --- a/hadoop/hadoop/Dockerfile +++ b/hadoop/hadoop/Dockerfile @@ -127,7 +127,7 @@ mv hadoop-dist/target/bom.json /stackable/hadoop-${NEW_VERSION}/hadoop-${NEW_VER python3 /build/vendored_js.py bom \ /build/vendored-js.json \ . \ - "/stackable/hadoop-${NEW_VERSION}/hadoop-vendored-js-${NEW_VERSION}.cdx.json" \ + "/stackable/hadoop-${NEW_VERSION}/hadoop-webapps-${NEW_VERSION}.cdx.json" \ "${ORIGINAL_VERSION}" \ "${CDXGEN_SPEC_VERSION}" diff --git a/hadoop/hadoop/stackable/vendored-js/3.3.6.json b/hadoop/hadoop/stackable/vendored-js/3.3.6.json index 09e7c5a72..0e2c034ab 100644 --- a/hadoop/hadoop/stackable/vendored-js/3.3.6.json +++ b/hadoop/hadoop/stackable/vendored-js/3.3.6.json @@ -1,5 +1,5 @@ { - "name": "hadoop-hdfs-webapps", + "name": "hadoop-webapps", "scan-dirs": [ "hadoop-hdfs-project/hadoop-hdfs/src/main/webapps", "hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps", diff --git a/hadoop/hadoop/stackable/vendored-js/3.4.2.json b/hadoop/hadoop/stackable/vendored-js/3.4.2.json index 2ad1dea02..140e9fcfa 100644 --- a/hadoop/hadoop/stackable/vendored-js/3.4.2.json +++ b/hadoop/hadoop/stackable/vendored-js/3.4.2.json @@ -1,5 +1,5 @@ { - "name": "hadoop-hdfs-webapps", + "name": "hadoop-webapps", "scan-dirs": [ "hadoop-hdfs-project/hadoop-hdfs/src/main/webapps", "hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps", diff --git a/hadoop/hadoop/stackable/vendored-js/3.4.3.json b/hadoop/hadoop/stackable/vendored-js/3.4.3.json index 2ad1dea02..140e9fcfa 100644 --- a/hadoop/hadoop/stackable/vendored-js/3.4.3.json +++ b/hadoop/hadoop/stackable/vendored-js/3.4.3.json @@ -1,5 +1,5 @@ { - "name": "hadoop-hdfs-webapps", + "name": "hadoop-webapps", "scan-dirs": [ "hadoop-hdfs-project/hadoop-hdfs/src/main/webapps", "hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps", diff --git a/hadoop/hadoop/stackable/vendored-js/3.5.0.json b/hadoop/hadoop/stackable/vendored-js/3.5.0.json index c95ca12bd..9fc4ede4f 100644 --- a/hadoop/hadoop/stackable/vendored-js/3.5.0.json +++ b/hadoop/hadoop/stackable/vendored-js/3.5.0.json @@ -1,5 +1,5 @@ { - "name": "hadoop-hdfs-webapps", + "name": "hadoop-webapps", "scan-dirs": [ "hadoop-hdfs-project/hadoop-hdfs/src/main/webapps", "hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/webapps", From 4afe0694a3c1276ec38243b478b743299aae79d5 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Mon, 24 Aug 2026 15:24:17 +0000 Subject: [PATCH 8/9] chore: Drop the redundant hashes initialization in vendored_js.py --- shared/sbom/vendored_js.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/shared/sbom/vendored_js.py b/shared/sbom/vendored_js.py index 6b7f9e5e0..065ad8531 100755 --- a/shared/sbom/vendored_js.py +++ b/shared/sbom/vendored_js.py @@ -116,8 +116,6 @@ def add(entry, location, sha256): component["purl"] = purl if entry.get("license"): component["licenses"] = [{"expression": entry["license"]}] - if sha256: - component["hashes"] = [] component["evidence"] = {"occurrences": []} components[key] = component From 3d7adafc116ea05fd939745267271ce96319a7e4 Mon Sep 17 00:00:00 2001 From: dervoeti Date: Mon, 24 Aug 2026 19:54:47 +0000 Subject: [PATCH 9/9] chore: Format the SBOM helper scripts with ruff --- hbase/hbase/stackable/hbase_webapps_deps.py | 32 ++++++-- shared/sbom/identify_js.py | 83 ++++++++++++++++----- shared/sbom/vendored_js.py | 43 ++++++++--- 3 files changed, 123 insertions(+), 35 deletions(-) diff --git a/hbase/hbase/stackable/hbase_webapps_deps.py b/hbase/hbase/stackable/hbase_webapps_deps.py index ac51bcf04..74e69a0da 100755 --- a/hbase/hbase/stackable/hbase_webapps_deps.py +++ b/hbase/hbase/stackable/hbase_webapps_deps.py @@ -53,11 +53,17 @@ def webjars(server_pom, versions): if fields.get("groupId") != "org.webjars": continue if not fields.get("artifactId") or not fields.get("version"): - raise SystemExit(f" without an artifactId or version: {fields}") + raise SystemExit( + f" without an artifactId or version: {fields}" + ) - version = PROPERTY.sub(lambda match: versions.get(match[1], match[0]), fields["version"]) + version = PROPERTY.sub( + lambda match: versions.get(match[1], match[0]), fields["version"] + ) if "${" in version: - raise SystemExit(f"Cannot resolve the version of {fields['artifactId']} from the root pom: {version}") + raise SystemExit( + f"Cannot resolve the version of {fields['artifactId']} from the root pom: {version}" + ) # bootstrap is unpacked twice, once for its JavaScript and once for its CSS. dependencies[fields["artifactId"]] = version @@ -65,14 +71,21 @@ def webjars(server_pom, versions): # Guard against upstream restructuring the pom, which would otherwise silently produce an SBOM # without any components. if not dependencies: - raise SystemExit("No org.webjars found in hbase-server/pom.xml, did the pom layout change?") + raise SystemExit( + "No org.webjars found in hbase-server/pom.xml, did the pom layout change?" + ) return dependencies def main(): - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) parser.add_argument("source_root", type=Path, help="the HBase source tree") - parser.add_argument("version", help="the HBase version, used as the version of the generated package") + parser.add_argument( + "version", + help="the HBase version, used as the version of the generated package", + ) parser.add_argument("output", type=Path, help="the package.json to write") arguments = parser.parse_args() @@ -80,7 +93,12 @@ def main(): ElementTree.parse(arguments.source_root / "hbase-server/pom.xml"), properties(ElementTree.parse(arguments.source_root / "pom.xml")), ) - package = {"name": "hbase-webapps", "version": arguments.version, "private": True, "dependencies": dependencies} + package = { + "name": "hbase-webapps", + "version": arguments.version, + "private": True, + "dependencies": dependencies, + } arguments.output.write_text(json.dumps(package, indent=2) + "\n") summary = ", ".join(f"{name}@{version}" for name, version in dependencies.items()) diff --git a/shared/sbom/identify_js.py b/shared/sbom/identify_js.py index e0d5b87ec..2c9c0c1c3 100755 --- a/shared/sbom/identify_js.py +++ b/shared/sbom/identify_js.py @@ -46,7 +46,11 @@ from urllib.error import URLError from urllib.request import urlopen -CACHE = Path(os.environ.get("IDENTIFY_JS_CACHE", Path(tempfile.gettempdir()) / "stackable-identify-js-cache")) +CACHE = Path( + os.environ.get( + "IDENTIFY_JS_CACHE", Path(tempfile.gettempdir()) / "stackable-identify-js-cache" + ) +) # Version strings are written in every conceivable way, so cast a wide net over the top of the file # and let the caller judge. Matching on bytes keeps minified files with odd encodings readable. @@ -56,7 +60,9 @@ # Banners such as "// https://d3js.org Version 4.1.0." separate with a space. All three # components are required here, otherwise every "Apache License, Version 2.0" header matches. re.compile(rb"\bversion\s+v?([0-9]+\.[0-9]+\.[0-9][\w.-]*)", re.IGNORECASE), - re.compile(rb"^/\*!?\s*([A-Za-z][\w.\- ]*?)\s+v?([0-9]+\.[0-9][\w.-]*)", re.MULTILINE), + re.compile( + rb"^/\*!?\s*([A-Za-z][\w.\- ]*?)\s+v?([0-9]+\.[0-9][\w.-]*)", re.MULTILINE + ), re.compile(rb"\bv([0-9]+\.[0-9]+\.[0-9][\w.-]*)"), ] @@ -77,7 +83,11 @@ def version_hints(contents): for pattern in HINTS: for match in pattern.finditer(contents[:3000]): # Rstrip because a version at the end of a sentence swallows the full stop. - hint = b" ".join(group for group in match.groups() if group).decode("latin1").rstrip(".-") + hint = ( + b" ".join(group for group in match.groups() if group) + .decode("latin1") + .rstrip(".-") + ) if hint not in hints: hints.append(hint) return hints @@ -88,16 +98,25 @@ def published_versions(package, prefix): order, which is not always ascending, and the advice to record the lowest match depends on the order being right.""" # The scope separator has to stay encoded, otherwise the registry sees two path segments. - with urlopen(f"https://registry.npmjs.org/{package.replace('/', '%2f')}") as response: + with urlopen( + f"https://registry.npmjs.org/{package.replace('/', '%2f')}" + ) as response: metadata = json.load(response) releases = [ (version, release["dist"]["tarball"]) for version, release in metadata.get("versions", {}).items() # A prerelease is never what a product vendored. - if "-" not in version and version.startswith(prefix) and release.get("dist", {}).get("tarball") + if "-" not in version + and version.startswith(prefix) + and release.get("dist", {}).get("tarball") ] - return sorted(releases, key=lambda release: [int(part) if part.isdigit() else 0 for part in release[0].split(".")]) + return sorted( + releases, + key=lambda release: [ + int(part) if part.isdigit() else 0 for part in release[0].split(".") + ], + ) def tarball_hashes(package, version, url): @@ -113,7 +132,9 @@ def tarball_hashes(package, version, url): with tarfile.open(archive_path) as archive: # Every member is below a "package/" directory that is of no interest here. return { - member.name.split("/", 1)[-1]: digests(archive.extractfile(member).read()) + member.name.split("/", 1)[-1]: digests( + archive.extractfile(member).read() + ) for member in archive if member.isfile() } @@ -127,7 +148,9 @@ def inspect(source_root, directories): for path in sorted((source_root / directory).rglob("*.js")): contents = path.read_bytes() hints = " | ".join(version_hints(contents)[:3]) or "-" - print("\t".join([str(path.relative_to(source_root)), sha256(contents), hints])) + print( + "\t".join([str(path.relative_to(source_root)), sha256(contents), hints]) + ) def identify(target, packages, prefix): @@ -143,40 +166,62 @@ def identify(target, packages, prefix): continue print(f"{package}: checking {len(releases)} version(s)") for version, url in releases: - for name, (digest, stripped) in tarball_hashes(package, version, url).items(): + for name, (digest, stripped) in tarball_hashes( + package, version, url + ).items(): if digest == wanted: matches.append((package, version, "identical")) print(f" MATCH {package}@{version} {name}") elif stripped == wanted_stripped: matches.append((package, version, "whitespace")) - print(f" MATCH {package}@{version} {name} (differs only in surrounding whitespace)") + print( + f" MATCH {package}@{version} {name} (differs only in surrounding whitespace)" + ) if not matches: - print("\nNo match. Fall back to the version the file states and note that in the manifest.") + print( + "\nNo match. Fall back to the version the file states and note that in the manifest." + ) return if any(kind == "whitespace" for _, _, kind in matches): - print("\nThe code is identical, only the surrounding whitespace differs, so the release applies.") - print("Record it and say in the manifest that the copy carries extra whitespace.") + print( + "\nThe code is identical, only the surrounding whitespace differs, so the release applies." + ) + print( + "Record it and say in the manifest that the copy carries extra whitespace." + ) if len(matches) > 1: package, version, _ = matches[0] - print(f"\nThe file is the same in {len(matches)} releases, so record the lowest,") + print( + f"\nThe file is the same in {len(matches)} releases, so record the lowest," + ) print(f"{package}@{version}, and note the ambiguity in the manifest.") def main(): - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) commands = parser.add_subparsers(dest="command", required=True) - inspect_command = commands.add_parser("inspect", help="list the JavaScript files below the given directories") + inspect_command = commands.add_parser( + "inspect", help="list the JavaScript files below the given directories" + ) inspect_command.add_argument("source_root", type=Path) inspect_command.add_argument("directories", nargs="+") - identify_command = commands.add_parser("identify", help="find the npm release a file came from") + identify_command = commands.add_parser( + "identify", help="find the npm release a file came from" + ) identify_command.add_argument("file", type=Path) - identify_command.add_argument("packages", nargs="+", help="npm packages the file might come from") - identify_command.add_argument("--prefix", default="", help='only check versions starting with this, e.g. "4."') + identify_command.add_argument( + "packages", nargs="+", help="npm packages the file might come from" + ) + identify_command.add_argument( + "--prefix", default="", help='only check versions starting with this, e.g. "4."' + ) arguments = parser.parse_args() if arguments.command == "inspect": diff --git a/shared/sbom/vendored_js.py b/shared/sbom/vendored_js.py index 065ad8531..2dc943e63 100755 --- a/shared/sbom/vendored_js.py +++ b/shared/sbom/vendored_js.py @@ -53,7 +53,9 @@ def verify(manifest, source_root): for library in manifest["libraries"]: if library["file"] in listed or library["file"] in own: - problems.append(f"DUPLICATE {library['file']}\n Listed more than once in the manifest.") + problems.append( + f"DUPLICATE {library['file']}\n Listed more than once in the manifest." + ) listed[library["file"]] = library found = scan(manifest, source_root) @@ -76,9 +78,13 @@ def verify(manifest, source_root): for file in listed: if file not in found: - problems.append(f"GONE {file}\n Listed in the manifest but no longer in the source tree.") + problems.append( + f"GONE {file}\n Listed in the manifest but no longer in the source tree." + ) for file in sorted(own - found.keys()): - problems.append(f'GONE {file}\n Listed in "own" but no longer in the source tree.') + problems.append( + f'GONE {file}\n Listed in "own" but no longer in the source tree.' + ) return found, problems @@ -121,7 +127,9 @@ def add(entry, location, sha256): component = components[key] if sha256: - component.setdefault("hashes", []).append({"alg": "SHA-256", "content": sha256}) + component.setdefault("hashes", []).append( + {"alg": "SHA-256", "content": sha256} + ) component["evidence"]["occurrences"].append({"location": location}) for library in manifest["libraries"]: @@ -145,18 +153,32 @@ def add(entry, location, sha256): "name": manifest["name"], "version": component_version, }, - "tools": {"components": [{"type": "application", "name": "vendored_js.py", "group": "tech.stackable"}]}, + "tools": { + "components": [ + { + "type": "application", + "name": "vendored_js.py", + "group": "tech.stackable", + } + ] + }, }, "components": list(components.values()), } def main(): - parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) commands = parser.add_subparsers(dest="command", required=True) - check_command = commands.add_parser("check", help="report every mismatch between the manifest and the source tree") - bom_command = commands.add_parser("bom", help="write the SBOM the manifest describes") + check_command = commands.add_parser( + "check", help="report every mismatch between the manifest and the source tree" + ) + bom_command = commands.add_parser( + "bom", help="write the SBOM the manifest describes" + ) for command in (check_command, bom_command): command.add_argument("manifest", type=Path) command.add_argument("source_root", type=Path) @@ -170,7 +192,10 @@ def main(): # Never generate an SBOM that we know to be wrong, so this also gates "bom". found, problems = verify(manifest, arguments.source_root) if problems: - print(f"{arguments.manifest} does not match the source tree ({len(problems)} problem(s)):\n", file=sys.stderr) + print( + f"{arguments.manifest} does not match the source tree ({len(problems)} problem(s)):\n", + file=sys.stderr, + ) print("\n".join(problems), file=sys.stderr) raise SystemExit(1)