From 27f05b768ea91dfb35578e728d0a73e022bcef29 Mon Sep 17 00:00:00 2001 From: Eliott B <47679741+eliottness@users.noreply.github.com> Date: Fri, 23 Oct 2026 19:33:12 +0200 Subject: [PATCH 1/3] Stop trusting compromised eMudhra roots in CI images Upgrade the default CI image to alpine 3.24.3, use a plain alpine image for the no-op reliability job, and patch the async-profiler-build base Dockerfiles (CA bundle, Node 22.23.4, botocore bundle, JDK cacerts) so the next REBUILD_IMAGES run publishes clean images (IR-62343). Jira: APPSEC-71294 --- .gitlab-ci.yml | 4 ++-- .gitlab/base/Dockerfile | 26 +++++++++++++++++++++++--- .gitlab/base/Dockerfile.musl | 8 +++++++- .gitlab/base/el7/Dockerfile | 9 +++++++++ .gitlab/base/el8/Dockerfile | 4 ++++ .gitlab/benchmarks/.gitlab-ci.yml | 2 +- .gitlab/benchmarks/child.gitlab-ci.yml | 2 +- .gitlab/build-deploy/.gitlab-ci.yml | 2 +- 8 files changed, 48 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 69bde4817c..5c17a72b3b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ # Copyright 2026, Datadog, Inc. -image: alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 +image: alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a variables: REGISTRY: registry.ddbuild.io @@ -196,7 +196,7 @@ generate-reliability-child-pipeline: else cat > generated-reliability.yml << 'NOOP' skip-reliability: - image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:newest + image: alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a tags: ["arch:amd64"] script: - echo "Label test:reliability not set — skipping" diff --git a/.gitlab/base/Dockerfile b/.gitlab/base/Dockerfile index e3ec03c998..3a1b5d06c9 100644 --- a/.gitlab/base/Dockerfile +++ b/.gitlab/base/Dockerfile @@ -9,6 +9,8 @@ FROM ${BASE_IMAGE} as base ARG CI_JOB_TOKEN WORKDIR /root +RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y ca-certificates \ + && update-ca-certificates -f RUN mkdir -p /usr/share/man/man1 # https://github.com/debuerreotype/docker-debian-artifacts/issues/24 # Alpine/musl images build from Dockerfile.musl, so this file installs only the # Debian toolchain. @@ -19,13 +21,31 @@ RUN mkdir -p /usr/share/man/man1 # https://github.com/debuerreotype/docker-debia # libclang-rt-dev is the unversioned metapackage: it provides # sanitizer/asan_interface.h from whichever clang runtime matches this release. RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ - curl git moreutils awscli amazon-ecr-credential-helper gnupg2 npm build-essential \ + curl git moreutils awscli amazon-ecr-credential-helper gnupg2 build-essential \ wget bsdmainutils clang jq zip unzip \ - cmake libgtest-dev libgmock-dev libc6-dbg llvm libclang-rt-dev + cmake libgtest-dev libgmock-dev libc6-dbg llvm libclang-rt-dev \ + && ln -sf /etc/ssl/certs/ca-certificates.crt /usr/lib/python3/dist-packages/awscli/botocore/cacert.pem +# Debian's packaged Node includes obsolete roots in both libnode and its headers. +# Use the fixed Node 22 release and verify the upstream archive before unpacking. +ARG TARGETARCH +RUN case "${TARGETARCH}" in \ + amd64) node_arch=x64; node_sha=b8492284c76a54baf58d61668dd0e41ee0fc98adabf9503d30f05d08f55b4406 ;; \ + arm64) node_arch=arm64; node_sha=f7a1e486dbc8b286847815b477d4fb5e9fb963ed4489c3e1588853d685f589f9 ;; \ + *) exit 1 ;; \ + esac \ + && curl -fsSLo /tmp/node.tar.gz "https://nodejs.org/dist/v22.23.4/node-v22.23.4-linux-${node_arch}.tar.gz" \ + && echo "${node_sha} /tmp/node.tar.gz" | sha256sum -c - \ + && tar -xzf /tmp/node.tar.gz -C /usr/local --strip-components=1 \ + && rm /tmp/node.tar.gz # Install JDK 21 and Maven via SDKMAN (glibc image only; musl uses Dockerfile.musl). RUN curl -s "https://get.sdkman.io" | bash && \ bash -c "source /root/.sdkman/bin/sdkman-init.sh && sdk install java 21.0.3-tem && sdk install maven" +# SDKMAN does not yet publish the fixed Temurin update; remove the two roots +# from the bundled JDK trust store until that version is available. +RUN /root/.sdkman/candidates/java/current/bin/keytool -list -v -cacerts -storepass changeit \ + | awk -F ': ' '/^Alias name:/{alias=$2} /Owner:.*(emSign ECC Root CA - G3|AAA Certificate Services)/{print alias}' \ + | while IFS= read -r alias; do /root/.sdkman/candidates/java/current/bin/keytool -delete -cacerts -storepass changeit -alias "$alias"; done ENV JAVA_HOME=/root/.sdkman/candidates/java/current ENV PATH="/root/.sdkman/candidates/java/current/bin:/root/.sdkman/candidates/maven/current/bin:${PATH}" RUN npm install -g --save-dev @datadog/datadog-ci -RUN rm -rf "/var/lib/apt/lists/*" \ No newline at end of file +RUN rm -rf "/var/lib/apt/lists/*" diff --git a/.gitlab/base/Dockerfile.musl b/.gitlab/base/Dockerfile.musl index 1bbc9ce5b9..36982fe759 100644 --- a/.gitlab/base/Dockerfile.musl +++ b/.gitlab/base/Dockerfile.musl @@ -3,5 +3,11 @@ FROM ${BASE_IMAGE} as base ARG CI_JOB_TOKEN WORKDIR /root -RUN apk update && apk add curl git moreutils aws-cli docker-credential-ecr-login gnupg alpine-sdk build-base wget npm hexdump linux-headers clang compiler-rt bash jq zip unzip +RUN apk upgrade --no-cache ca-certificates ca-certificates-bundle +RUN apk update && apk add curl git moreutils aws-cli docker-credential-ecr-login gnupg alpine-sdk build-base wget npm hexdump linux-headers clang compiler-rt bash jq zip unzip \ + && ln -sf /etc/ssl/certs/ca-certificates.crt /usr/lib/python3.12/site-packages/awscli/botocore/cacert.pem +# The JDK comes from BASE_IMAGE; remove the two distrusted anchors from its cacerts. +RUN keytool -list -v -cacerts -storepass changeit | awk -F ': ' \ + '/^Alias name:/{alias=$2} /Owner:.*(emSign ECC Root CA - G3|AAA Certificate Services)/{print alias}' \ + | while IFS= read -r alias; do keytool -delete -cacerts -storepass changeit -alias "$alias"; done RUN npm install -g --save-dev @datadog/datadog-ci diff --git a/.gitlab/base/el7/Dockerfile b/.gitlab/base/el7/Dockerfile index 0dafec4fa9..6f3e8b6579 100644 --- a/.gitlab/base/el7/Dockerfile +++ b/.gitlab/base/el7/Dockerfile @@ -23,14 +23,23 @@ ARG BASE_IMAGE=oraclelinux:7 FROM ${BASE_IMAGE} as base WORKDIR /root +RUN yum -y update ca-certificates \ + && ln -sf /usr/lib64/pkcs11/p11-kit-trust.so /usr/lib64/nss/libnssckbi.so # python3 (3.6.8 on this base repo -- no newer python3/SCL repo exists for # Oracle Linux 7) runs the quarantine classifier (.github/scripts/ # run_tests_with_retry.sh, quarantine.py, flake_report.py) that # functional-tests.sh wraps its test run in. RUN yum -y install git make zip unzip which wget curl jq gcc-c++ python3 \ && yum -y clean all +# OL7 has no fixed ca-certificates package. Preserve every other p11-kit +# object and its purpose restrictions while removing the compromised roots. +RUN python3 -c 'from pathlib import Path; p=Path("/usr/share/pki/ca-trust-source/ca-bundle.trust.p11-kit"); mark=b"[p11-kit-object-v1]"; parts=p.read_bytes().split(mark); bad=(b"emSign ECC Root CA - G3", b"AAA Certificate Services"); drop=lambda part: any(name in part for name in bad); assert all(any(name in part for part in parts[1:]) for name in bad); p.write_bytes(parts[0] + b"".join(mark + part for part in parts[1:] if not drop(part)))' \ + && update-ca-trust extract RUN curl -s "https://get.sdkman.io" | bash && \ bash -c "source /root/.sdkman/bin/sdkman-init.sh && sdk install java 21.0.3-tem" +RUN /root/.sdkman/candidates/java/current/bin/keytool -list -v -cacerts -storepass changeit \ + | awk -F ': ' '/^Alias name:/{alias=$2} /Owner:.*(emSign ECC Root CA - G3|AAA Certificate Services)/{print alias}' \ + | while IFS= read -r alias; do /root/.sdkman/candidates/java/current/bin/keytool -delete -cacerts -storepass changeit -alias "$alias"; done ENV JAVA_HOME=/root/.sdkman/candidates/java/current ENV PATH="/root/.sdkman/candidates/java/current/bin:${PATH}" diff --git a/.gitlab/base/el8/Dockerfile b/.gitlab/base/el8/Dockerfile index 9bd7b8b2b8..90c98738bb 100644 --- a/.gitlab/base/el8/Dockerfile +++ b/.gitlab/base/el8/Dockerfile @@ -17,6 +17,7 @@ FROM ${BASE_IMAGE} as base ARG CI_JOB_TOKEN WORKDIR /root +RUN dnf -y update ca-certificates && update-ca-trust extract # gcc-toolset-14 carries its own libstdc++.a/libgcc.a, which is what # -static-libstdc++ resolves against; no separate *-static package is needed. RUN dnf -y install gcc-toolset-14 \ @@ -33,5 +34,8 @@ ENV PATH="/opt/rh/gcc-toolset-14/root/usr/bin:${PATH}" \ # current candidate (.gitlab/scripts/build.sh). RUN curl -s "https://get.sdkman.io" | bash && \ bash -c "source /root/.sdkman/bin/sdkman-init.sh && sdk install java 21.0.3-tem && sdk install maven" +RUN /root/.sdkman/candidates/java/current/bin/keytool -list -v -cacerts -storepass changeit \ + | awk -F ': ' '/^Alias name:/{alias=$2} /Owner:.*(emSign ECC Root CA - G3|AAA Certificate Services)/{print alias}' \ + | while IFS= read -r alias; do /root/.sdkman/candidates/java/current/bin/keytool -delete -cacerts -storepass changeit -alias "$alias"; done ENV JAVA_HOME=/root/.sdkman/candidates/java/current ENV PATH="/root/.sdkman/candidates/java/current/bin:/root/.sdkman/candidates/maven/current/bin:${PATH}" diff --git a/.gitlab/benchmarks/.gitlab-ci.yml b/.gitlab/benchmarks/.gitlab-ci.yml index 3e9ffe0fa2..4a22f93225 100644 --- a/.gitlab/benchmarks/.gitlab-ci.yml +++ b/.gitlab/benchmarks/.gitlab-ci.yml @@ -65,7 +65,7 @@ generate-benchmarks-child-pipeline: echo "Branch is below another PR in its stack — benchmarks skipped" cat > generated-benchmarks.yml << 'NOOP' skip-benchmarks: - image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:newest + image: alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a tags: ["arch:amd64"] script: - echo "Branch is not the top of its PR stack — skipping benchmarks" diff --git a/.gitlab/benchmarks/child.gitlab-ci.yml b/.gitlab/benchmarks/child.gitlab-ci.yml index d87065892f..126e7351fb 100644 --- a/.gitlab/benchmarks/child.gitlab-ci.yml +++ b/.gitlab/benchmarks/child.gitlab-ci.yml @@ -23,7 +23,7 @@ stages: # hop: without this guard the BP pipeline would consume empty versions. benchmarks-inputs-guard: stage: benchmarks - image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:newest + image: alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a tags: ["arch:amd64"] needs: [] rules: diff --git a/.gitlab/build-deploy/.gitlab-ci.yml b/.gitlab/build-deploy/.gitlab-ci.yml index 3b2df7626d..cb5525390f 100644 --- a/.gitlab/build-deploy/.gitlab-ci.yml +++ b/.gitlab/build-deploy/.gitlab-ci.yml @@ -1,4 +1,4 @@ -image: alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 +image: alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a variables: # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/2040944172 From 2c6ec878f6b2082fb5b4da490a0ecb6cd07fafee Mon Sep 17 00:00:00 2001 From: Eliott B <47679741+eliottness@users.noreply.github.com> Date: Mon, 26 Oct 2026 13:28:24 +0100 Subject: [PATCH 2/3] Add copyright headers and keep the benchmark guard jobs on their original image Add the 2026 Datadog copyright line to the five base and build-deploy files this PR modifies (AGENTS.md: add a copyright line to files missing one). Restore .gitlab/benchmarks/.gitlab-ci.yml and child.gitlab-ci.yml to main: the merge-queue pipeline for this PR failed in benchmarks-inputs-guard, the job whose image this PR had moved from the Datadog-built benchmarking-platform-tools-ubuntu image to Docker Hub alpine. Those jobs only echo variables, so they go back to their original image; that image is now a follow-up (rebuild at source in DataDog/images). Jira: APPSEC-71294 (parent APPSEC-71273) --- .gitlab/base/Dockerfile | 2 ++ .gitlab/base/Dockerfile.musl | 2 ++ .gitlab/base/el7/Dockerfile | 2 ++ .gitlab/base/el8/Dockerfile | 2 ++ .gitlab/benchmarks/.gitlab-ci.yml | 2 +- .gitlab/benchmarks/child.gitlab-ci.yml | 2 +- .gitlab/build-deploy/.gitlab-ci.yml | 2 ++ 7 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.gitlab/base/Dockerfile b/.gitlab/base/Dockerfile index 3a1b5d06c9..c672472394 100644 --- a/.gitlab/base/Dockerfile +++ b/.gitlab/base/Dockerfile @@ -1,3 +1,5 @@ +# Copyright 2026, Datadog, Inc. + # CI always passes BASE_IMAGE explicitly (OPENJDK_BASE_IMAGE / # OPENJDK_BASE_IMAGE_ARM64), both of which are this same digest: the shipped # binaries are built elsewhere (.gitlab/base/el8/Dockerfile), so neither arch diff --git a/.gitlab/base/Dockerfile.musl b/.gitlab/base/Dockerfile.musl index 36982fe759..b2c5c9ddff 100644 --- a/.gitlab/base/Dockerfile.musl +++ b/.gitlab/base/Dockerfile.musl @@ -1,3 +1,5 @@ +# Copyright 2026, Datadog, Inc. + ARG BASE_IMAGE=eclipse-temurin:21-jdk-alpine@sha256:c98f0d2e171c898bf896dc4166815d28a56d428e218190a1f35cdc7d82efd61f FROM ${BASE_IMAGE} as base ARG CI_JOB_TOKEN diff --git a/.gitlab/base/el7/Dockerfile b/.gitlab/base/el7/Dockerfile index 6f3e8b6579..1a160ba3b9 100644 --- a/.gitlab/base/el7/Dockerfile +++ b/.gitlab/base/el7/Dockerfile @@ -1,3 +1,5 @@ +# Copyright 2026, Datadog, Inc. + # Runtime-only image for exercising the shipped x64 binary on its actual # glibc floor (2.17). Oracle Linux 7 is the best-maintained EL7 derivative — # live CDN, current package levels, no ELS subscription needed — see PR #790. diff --git a/.gitlab/base/el8/Dockerfile b/.gitlab/base/el8/Dockerfile index 90c98738bb..7b8725497c 100644 --- a/.gitlab/base/el8/Dockerfile +++ b/.gitlab/base/el8/Dockerfile @@ -1,3 +1,5 @@ +# Copyright 2026, Datadog, Inc. + # Build image for the shipped glibc binaries, both architectures. # # AlmaLinux 8 (glibc 2.28) with gcc-toolset-14. Release builds link diff --git a/.gitlab/benchmarks/.gitlab-ci.yml b/.gitlab/benchmarks/.gitlab-ci.yml index 4a22f93225..3e9ffe0fa2 100644 --- a/.gitlab/benchmarks/.gitlab-ci.yml +++ b/.gitlab/benchmarks/.gitlab-ci.yml @@ -65,7 +65,7 @@ generate-benchmarks-child-pipeline: echo "Branch is below another PR in its stack — benchmarks skipped" cat > generated-benchmarks.yml << 'NOOP' skip-benchmarks: - image: alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a + image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:newest tags: ["arch:amd64"] script: - echo "Branch is not the top of its PR stack — skipping benchmarks" diff --git a/.gitlab/benchmarks/child.gitlab-ci.yml b/.gitlab/benchmarks/child.gitlab-ci.yml index 126e7351fb..d87065892f 100644 --- a/.gitlab/benchmarks/child.gitlab-ci.yml +++ b/.gitlab/benchmarks/child.gitlab-ci.yml @@ -23,7 +23,7 @@ stages: # hop: without this guard the BP pipeline would consume empty versions. benchmarks-inputs-guard: stage: benchmarks - image: alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a + image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:newest tags: ["arch:amd64"] needs: [] rules: diff --git a/.gitlab/build-deploy/.gitlab-ci.yml b/.gitlab/build-deploy/.gitlab-ci.yml index cb5525390f..6e02ed9f49 100644 --- a/.gitlab/build-deploy/.gitlab-ci.yml +++ b/.gitlab/build-deploy/.gitlab-ci.yml @@ -1,3 +1,5 @@ +# Copyright 2026, Datadog, Inc. + image: alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a variables: From 44cbb26105a659417942b760c66336716db47c54 Mon Sep 17 00:00:00 2001 From: Eliott B <47679741+eliottness@users.noreply.github.com> Date: Mon, 26 Oct 2026 16:39:27 +0100 Subject: [PATCH 3/3] Keep the GitLab CI config on images from approved registries The merge-queue pipelines show that Datadog's GitLab runners only admit images from approved registries: the first attempt failed with "ValidatingAdmissionPolicy 'third-party-registry' ... Disallowed image(s): alpine:3.24.3@sha256:be2656061b53" (job 2174477532), and the second failed in the reliability child pipeline, whose skip job this PR had also moved to Docker Hub alpine. Restore .gitlab-ci.yml and .gitlab/build-deploy/.gitlab-ci.yml to main's version. Moving the default image to Alpine 3.24 is a follow-up: it needs a mirrored alpine in registry.ddbuild.io first. Jira: APPSEC-71294 (parent APPSEC-71273) --- .gitlab-ci.yml | 6 ++---- .gitlab/build-deploy/.gitlab-ci.yml | 4 +--- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9a3db30a8d..3c57bbccc1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,4 @@ -# Copyright 2026, Datadog, Inc. - -image: alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a +image: alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 variables: REGISTRY: registry.ddbuild.io @@ -193,7 +191,7 @@ generate-reliability-child-pipeline: else cat > generated-reliability.yml << 'NOOP' skip-reliability: - image: alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a + image: registry.ddbuild.io/images/benchmarking-platform-tools-ubuntu:newest tags: ["arch:amd64"] script: - echo "Label test:reliability not set — skipping" diff --git a/.gitlab/build-deploy/.gitlab-ci.yml b/.gitlab/build-deploy/.gitlab-ci.yml index 6e02ed9f49..3b2df7626d 100644 --- a/.gitlab/build-deploy/.gitlab-ci.yml +++ b/.gitlab/build-deploy/.gitlab-ci.yml @@ -1,6 +1,4 @@ -# Copyright 2026, Datadog, Inc. - -image: alpine:3.24.3@sha256:be2656061b536c1883c4034790e21d0a09c5aaaf51968b89368a3c0371ea2c9a +image: alpine:3.22.1@sha256:4bcff63911fcb4448bd4fdacec207030997caf25e9bea4045fa6c8c44de311d1 variables: # Generated by https://gitlab.ddbuild.io/DataDog/java-profiler/-/jobs/2040944172