From e82966521b7ba9bcc920e9f0cce5e195a6ca212d Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 7 Sep 2026 02:44:44 -0700 Subject: [PATCH 1/5] feat(images): add OCI source metadata --- .github/workflows/gateway-release.yml | 32 +++++++++++++++++--- .github/workflows/kms-release.yml | 32 +++++++++++++++++--- .github/workflows/verifier-release.yml | 31 ++++++++++++++++--- dstack/build/shared/build-lib.sh | 4 +++ dstack/gateway/dstack-app/builder/Dockerfile | 25 +++++++++++++++ dstack/kms/dstack-app/builder/Dockerfile | 25 +++++++++++++++ dstack/verifier/builder/Dockerfile | 25 +++++++++++++++ 7 files changed, 160 insertions(+), 14 deletions(-) diff --git a/.github/workflows/gateway-release.yml b/.github/workflows/gateway-release.yml index 07d12796f..26bc5c27d 100644 --- a/.github/workflows/gateway-release.yml +++ b/.github/workflows/gateway-release.yml @@ -22,12 +22,21 @@ jobs: - name: Checkout repository uses: actions/checkout@v5 - - name: Parse version from tag + - name: Parse and check version run: | - # Extract version from tag (e.g., gateway-v1.2.3 -> 1.2.3) - VERSION=${GITHUB_REF#refs/tags/gateway-v} - echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "Parsed version: $VERSION" + case "${GITHUB_REF}" in + refs/tags/gateway-v*) VERSION=${GITHUB_REF#refs/tags/gateway-v} ;; + *) + echo "This release workflow must run on a gateway-v* tag; got ${GITHUB_REF}." >&2 + exit 1 + ;; + esac + if [ -z "${VERSION}" ]; then + echo "The release version is empty." >&2 + exit 1 + fi + echo "VERSION=${VERSION}" >> "$GITHUB_ENV" + echo "Parsed version: ${VERSION}" - name: Log in to Docker Hub uses: docker/login-action@v3 @@ -58,7 +67,20 @@ jobs: build-shared=dstack/build/shared build-args: | DSTACK_REV=${{ env.GIT_REV }} + IMAGE_VERSION=${{ env.VERSION }} + IMAGE_SOURCE_URL=${{ github.server_url }}/${{ github.repository }} SOURCE_DATE_EPOCH=${{ env.TIMESTAMP }} + annotations: | + manifest:org.opencontainers.image.title=dstack-gateway + manifest:org.opencontainers.image.description=Gateway service for dstack confidential applications + manifest:org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + manifest:org.opencontainers.image.revision=${{ env.GIT_REV }} + manifest:org.opencontainers.image.version=${{ env.VERSION }} + manifest:org.opencontainers.image.url=${{ github.server_url }}/${{ github.repository }}/tree/${{ env.GIT_REV }}/dstack/gateway + manifest:org.opencontainers.image.documentation=${{ github.server_url }}/${{ github.repository }}/blob/${{ env.GIT_REV }}/dstack/gateway/dstack-app/builder/README.md + manifest:org.opencontainers.image.licenses=Apache-2.0 + manifest:org.opencontainers.image.base.name=docker.io/library/debian:bookworm + manifest:org.opencontainers.image.base.digest=sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe - name: Generate artifact attestation uses: actions/attest-build-provenance@v1 diff --git a/.github/workflows/kms-release.yml b/.github/workflows/kms-release.yml index 0b33c5f88..c44f1de44 100644 --- a/.github/workflows/kms-release.yml +++ b/.github/workflows/kms-release.yml @@ -29,12 +29,21 @@ jobs: dstack/kms/auth-eth/lib/openzeppelin-contracts-upgradeable \ dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades - - name: Parse version from tag + - name: Parse and check version run: | - # Extract version from tag (e.g., kms-v1.2.3 -> 1.2.3) - VERSION=${GITHUB_REF#refs/tags/kms-v} - echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "Parsed version: $VERSION" + case "${GITHUB_REF}" in + refs/tags/kms-v*) VERSION=${GITHUB_REF#refs/tags/kms-v} ;; + *) + echo "This release workflow must run on a kms-v* tag; got ${GITHUB_REF}." >&2 + exit 1 + ;; + esac + if [ -z "${VERSION}" ]; then + echo "The release version is empty." >&2 + exit 1 + fi + echo "VERSION=${VERSION}" >> "$GITHUB_ENV" + echo "Parsed version: ${VERSION}" - name: Log in to Docker Hub uses: docker/login-action@v3 @@ -65,8 +74,21 @@ jobs: build-shared=dstack/build/shared build-args: | DSTACK_REV=${{ env.GIT_REV }} + IMAGE_VERSION=${{ env.VERSION }} + IMAGE_SOURCE_URL=${{ github.server_url }}/${{ github.repository }} DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }} SOURCE_DATE_EPOCH=${{ env.TIMESTAMP }} + annotations: | + manifest:org.opencontainers.image.title=dstack-kms + manifest:org.opencontainers.image.description=Key management service for dstack confidential applications + manifest:org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + manifest:org.opencontainers.image.revision=${{ env.GIT_REV }} + manifest:org.opencontainers.image.version=${{ env.VERSION }} + manifest:org.opencontainers.image.url=${{ github.server_url }}/${{ github.repository }}/tree/${{ env.GIT_REV }}/dstack/kms + manifest:org.opencontainers.image.documentation=${{ github.server_url }}/${{ github.repository }}/blob/${{ env.GIT_REV }}/dstack/kms/README.md + manifest:org.opencontainers.image.licenses=Apache-2.0 + manifest:org.opencontainers.image.base.name=docker.io/library/debian:bookworm + manifest:org.opencontainers.image.base.digest=sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe - name: Generate artifact attestation uses: actions/attest-build-provenance@v1 diff --git a/.github/workflows/verifier-release.yml b/.github/workflows/verifier-release.yml index d02d5b3c2..92243ba82 100644 --- a/.github/workflows/verifier-release.yml +++ b/.github/workflows/verifier-release.yml @@ -22,11 +22,21 @@ jobs: - name: Checkout repository uses: actions/checkout@v5 - - name: Parse version from tag + - name: Parse and check version run: | - VERSION=${GITHUB_REF#refs/tags/verifier-v} - echo "VERSION=$VERSION" >> $GITHUB_ENV - echo "Parsed version: $VERSION" + case "${GITHUB_REF}" in + refs/tags/verifier-v*) VERSION=${GITHUB_REF#refs/tags/verifier-v} ;; + *) + echo "This release workflow must run on a verifier-v* tag; got ${GITHUB_REF}." >&2 + exit 1 + ;; + esac + if [ -z "${VERSION}" ]; then + echo "The release version is empty." >&2 + exit 1 + fi + echo "VERSION=${VERSION}" >> "$GITHUB_ENV" + echo "Parsed version: ${VERSION}" - name: Log in to Docker Hub uses: docker/login-action@v3 @@ -60,8 +70,21 @@ jobs: build-shared=dstack/build/shared build-args: | DSTACK_REV=${{ env.GIT_REV }} + IMAGE_VERSION=${{ env.VERSION }} + IMAGE_SOURCE_URL=${{ github.server_url }}/${{ github.repository }} DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}.git SOURCE_DATE_EPOCH=${{ env.TIMESTAMP }} + annotations: | + manifest:org.opencontainers.image.title=dstack-verifier + manifest:org.opencontainers.image.description=Remote attestation verification service for dstack + manifest:org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} + manifest:org.opencontainers.image.revision=${{ env.GIT_REV }} + manifest:org.opencontainers.image.version=${{ env.VERSION }} + manifest:org.opencontainers.image.url=${{ github.server_url }}/${{ github.repository }}/tree/${{ env.GIT_REV }}/dstack/verifier + manifest:org.opencontainers.image.documentation=${{ github.server_url }}/${{ github.repository }}/blob/${{ env.GIT_REV }}/dstack/verifier/README.md + manifest:org.opencontainers.image.licenses=Apache-2.0 + manifest:org.opencontainers.image.base.name=docker.io/library/debian:bookworm + manifest:org.opencontainers.image.base.digest=sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe - name: Generate artifact attestation uses: actions/attest-build-provenance@v1 diff --git a/dstack/build/shared/build-lib.sh b/dstack/build/shared/build-lib.sh index d24d368d4..c621a0244 100755 --- a/dstack/build/shared/build-lib.sh +++ b/dstack/build/shared/build-lib.sh @@ -12,6 +12,8 @@ # DOCKERFILE - path to the Dockerfile # GIT_REV - git revision to build # DSTACK_SRC_URL - git URL for dstack source +# IMAGE_VERSION - version recorded in the image metadata +# IMAGE_SOURCE_URL - repository URL recorded in the image metadata set -euo pipefail @@ -52,6 +54,8 @@ docker_build() { --build-arg "SOURCE_DATE_EPOCH=$commit_timestamp" --build-arg "DSTACK_REV=$GIT_REV" --build-arg "DSTACK_SRC_URL=$DSTACK_SRC_URL" + --build-arg "IMAGE_VERSION=${IMAGE_VERSION:-dev}" + --build-arg "IMAGE_SOURCE_URL=${IMAGE_SOURCE_URL:-${DSTACK_SRC_URL%.git}}" ) if [ -n "${NO_CACHE:-}" ]; then diff --git a/dstack/gateway/dstack-app/builder/Dockerfile b/dstack/gateway/dstack-app/builder/Dockerfile index e91b00486..8acdaea6d 100644 --- a/dstack/gateway/dstack-app/builder/Dockerfile +++ b/dstack/gateway/dstack-app/builder/Dockerfile @@ -27,6 +27,19 @@ RUN cd repo/dstack && cargo build --release -p dstack-gateway --target x86_64-un RUN echo "${DSTACK_REV}" > /build/.GIT_REV FROM debian:bookworm@sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe +ARG DSTACK_REV +ARG IMAGE_VERSION=dev +ARG IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack +LABEL org.opencontainers.image.title="dstack-gateway" \ + org.opencontainers.image.description="Gateway service for dstack confidential applications" \ + org.opencontainers.image.source="${IMAGE_SOURCE_URL}" \ + org.opencontainers.image.revision="${DSTACK_REV}" \ + org.opencontainers.image.version="${IMAGE_VERSION}" \ + org.opencontainers.image.url="${IMAGE_SOURCE_URL}/tree/${DSTACK_REV}/dstack/gateway" \ + org.opencontainers.image.documentation="${IMAGE_SOURCE_URL}/blob/${DSTACK_REV}/dstack/gateway/dstack-app/builder/README.md" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.base.name="docker.io/library/debian:bookworm" \ + org.opencontainers.image.base.digest="sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe" COPY --from=build-shared pin-packages.sh /build/ COPY ./shared/pinned-packages.txt /build/ WORKDIR /build @@ -41,6 +54,18 @@ RUN ./pin-packages.sh ./pinned-packages.txt && \ && rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache COPY --from=gateway-builder /build/repo/dstack/target/x86_64-unknown-linux-musl/release/dstack-gateway /usr/local/bin/dstack-gateway COPY --from=gateway-builder /build/.GIT_REV /etc/ +RUN mkdir -p /etc/dstack-gateway && printf '%s\n' \ + "org.opencontainers.image.title=dstack-gateway" \ + "org.opencontainers.image.description=Gateway service for dstack confidential applications" \ + "org.opencontainers.image.source=${IMAGE_SOURCE_URL}" \ + "org.opencontainers.image.revision=${DSTACK_REV}" \ + "org.opencontainers.image.version=${IMAGE_VERSION}" \ + "org.opencontainers.image.url=${IMAGE_SOURCE_URL}/tree/${DSTACK_REV}/dstack/gateway" \ + "org.opencontainers.image.documentation=${IMAGE_SOURCE_URL}/blob/${DSTACK_REV}/dstack/gateway/dstack-app/builder/README.md" \ + "org.opencontainers.image.licenses=Apache-2.0" \ + "org.opencontainers.image.base.name=docker.io/library/debian:bookworm" \ + "org.opencontainers.image.base.digest=sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe" \ + > /etc/dstack-gateway/build-info WORKDIR /app COPY entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh diff --git a/dstack/kms/dstack-app/builder/Dockerfile b/dstack/kms/dstack-app/builder/Dockerfile index 24dd7bce4..2af502383 100644 --- a/dstack/kms/dstack-app/builder/Dockerfile +++ b/dstack/kms/dstack-app/builder/Dockerfile @@ -27,6 +27,19 @@ RUN cd repo/dstack && cargo build --release -p dstack-kms --target x86_64-unknow RUN echo "${DSTACK_REV}" > /build/.GIT_REV FROM debian:bookworm@sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe +ARG DSTACK_REV +ARG IMAGE_VERSION=dev +ARG IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack +LABEL org.opencontainers.image.title="dstack-kms" \ + org.opencontainers.image.description="Key management service for dstack confidential applications" \ + org.opencontainers.image.source="${IMAGE_SOURCE_URL}" \ + org.opencontainers.image.revision="${DSTACK_REV}" \ + org.opencontainers.image.version="${IMAGE_VERSION}" \ + org.opencontainers.image.url="${IMAGE_SOURCE_URL}/tree/${DSTACK_REV}/dstack/kms" \ + org.opencontainers.image.documentation="${IMAGE_SOURCE_URL}/blob/${DSTACK_REV}/dstack/kms/README.md" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.base.name="docker.io/library/debian:bookworm" \ + org.opencontainers.image.base.digest="sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe" COPY --from=build-shared /pin-packages.sh /build/ COPY ./shared/pinned-packages.txt /build/ WORKDIR /build @@ -42,4 +55,16 @@ RUN ./pin-packages.sh ./pinned-packages.txt && \ WORKDIR / COPY --from=kms-builder /build/repo/dstack/target/x86_64-unknown-linux-musl/release/dstack-kms /usr/local/bin/dstack-kms COPY --from=kms-builder /build/.GIT_REV /etc/ +RUN mkdir -p /etc/dstack-kms && printf '%s\n' \ + "org.opencontainers.image.title=dstack-kms" \ + "org.opencontainers.image.description=Key management service for dstack confidential applications" \ + "org.opencontainers.image.source=${IMAGE_SOURCE_URL}" \ + "org.opencontainers.image.revision=${DSTACK_REV}" \ + "org.opencontainers.image.version=${IMAGE_VERSION}" \ + "org.opencontainers.image.url=${IMAGE_SOURCE_URL}/tree/${DSTACK_REV}/dstack/kms" \ + "org.opencontainers.image.documentation=${IMAGE_SOURCE_URL}/blob/${DSTACK_REV}/dstack/kms/README.md" \ + "org.opencontainers.image.licenses=Apache-2.0" \ + "org.opencontainers.image.base.name=docker.io/library/debian:bookworm" \ + "org.opencontainers.image.base.digest=sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe" \ + > /etc/dstack-kms/build-info CMD ["dstack-kms"] diff --git a/dstack/verifier/builder/Dockerfile b/dstack/verifier/builder/Dockerfile index 5b9245200..7a70a588b 100644 --- a/dstack/verifier/builder/Dockerfile +++ b/dstack/verifier/builder/Dockerfile @@ -31,6 +31,19 @@ RUN cd repo/dstack && cargo build --release -p dstack-verifier --target x86_64-u RUN echo "${DSTACK_REV}" > /build/.GIT_REV FROM debian:bookworm@sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe +ARG DSTACK_REV +ARG IMAGE_VERSION=dev +ARG IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack +LABEL org.opencontainers.image.title="dstack-verifier" \ + org.opencontainers.image.description="Remote attestation verification service for dstack" \ + org.opencontainers.image.source="${IMAGE_SOURCE_URL}" \ + org.opencontainers.image.revision="${DSTACK_REV}" \ + org.opencontainers.image.version="${IMAGE_VERSION}" \ + org.opencontainers.image.url="${IMAGE_SOURCE_URL}/tree/${DSTACK_REV}/dstack/verifier" \ + org.opencontainers.image.documentation="${IMAGE_SOURCE_URL}/blob/${DSTACK_REV}/dstack/verifier/README.md" \ + org.opencontainers.image.licenses="Apache-2.0" \ + org.opencontainers.image.base.name="docker.io/library/debian:bookworm" \ + org.opencontainers.image.base.digest="sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe" COPY --from=build-shared pin-packages.sh /build/ COPY builder/shared/pinned-packages.txt /build/ WORKDIR /build @@ -42,6 +55,18 @@ RUN ./pin-packages.sh ./pinned-packages.txt && \ && rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache COPY --from=verifier-builder /build/repo/dstack/target/x86_64-unknown-linux-musl/release/dstack-verifier /usr/local/bin/dstack-verifier COPY --from=verifier-builder /build/.GIT_REV /etc/ +RUN mkdir -p /etc/dstack-verifier && printf '%s\n' \ + "org.opencontainers.image.title=dstack-verifier" \ + "org.opencontainers.image.description=Remote attestation verification service for dstack" \ + "org.opencontainers.image.source=${IMAGE_SOURCE_URL}" \ + "org.opencontainers.image.revision=${DSTACK_REV}" \ + "org.opencontainers.image.version=${IMAGE_VERSION}" \ + "org.opencontainers.image.url=${IMAGE_SOURCE_URL}/tree/${DSTACK_REV}/dstack/verifier" \ + "org.opencontainers.image.documentation=${IMAGE_SOURCE_URL}/blob/${DSTACK_REV}/dstack/verifier/README.md" \ + "org.opencontainers.image.licenses=Apache-2.0" \ + "org.opencontainers.image.base.name=docker.io/library/debian:bookworm" \ + "org.opencontainers.image.base.digest=sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe" \ + > /etc/dstack-verifier/build-info RUN mkdir -p /etc/dstack COPY dstack-verifier.toml /etc/dstack/dstack-verifier.toml WORKDIR /var/lib/dstack-verifier From 13900e1fd74e213cb2bf914fb5982d55f0d16375 Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Tue, 8 Sep 2026 00:42:00 +0800 Subject: [PATCH 2/5] Fix DSTACK_SRC_URL formatting in gateway-release.yml Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .github/workflows/gateway-release.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/gateway-release.yml b/.github/workflows/gateway-release.yml index 26bc5c27d..638b41ece 100644 --- a/.github/workflows/gateway-release.yml +++ b/.github/workflows/gateway-release.yml @@ -69,6 +69,7 @@ jobs: DSTACK_REV=${{ env.GIT_REV }} IMAGE_VERSION=${{ env.VERSION }} IMAGE_SOURCE_URL=${{ github.server_url }}/${{ github.repository }} + DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}.git SOURCE_DATE_EPOCH=${{ env.TIMESTAMP }} annotations: | manifest:org.opencontainers.image.title=dstack-gateway From c8b7f73374b2cf1a35fa8b0ddc08a1006a497d9e Mon Sep 17 00:00:00 2001 From: Kevin Wang Date: Mon, 7 Sep 2026 20:11:21 -0700 Subject: [PATCH 3/5] ci(images): build release images through the component build scripts --- .github/workflows/gateway-release.yml | 65 +++++----- .github/workflows/kms-release.yml | 65 +++++----- .github/workflows/verifier-release.yml | 67 +++++----- dstack/build/shared/build-lib.sh | 118 +++++++++++++++++- dstack/gateway/README.md | 43 +++++++ dstack/gateway/dstack-app/builder/Dockerfile | 30 +---- .../gateway/dstack-app/builder/build-image.sh | 16 ++- dstack/kms/dstack-app/builder/Dockerfile | 30 +---- dstack/kms/dstack-app/builder/build-image.sh | 14 ++- dstack/verifier/builder/Dockerfile | 30 +---- dstack/verifier/builder/build-image.sh | 14 ++- 11 files changed, 289 insertions(+), 203 deletions(-) create mode 100644 dstack/gateway/README.md diff --git a/.github/workflows/gateway-release.yml b/.github/workflows/gateway-release.yml index 638b41ece..524284b0d 100644 --- a/.github/workflows/gateway-release.yml +++ b/.github/workflows/gateway-release.yml @@ -6,6 +6,11 @@ name: Gateway Release on: workflow_dispatch: + inputs: + version: + description: Release version without the gateway-v prefix + required: true + type: string push: tags: - 'gateway-v*' @@ -23,16 +28,15 @@ jobs: uses: actions/checkout@v5 - name: Parse and check version + env: + DISPATCH_VERSION: ${{ inputs.version }} run: | case "${GITHUB_REF}" in refs/tags/gateway-v*) VERSION=${GITHUB_REF#refs/tags/gateway-v} ;; - *) - echo "This release workflow must run on a gateway-v* tag; got ${GITHUB_REF}." >&2 - exit 1 - ;; + *) VERSION=${DISPATCH_VERSION} ;; esac if [ -z "${VERSION}" ]; then - echo "The release version is empty." >&2 + echo "This release workflow needs a gateway-v* tag or a version input; got ${GITHUB_REF}." >&2 exit 1 fi echo "VERSION=${VERSION}" >> "$GITHUB_ENV" @@ -47,41 +51,30 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Get Git commit timestamps - run: | - echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV - echo "GIT_REV=$(git rev-parse HEAD)" >> $GITHUB_ENV + - name: Resolve the revision to build + run: echo "GIT_REV=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + # Release images are built through the same script contributors run by + # hand, so a published image can be reproduced and checked digest-for-digest. + # See the builder README for the reproduction recipe. - name: Build and push Docker image id: build-and-push - uses: docker/build-push-action@v5 env: - SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }} - with: - context: dstack/gateway/dstack-app/builder - push: true - tags: ${{ vars.DOCKERHUB_ORG }}/dstack-gateway:${{ env.VERSION }} - platforms: linux/amd64 - provenance: false - build-contexts: | - build-shared=dstack/build/shared - build-args: | - DSTACK_REV=${{ env.GIT_REV }} - IMAGE_VERSION=${{ env.VERSION }} - IMAGE_SOURCE_URL=${{ github.server_url }}/${{ github.repository }} - DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}.git - SOURCE_DATE_EPOCH=${{ env.TIMESTAMP }} - annotations: | - manifest:org.opencontainers.image.title=dstack-gateway - manifest:org.opencontainers.image.description=Gateway service for dstack confidential applications - manifest:org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} - manifest:org.opencontainers.image.revision=${{ env.GIT_REV }} - manifest:org.opencontainers.image.version=${{ env.VERSION }} - manifest:org.opencontainers.image.url=${{ github.server_url }}/${{ github.repository }}/tree/${{ env.GIT_REV }}/dstack/gateway - manifest:org.opencontainers.image.documentation=${{ github.server_url }}/${{ github.repository }}/blob/${{ env.GIT_REV }}/dstack/gateway/dstack-app/builder/README.md - manifest:org.opencontainers.image.licenses=Apache-2.0 - manifest:org.opencontainers.image.base.name=docker.io/library/debian:bookworm - manifest:org.opencontainers.image.base.digest=sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe + PUSH: "1" + METADATA_FILE: ${{ runner.temp }}/build-metadata.json + GIT_REV: ${{ env.GIT_REV }} + IMAGE_VERSION: ${{ env.VERSION }} + IMAGE_SOURCE_URL: ${{ github.server_url }}/${{ github.repository }} + DSTACK_SRC_URL: ${{ github.server_url }}/${{ github.repository }}.git + run: | + dstack/gateway/dstack-app/builder/build-image.sh \ + ${{ vars.DOCKERHUB_ORG }}/dstack-gateway:${{ env.VERSION }} + digest=$(jq -r '."containerimage.digest"' "$METADATA_FILE") + if [ -z "$digest" ] || [ "$digest" = null ]; then + echo "the build did not report an image digest" >&2 + exit 1 + fi + echo "digest=$digest" >> "$GITHUB_OUTPUT" - name: Generate artifact attestation uses: actions/attest-build-provenance@v1 diff --git a/.github/workflows/kms-release.yml b/.github/workflows/kms-release.yml index c44f1de44..6ed9deb5f 100644 --- a/.github/workflows/kms-release.yml +++ b/.github/workflows/kms-release.yml @@ -6,6 +6,11 @@ name: KMS Release on: workflow_dispatch: + inputs: + version: + description: Release version without the kms-v prefix + required: true + type: string push: tags: - 'kms-v*' @@ -30,16 +35,15 @@ jobs: dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades - name: Parse and check version + env: + DISPATCH_VERSION: ${{ inputs.version }} run: | case "${GITHUB_REF}" in refs/tags/kms-v*) VERSION=${GITHUB_REF#refs/tags/kms-v} ;; - *) - echo "This release workflow must run on a kms-v* tag; got ${GITHUB_REF}." >&2 - exit 1 - ;; + *) VERSION=${DISPATCH_VERSION} ;; esac if [ -z "${VERSION}" ]; then - echo "The release version is empty." >&2 + echo "This release workflow needs a kms-v* tag or a version input; got ${GITHUB_REF}." >&2 exit 1 fi echo "VERSION=${VERSION}" >> "$GITHUB_ENV" @@ -54,41 +58,30 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Get Git commit timestamps - run: | - echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV - echo "GIT_REV=$(git rev-parse HEAD)" >> $GITHUB_ENV + - name: Resolve the revision to build + run: echo "GIT_REV=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + # Release images are built through the same script contributors run by + # hand, so a published image can be reproduced and checked digest-for-digest. + # See the builder README for the reproduction recipe. - name: Build and push Docker image id: build-and-push - uses: docker/build-push-action@v5 env: - SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }} - with: - context: dstack/kms/dstack-app/builder - push: true - tags: ${{ vars.DOCKERHUB_ORG }}/dstack-kms:${{ env.VERSION }} - platforms: linux/amd64 - provenance: false - build-contexts: | - build-shared=dstack/build/shared - build-args: | - DSTACK_REV=${{ env.GIT_REV }} - IMAGE_VERSION=${{ env.VERSION }} - IMAGE_SOURCE_URL=${{ github.server_url }}/${{ github.repository }} - DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }} - SOURCE_DATE_EPOCH=${{ env.TIMESTAMP }} - annotations: | - manifest:org.opencontainers.image.title=dstack-kms - manifest:org.opencontainers.image.description=Key management service for dstack confidential applications - manifest:org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} - manifest:org.opencontainers.image.revision=${{ env.GIT_REV }} - manifest:org.opencontainers.image.version=${{ env.VERSION }} - manifest:org.opencontainers.image.url=${{ github.server_url }}/${{ github.repository }}/tree/${{ env.GIT_REV }}/dstack/kms - manifest:org.opencontainers.image.documentation=${{ github.server_url }}/${{ github.repository }}/blob/${{ env.GIT_REV }}/dstack/kms/README.md - manifest:org.opencontainers.image.licenses=Apache-2.0 - manifest:org.opencontainers.image.base.name=docker.io/library/debian:bookworm - manifest:org.opencontainers.image.base.digest=sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe + PUSH: "1" + METADATA_FILE: ${{ runner.temp }}/build-metadata.json + GIT_REV: ${{ env.GIT_REV }} + IMAGE_VERSION: ${{ env.VERSION }} + IMAGE_SOURCE_URL: ${{ github.server_url }}/${{ github.repository }} + DSTACK_SRC_URL: ${{ github.server_url }}/${{ github.repository }}.git + run: | + dstack/kms/dstack-app/builder/build-image.sh \ + ${{ vars.DOCKERHUB_ORG }}/dstack-kms:${{ env.VERSION }} + digest=$(jq -r '."containerimage.digest"' "$METADATA_FILE") + if [ -z "$digest" ] || [ "$digest" = null ]; then + echo "the build did not report an image digest" >&2 + exit 1 + fi + echo "digest=$digest" >> "$GITHUB_OUTPUT" - name: Generate artifact attestation uses: actions/attest-build-provenance@v1 diff --git a/.github/workflows/verifier-release.yml b/.github/workflows/verifier-release.yml index 92243ba82..b242c6d5a 100644 --- a/.github/workflows/verifier-release.yml +++ b/.github/workflows/verifier-release.yml @@ -6,6 +6,11 @@ name: Verifier Release on: workflow_dispatch: + inputs: + version: + description: Release version without the verifier-v prefix + required: true + type: string push: tags: - 'verifier-v*' @@ -23,16 +28,15 @@ jobs: uses: actions/checkout@v5 - name: Parse and check version + env: + DISPATCH_VERSION: ${{ inputs.version }} run: | case "${GITHUB_REF}" in refs/tags/verifier-v*) VERSION=${GITHUB_REF#refs/tags/verifier-v} ;; - *) - echo "This release workflow must run on a verifier-v* tag; got ${GITHUB_REF}." >&2 - exit 1 - ;; + *) VERSION=${DISPATCH_VERSION} ;; esac if [ -z "${VERSION}" ]; then - echo "The release version is empty." >&2 + echo "This release workflow needs a verifier-v* tag or a version input; got ${GITHUB_REF}." >&2 exit 1 fi echo "VERSION=${VERSION}" >> "$GITHUB_ENV" @@ -47,44 +51,31 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - - name: Get Git commit timestamps - run: | - echo "TIMESTAMP=$(git log -1 --pretty=%ct)" >> $GITHUB_ENV - echo "GIT_REV=$(git rev-parse HEAD)" >> $GITHUB_ENV + - name: Resolve the revision to build + run: echo "GIT_REV=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + # Release images are built through the same script contributors run by + # hand, so a published image can be reproduced and checked digest-for-digest. + # See the builder README for the reproduction recipe. - name: Build and push Docker image id: build-and-push - uses: docker/build-push-action@v5 env: - SOURCE_DATE_EPOCH: ${{ env.TIMESTAMP }} - with: - context: dstack/verifier - file: dstack/verifier/builder/Dockerfile - push: true - tags: | - ${{ vars.DOCKERHUB_ORG }}/dstack-verifier:${{ env.VERSION }} + PUSH: "1" + METADATA_FILE: ${{ runner.temp }}/build-metadata.json + GIT_REV: ${{ env.GIT_REV }} + IMAGE_VERSION: ${{ env.VERSION }} + IMAGE_SOURCE_URL: ${{ github.server_url }}/${{ github.repository }} + DSTACK_SRC_URL: ${{ github.server_url }}/${{ github.repository }}.git + run: | + dstack/verifier/builder/build-image.sh \ + ${{ vars.DOCKERHUB_ORG }}/dstack-verifier:${{ env.VERSION }} \ ${{ vars.DOCKERHUB_ORG }}/dstack-verifier:latest - platforms: linux/amd64 - provenance: false - build-contexts: | - build-shared=dstack/build/shared - build-args: | - DSTACK_REV=${{ env.GIT_REV }} - IMAGE_VERSION=${{ env.VERSION }} - IMAGE_SOURCE_URL=${{ github.server_url }}/${{ github.repository }} - DSTACK_SRC_URL=${{ github.server_url }}/${{ github.repository }}.git - SOURCE_DATE_EPOCH=${{ env.TIMESTAMP }} - annotations: | - manifest:org.opencontainers.image.title=dstack-verifier - manifest:org.opencontainers.image.description=Remote attestation verification service for dstack - manifest:org.opencontainers.image.source=${{ github.server_url }}/${{ github.repository }} - manifest:org.opencontainers.image.revision=${{ env.GIT_REV }} - manifest:org.opencontainers.image.version=${{ env.VERSION }} - manifest:org.opencontainers.image.url=${{ github.server_url }}/${{ github.repository }}/tree/${{ env.GIT_REV }}/dstack/verifier - manifest:org.opencontainers.image.documentation=${{ github.server_url }}/${{ github.repository }}/blob/${{ env.GIT_REV }}/dstack/verifier/README.md - manifest:org.opencontainers.image.licenses=Apache-2.0 - manifest:org.opencontainers.image.base.name=docker.io/library/debian:bookworm - manifest:org.opencontainers.image.base.digest=sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe + digest=$(jq -r '."containerimage.digest"' "$METADATA_FILE") + if [ -z "$digest" ] || [ "$digest" = null ]; then + echo "the build did not report an image digest" >&2 + exit 1 + fi + echo "digest=$digest" >> "$GITHUB_OUTPUT" - name: Generate artifact attestation uses: actions/attest-build-provenance@v1 diff --git a/dstack/build/shared/build-lib.sh b/dstack/build/shared/build-lib.sh index c621a0244..cc9e8fc07 100755 --- a/dstack/build/shared/build-lib.sh +++ b/dstack/build/shared/build-lib.sh @@ -6,14 +6,28 @@ # Shared build library for reproducible Docker image builds. # +# Release CI runs the very same component build-image.sh that a user runs by +# hand, so everything that ends up in the published image -- layers, config +# labels and manifest annotations -- is reproducible outside CI. +# # Expected variables (set by the sourcing script): # REPO_ROOT - absolute path to the monorepo root # CONTEXT_DIR - Docker build context directory # DOCKERFILE - path to the Dockerfile # GIT_REV - git revision to build # DSTACK_SRC_URL - git URL for dstack source -# IMAGE_VERSION - version recorded in the image metadata +# +# Optional variables: +# IMAGE_VERSION - version recorded in the image metadata (default: dev). +# Release CI passes the tag version; pass the same value to +# reproduce a published image. # IMAGE_SOURCE_URL - repository URL recorded in the image metadata +# PUSH - non-empty to push the image instead of loading it +# OCI_TAR - path to also write an OCI archive to. The manifest digest +# of that archive is what a registry reports, so this is the +# way to check a local rebuild against a published digest. +# METADATA_FILE - path buildx writes its build metadata (incl. digest) to +# NO_CACHE - non-empty to build without the layer cache set -euo pipefail @@ -38,10 +52,63 @@ extract_packages() { >"$pkg_list_file" } +# Print the OCI metadata for a component image as "key=value" lines. +# +# This is the single source of truth for that metadata: docker_build feeds the +# result to the config labels, the manifest annotations and the in-image +# /etc//build-info file, so the three can never disagree. +# +# $1 - image title, e.g. dstack-kms +# $2 - one-line description +# $3 - component directory within the repo, e.g. dstack/kms +# $4 - documentation file within the repo, e.g. dstack/kms/README.md +image_metadata() { + local title=$1 + local description=$2 + local component_dir=$3 + local documentation=$4 + + local source_url=${IMAGE_SOURCE_URL:-${DSTACK_SRC_URL%.git}} + + # Read the pinned base image out of the Dockerfile's last FROM so the + # base.name/base.digest claims cannot drift from the layer they describe. + local base_ref + base_ref=$(awk '$1 == "FROM" { ref = $2 } END { print ref }' "$DOCKERFILE") + if [[ "$base_ref" != *@sha256:* ]]; then + echo "the final FROM in $DOCKERFILE must pin a digest; got '$base_ref'" >&2 + return 1 + fi + local base_name=${base_ref%@*} + local base_digest=${base_ref#*@} + # Normalize a bare official-image reference to its canonical form. + if [[ "$base_name" != */* ]]; then + base_name="docker.io/library/$base_name" + fi + + printf '%s\n' \ + "org.opencontainers.image.title=$title" \ + "org.opencontainers.image.description=$description" \ + "org.opencontainers.image.source=$source_url" \ + "org.opencontainers.image.revision=$GIT_REV" \ + "org.opencontainers.image.version=${IMAGE_VERSION:-dev}" \ + "org.opencontainers.image.url=$source_url/tree/$GIT_REV/$component_dir" \ + "org.opencontainers.image.documentation=$source_url/blob/$GIT_REV/$documentation" \ + "org.opencontainers.image.licenses=Apache-2.0" \ + "org.opencontainers.image.base.name=$base_name" \ + "org.opencontainers.image.base.digest=$base_digest" +} + +# docker_build <tags> [target] [pkg_list_file] [metadata] +# +# tags - newline-separated list of image tags +# target - build stage to stop at, empty for the final image +# pkg_list_file - where to record the installed Debian packages +# metadata - "key=value" lines from image_metadata, empty to record none docker_build() { - local image_name=$1 + local tags=$1 local target=${2:-} local pkg_list_file=${3:-} + local metadata=${4:-} local commit_timestamp commit_timestamp=$(git -C "$REPO_ROOT" show -s --format=%ct "$GIT_REV") @@ -49,15 +116,54 @@ docker_build() { local args=( --builder "$BUILDKIT_BUILDER" --progress=plain - --output "type=docker,name=$image_name,rewrite-timestamp=true" + # The Rust stages cross-compile to x86_64-unknown-linux-musl, so pin the + # platform rather than inheriting the host's -- an arm64 workstation must + # still reproduce the released amd64 image. + --platform linux/amd64 + # BuildKit attaches a provenance attestation by default when pushing, + # which carries build timestamps and turns the pushed tag into an index. + # Both would make the digest unreproducible. + --provenance=false --build-context "build-shared=$BUILD_SHARED_DIR" --build-arg "SOURCE_DATE_EPOCH=$commit_timestamp" --build-arg "DSTACK_REV=$GIT_REV" --build-arg "DSTACK_SRC_URL=$DSTACK_SRC_URL" - --build-arg "IMAGE_VERSION=${IMAGE_VERSION:-dev}" - --build-arg "IMAGE_SOURCE_URL=${IMAGE_SOURCE_URL:-${DSTACK_SRC_URL%.git}}" ) + local tag + while IFS= read -r tag; do + [ -n "$tag" ] || continue + args+=(--tag "$tag") + done <<<"$tags" + + if [ -n "$metadata" ]; then + local pair + while IFS= read -r pair; do + [ -n "$pair" ] || continue + args+=(--label "$pair" --annotation "manifest:$pair") + done <<<"$metadata" + args+=(--build-arg "IMAGE_METADATA=$metadata") + fi + + # Only the final image is publishable. Intermediate stages are built purely + # to extract their package lists and must never reach a registry. + # + # oci-mediatypes keeps the pushed manifest byte-identical to the OCI archive + # a local rebuild produces, which is what makes the digests comparable. + if [ -n "${PUSH:-}" ] && [ -z "$target" ]; then + args+=(--output "type=image,push=true,oci-mediatypes=true,rewrite-timestamp=true") + fi + if [ -n "${OCI_TAR:-}" ] && [ -z "$target" ]; then + args+=(--output "type=oci,oci-mediatypes=true,rewrite-timestamp=true,dest=$OCI_TAR") + fi + # Always load locally as well: extract_packages inspects the built image. + args+=(--output "type=docker,rewrite-timestamp=true") + # Guard this like the outputs above: the intermediate stage builds run after + # the final one and would otherwise overwrite its reported digest. + if [ -n "${METADATA_FILE:-}" ] && [ -z "$target" ]; then + args+=(--metadata-file "$METADATA_FILE") + fi + if [ -n "${NO_CACHE:-}" ]; then args+=(--no-cache) fi @@ -70,7 +176,7 @@ docker_build() { --file "$DOCKERFILE" \ "$CONTEXT_DIR" - extract_packages "$image_name" "$pkg_list_file" + extract_packages "$(head -n1 <<<"$tags")" "$pkg_list_file" } # Verify that pinned-packages files haven't changed (idempotency check). diff --git a/dstack/gateway/README.md b/dstack/gateway/README.md new file mode 100644 index 000000000..af79ee75f --- /dev/null +++ b/dstack/gateway/README.md @@ -0,0 +1,43 @@ +# dstack-gateway + +The reverse proxy that gives deployed dstack apps zero-trust network access. It +terminates TLS, provisions certificates over ACME DNS-01, and routes traffic to +CVMs across a WireGuard mesh it manages itself. + +## Ingress mapping + +Apps are addressed as `<id>[-[<port>][s|g]].<base_domain>`: + +| Form | Behaviour | +| --- | --- | +| `<id>.<base_domain>` | TLS terminated at the gateway, forwarded as TCP | +| `<id>-<port>s.<base_domain>` | TLS passthrough to the app | +| `<id>-<port>g.<base_domain>` | TLS terminated, forwarded as HTTP/2 (gRPC) | + +`<id>` is the app ID or the instance ID; `<port>` defaults to 80. + +## Configuration + +`gateway.toml` holds the base domain, the certificate and WireGuard settings, +and the admin API credentials. Operator-facing admin RPCs are served on a +separate listener behind the shared HTTP authenticator, the same way the KMS and +VMM expose theirs. + +## Running + +Build and run from the workspace root: + +```bash +cargo build --release -p dstack-gateway +sudo ./target/release/dstack-gateway -c gateway.toml +``` + +The gateway needs `CAP_NET_ADMIN` to manage its WireGuard interface, hence +`sudo`. + +## Further reading + +- [Cluster deployment](docs/cluster-deployment.md) — running a replicated gateway +- [Builder image](dstack-app/builder/README.md) — reproducible container image +- [Deployment guide](../../docs/deployment.md) +- [Security guide](../../docs/security-guide/security-guide.md) diff --git a/dstack/gateway/dstack-app/builder/Dockerfile b/dstack/gateway/dstack-app/builder/Dockerfile index 8acdaea6d..42a78560f 100644 --- a/dstack/gateway/dstack-app/builder/Dockerfile +++ b/dstack/gateway/dstack-app/builder/Dockerfile @@ -27,19 +27,6 @@ RUN cd repo/dstack && cargo build --release -p dstack-gateway --target x86_64-un RUN echo "${DSTACK_REV}" > /build/.GIT_REV FROM debian:bookworm@sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe -ARG DSTACK_REV -ARG IMAGE_VERSION=dev -ARG IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack -LABEL org.opencontainers.image.title="dstack-gateway" \ - org.opencontainers.image.description="Gateway service for dstack confidential applications" \ - org.opencontainers.image.source="${IMAGE_SOURCE_URL}" \ - org.opencontainers.image.revision="${DSTACK_REV}" \ - org.opencontainers.image.version="${IMAGE_VERSION}" \ - org.opencontainers.image.url="${IMAGE_SOURCE_URL}/tree/${DSTACK_REV}/dstack/gateway" \ - org.opencontainers.image.documentation="${IMAGE_SOURCE_URL}/blob/${DSTACK_REV}/dstack/gateway/dstack-app/builder/README.md" \ - org.opencontainers.image.licenses="Apache-2.0" \ - org.opencontainers.image.base.name="docker.io/library/debian:bookworm" \ - org.opencontainers.image.base.digest="sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe" COPY --from=build-shared pin-packages.sh /build/ COPY ./shared/pinned-packages.txt /build/ WORKDIR /build @@ -54,18 +41,11 @@ RUN ./pin-packages.sh ./pinned-packages.txt && \ && rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache COPY --from=gateway-builder /build/repo/dstack/target/x86_64-unknown-linux-musl/release/dstack-gateway /usr/local/bin/dstack-gateway COPY --from=gateway-builder /build/.GIT_REV /etc/ -RUN mkdir -p /etc/dstack-gateway && printf '%s\n' \ - "org.opencontainers.image.title=dstack-gateway" \ - "org.opencontainers.image.description=Gateway service for dstack confidential applications" \ - "org.opencontainers.image.source=${IMAGE_SOURCE_URL}" \ - "org.opencontainers.image.revision=${DSTACK_REV}" \ - "org.opencontainers.image.version=${IMAGE_VERSION}" \ - "org.opencontainers.image.url=${IMAGE_SOURCE_URL}/tree/${DSTACK_REV}/dstack/gateway" \ - "org.opencontainers.image.documentation=${IMAGE_SOURCE_URL}/blob/${DSTACK_REV}/dstack/gateway/dstack-app/builder/README.md" \ - "org.opencontainers.image.licenses=Apache-2.0" \ - "org.opencontainers.image.base.name=docker.io/library/debian:bookworm" \ - "org.opencontainers.image.base.digest=sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe" \ - > /etc/dstack-gateway/build-info +# The OCI metadata is generated once by build-lib.sh and injected here, as +# the config labels and as the manifest annotations, so the three views of +# the image can never disagree. +ARG IMAGE_METADATA +RUN mkdir -p /etc/dstack-gateway && printf '%s\n' "${IMAGE_METADATA}" > /etc/dstack-gateway/build-info WORKDIR /app COPY entrypoint.sh /app/entrypoint.sh RUN chmod +x /app/entrypoint.sh diff --git a/dstack/gateway/dstack-app/builder/build-image.sh b/dstack/gateway/dstack-app/builder/build-image.sh index 3d86546f1..612cac220 100755 --- a/dstack/gateway/dstack-app/builder/build-image.sh +++ b/dstack/gateway/dstack-app/builder/build-image.sh @@ -11,14 +11,16 @@ REPO_ROOT=$(git -C "$SCRIPT_DIR" rev-parse --show-toplevel) CONTEXT_DIR="$SCRIPT_DIR" SHARED_DIR="$SCRIPT_DIR/shared" DOCKERFILE="$SCRIPT_DIR/Dockerfile" +export CONTEXT_DIR DOCKERFILE +# shellcheck source=/dev/null source "$REPO_ROOT/dstack/build/shared/build-lib.sh" -NAME=${1:-} -if [ -z "$NAME" ]; then - echo "Usage: $0 <image-name>[:<tag>]" >&2 +if [ "$#" -eq 0 ]; then + echo "Usage: $0 <image-name>[:<tag>]..." >&2 exit 1 fi +TAGS=$(printf '%s\n' "$@") NO_CACHE=${NO_CACHE:-} GIT_REV=${GIT_REV:-HEAD} @@ -30,7 +32,13 @@ ensure_buildkit touch "$SHARED_DIR/builder-pinned-packages.txt" touch "$SHARED_DIR/pinned-packages.txt" -docker_build "$NAME" "" "$SHARED_DIR/pinned-packages.txt" +METADATA=$(image_metadata \ + "dstack-gateway" \ + "Gateway service for dstack confidential applications" \ + "dstack/gateway" \ + "dstack/gateway/README.md") + +docker_build "$TAGS" "" "$SHARED_DIR/pinned-packages.txt" "$METADATA" docker_build "gateway-builder-temp" "gateway-builder" "$SHARED_DIR/builder-pinned-packages.txt" check_clean_tree "$SHARED_DIR" diff --git a/dstack/kms/dstack-app/builder/Dockerfile b/dstack/kms/dstack-app/builder/Dockerfile index 2af502383..91d9e0a04 100644 --- a/dstack/kms/dstack-app/builder/Dockerfile +++ b/dstack/kms/dstack-app/builder/Dockerfile @@ -27,19 +27,6 @@ RUN cd repo/dstack && cargo build --release -p dstack-kms --target x86_64-unknow RUN echo "${DSTACK_REV}" > /build/.GIT_REV FROM debian:bookworm@sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe -ARG DSTACK_REV -ARG IMAGE_VERSION=dev -ARG IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack -LABEL org.opencontainers.image.title="dstack-kms" \ - org.opencontainers.image.description="Key management service for dstack confidential applications" \ - org.opencontainers.image.source="${IMAGE_SOURCE_URL}" \ - org.opencontainers.image.revision="${DSTACK_REV}" \ - org.opencontainers.image.version="${IMAGE_VERSION}" \ - org.opencontainers.image.url="${IMAGE_SOURCE_URL}/tree/${DSTACK_REV}/dstack/kms" \ - org.opencontainers.image.documentation="${IMAGE_SOURCE_URL}/blob/${DSTACK_REV}/dstack/kms/README.md" \ - org.opencontainers.image.licenses="Apache-2.0" \ - org.opencontainers.image.base.name="docker.io/library/debian:bookworm" \ - org.opencontainers.image.base.digest="sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe" COPY --from=build-shared /pin-packages.sh /build/ COPY ./shared/pinned-packages.txt /build/ WORKDIR /build @@ -55,16 +42,9 @@ RUN ./pin-packages.sh ./pinned-packages.txt && \ WORKDIR / COPY --from=kms-builder /build/repo/dstack/target/x86_64-unknown-linux-musl/release/dstack-kms /usr/local/bin/dstack-kms COPY --from=kms-builder /build/.GIT_REV /etc/ -RUN mkdir -p /etc/dstack-kms && printf '%s\n' \ - "org.opencontainers.image.title=dstack-kms" \ - "org.opencontainers.image.description=Key management service for dstack confidential applications" \ - "org.opencontainers.image.source=${IMAGE_SOURCE_URL}" \ - "org.opencontainers.image.revision=${DSTACK_REV}" \ - "org.opencontainers.image.version=${IMAGE_VERSION}" \ - "org.opencontainers.image.url=${IMAGE_SOURCE_URL}/tree/${DSTACK_REV}/dstack/kms" \ - "org.opencontainers.image.documentation=${IMAGE_SOURCE_URL}/blob/${DSTACK_REV}/dstack/kms/README.md" \ - "org.opencontainers.image.licenses=Apache-2.0" \ - "org.opencontainers.image.base.name=docker.io/library/debian:bookworm" \ - "org.opencontainers.image.base.digest=sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe" \ - > /etc/dstack-kms/build-info +# The OCI metadata is generated once by build-lib.sh and injected here, as +# the config labels and as the manifest annotations, so the three views of +# the image can never disagree. +ARG IMAGE_METADATA +RUN mkdir -p /etc/dstack-kms && printf '%s\n' "${IMAGE_METADATA}" > /etc/dstack-kms/build-info CMD ["dstack-kms"] diff --git a/dstack/kms/dstack-app/builder/build-image.sh b/dstack/kms/dstack-app/builder/build-image.sh index f126fe0ea..e5e3ec476 100755 --- a/dstack/kms/dstack-app/builder/build-image.sh +++ b/dstack/kms/dstack-app/builder/build-image.sh @@ -16,11 +16,11 @@ export CONTEXT_DIR DOCKERFILE # shellcheck source=/dev/null source "$REPO_ROOT/dstack/build/shared/build-lib.sh" -NAME=${1:-} -if [ -z "$NAME" ]; then - echo "Usage: $0 <image-name>[:<tag>]" >&2 +if [ "$#" -eq 0 ]; then + echo "Usage: $0 <image-name>[:<tag>]..." >&2 exit 1 fi +TAGS=$(printf '%s\n' "$@") NO_CACHE=${NO_CACHE:-} GIT_REV=${GIT_REV:-HEAD} @@ -32,7 +32,13 @@ ensure_buildkit touch "$SHARED_DIR/builder-pinned-packages.txt" touch "$SHARED_DIR/pinned-packages.txt" -docker_build "$NAME" "" "$SHARED_DIR/pinned-packages.txt" +METADATA=$(image_metadata \ + "dstack-kms" \ + "Key management service for dstack confidential applications" \ + "dstack/kms" \ + "dstack/kms/README.md") + +docker_build "$TAGS" "" "$SHARED_DIR/pinned-packages.txt" "$METADATA" docker_build "kms-builder-temp" "kms-builder" "$SHARED_DIR/builder-pinned-packages.txt" check_clean_tree "$SHARED_DIR" diff --git a/dstack/verifier/builder/Dockerfile b/dstack/verifier/builder/Dockerfile index 7a70a588b..58f36bd95 100644 --- a/dstack/verifier/builder/Dockerfile +++ b/dstack/verifier/builder/Dockerfile @@ -31,19 +31,6 @@ RUN cd repo/dstack && cargo build --release -p dstack-verifier --target x86_64-u RUN echo "${DSTACK_REV}" > /build/.GIT_REV FROM debian:bookworm@sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe -ARG DSTACK_REV -ARG IMAGE_VERSION=dev -ARG IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack -LABEL org.opencontainers.image.title="dstack-verifier" \ - org.opencontainers.image.description="Remote attestation verification service for dstack" \ - org.opencontainers.image.source="${IMAGE_SOURCE_URL}" \ - org.opencontainers.image.revision="${DSTACK_REV}" \ - org.opencontainers.image.version="${IMAGE_VERSION}" \ - org.opencontainers.image.url="${IMAGE_SOURCE_URL}/tree/${DSTACK_REV}/dstack/verifier" \ - org.opencontainers.image.documentation="${IMAGE_SOURCE_URL}/blob/${DSTACK_REV}/dstack/verifier/README.md" \ - org.opencontainers.image.licenses="Apache-2.0" \ - org.opencontainers.image.base.name="docker.io/library/debian:bookworm" \ - org.opencontainers.image.base.digest="sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe" COPY --from=build-shared pin-packages.sh /build/ COPY builder/shared/pinned-packages.txt /build/ WORKDIR /build @@ -55,18 +42,11 @@ RUN ./pin-packages.sh ./pinned-packages.txt && \ && rm -rf /var/lib/apt/lists/* /var/log/* /var/cache/ldconfig/aux-cache COPY --from=verifier-builder /build/repo/dstack/target/x86_64-unknown-linux-musl/release/dstack-verifier /usr/local/bin/dstack-verifier COPY --from=verifier-builder /build/.GIT_REV /etc/ -RUN mkdir -p /etc/dstack-verifier && printf '%s\n' \ - "org.opencontainers.image.title=dstack-verifier" \ - "org.opencontainers.image.description=Remote attestation verification service for dstack" \ - "org.opencontainers.image.source=${IMAGE_SOURCE_URL}" \ - "org.opencontainers.image.revision=${DSTACK_REV}" \ - "org.opencontainers.image.version=${IMAGE_VERSION}" \ - "org.opencontainers.image.url=${IMAGE_SOURCE_URL}/tree/${DSTACK_REV}/dstack/verifier" \ - "org.opencontainers.image.documentation=${IMAGE_SOURCE_URL}/blob/${DSTACK_REV}/dstack/verifier/README.md" \ - "org.opencontainers.image.licenses=Apache-2.0" \ - "org.opencontainers.image.base.name=docker.io/library/debian:bookworm" \ - "org.opencontainers.image.base.digest=sha256:0d8498a0e9e6a60011df39aab78534cfe940785e7c59d19dfae1eb53ea59babe" \ - > /etc/dstack-verifier/build-info +# The OCI metadata is generated once by build-lib.sh and injected here, as +# the config labels and as the manifest annotations, so the three views of +# the image can never disagree. +ARG IMAGE_METADATA +RUN mkdir -p /etc/dstack-verifier && printf '%s\n' "${IMAGE_METADATA}" > /etc/dstack-verifier/build-info RUN mkdir -p /etc/dstack COPY dstack-verifier.toml /etc/dstack/dstack-verifier.toml WORKDIR /var/lib/dstack-verifier diff --git a/dstack/verifier/builder/build-image.sh b/dstack/verifier/builder/build-image.sh index d9db1288e..dfe1ebddc 100755 --- a/dstack/verifier/builder/build-image.sh +++ b/dstack/verifier/builder/build-image.sh @@ -16,11 +16,11 @@ export CONTEXT_DIR DOCKERFILE # shellcheck source=/dev/null source "$REPO_ROOT/dstack/build/shared/build-lib.sh" -NAME=${1:-} -if [ -z "$NAME" ]; then - echo "Usage: $0 <image-name>[:<tag>]" >&2 +if [ "$#" -eq 0 ]; then + echo "Usage: $0 <image-name>[:<tag>]..." >&2 exit 1 fi +TAGS=$(printf '%s\n' "$@") NO_CACHE=${NO_CACHE:-} GIT_REV=${GIT_REV:-HEAD} @@ -33,7 +33,13 @@ mkdir -p "$SHARED_DIR" touch "$SHARED_DIR/builder-pinned-packages.txt" touch "$SHARED_DIR/pinned-packages.txt" -docker_build "$NAME" "" "$SHARED_DIR/pinned-packages.txt" +METADATA=$(image_metadata \ + "dstack-verifier" \ + "Remote attestation verification service for dstack" \ + "dstack/verifier" \ + "dstack/verifier/README.md") + +docker_build "$TAGS" "" "$SHARED_DIR/pinned-packages.txt" "$METADATA" docker_build "verifier-builder-temp" "verifier-builder" "$SHARED_DIR/builder-pinned-packages.txt" check_clean_tree "$SHARED_DIR" From a81350ab605ee3dc80a5f89673f66b2b3bfa9f9d Mon Sep 17 00:00:00 2001 From: Kevin Wang <wy721@qq.com> Date: Mon, 7 Sep 2026 20:11:21 -0700 Subject: [PATCH 4/5] docs(images): document reproducing a released image --- dstack/gateway/dstack-app/builder/README.md | 81 +++++++++++++-------- dstack/kms/dstack-app/builder/README.md | 48 +++++++++++- dstack/verifier/builder/README.md | 78 ++++++++++++++++++++ 3 files changed, 177 insertions(+), 30 deletions(-) create mode 100644 dstack/verifier/builder/README.md diff --git a/dstack/gateway/dstack-app/builder/README.md b/dstack/gateway/dstack-app/builder/README.md index a49a139e0..b4f63bf56 100644 --- a/dstack/gateway/dstack-app/builder/README.md +++ b/dstack/gateway/dstack-app/builder/README.md @@ -1,11 +1,14 @@ -# dstack KMS Builder +# dstack Gateway Builder -This directory contains the necessary files to build and run the dstack-kms Docker image for development. +This directory contains the files needed to build the dstack-gateway Docker +image reproducibly. ## Overview -The builder creates a Docker image that includes: -- The dstack-kms service compiled from Rust source code +The builder produces a Docker image containing the dstack-gateway service +compiled from Rust source. The build pins its base images by digest, pins the +Debian package set (`shared/pinned-packages.txt`), and normalizes timestamps, so +the same revision always yields the same image. ## Prerequisites @@ -14,41 +17,61 @@ The builder creates a Docker image that includes: ## Building the Image -To build the KMS Docker image, use the provided `build-image.sh` script: - ```bash -./build-image.sh <image-name>[:<tag>] +./build-image.sh <image-name>[:<tag>]... ``` For example: + ```bash -./build-image.sh kvin/kms +./build-image.sh my-org/dstack-gateway:dev ``` -## Running the Built Image +Optional environment variables: + +| Variable | Purpose | +| --- | --- | +| `GIT_REV` | Revision to build (default `HEAD`) | +| `IMAGE_VERSION` | Version recorded in the image metadata (default `dev`) | +| `IMAGE_SOURCE_URL` | Repository URL recorded in the image metadata | +| `NO_CACHE` | Set to any value to build without the layer cache | +| `OCI_TAR` | Also write an OCI archive here, for digest comparison | +| `PUSH` | Set to any value to push the tags instead of only loading them | + +## Reproducing a released image + +Release CI runs this same script, so a published image can be rebuilt and +checked digest-for-digest. Pass the release tag's revision and version: + +```bash +GIT_REV=gateway-v0.6.0 \ +IMAGE_VERSION=0.6.0 \ +IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack \ +OCI_TAR=/tmp/gateway.oci.tar \ + ./build-image.sh dstacktee/dstack-gateway:0.6.0 -### Using Docker Compose - -The easiest way to run the KMS service is using the provided `docker-compose.yaml`: - -```yaml -services: - kms: - image: kvin/kms - ports: - - "8003:8000" - volumes: - - /var/run/docker.sock:/var/run/docker.sock - - ./kms:/kms - environment: - - IMAGE_DOWNLOAD_URL=${IMAGE_DOWNLOAD_URL:-http://localhost:8001/mr_{OS_IMAGE_HASH}.tar.gz} - - AUTH_TYPE=dev - - DEV_DOMAIN=kms.1022.dstack.org - - QUOTE_ENABLED=false +python3 -c 'import json,tarfile;t=tarfile.open("/tmp/gateway.oci.tar");print(json.load(t.extractfile("index.json"))["manifests"][0]["digest"])' ``` -To start the service: +`IMAGE_VERSION` is part of the image metadata, so it must match the release for +the digests to match. The printed digest is what the registry reports for +`dstacktee/dstack-gateway:0.6.0`; compare it with: ```bash -docker-compose up +docker buildx imagetools inspect dstacktee/dstack-gateway:0.6.0 --format '{{.Manifest.Digest}}' ``` + +## Image metadata + +The image carries its provenance as OCI metadata in three places, all generated +from one definition in `dstack/build/shared/build-lib.sh`: + +- config labels — `docker inspect -f '{{json .Config.Labels}}' <image>` +- manifest annotations — `docker buildx imagetools inspect <image>` +- `/etc/dstack-gateway/build-info` inside the image, readable from within the CVM + +## Running the Built Image + +The gateway is normally deployed as a dstack app; see +[`../deploy-to-vmm.sh`](../deploy-to-vmm.sh) and the +[gateway README](../../README.md). diff --git a/dstack/kms/dstack-app/builder/README.md b/dstack/kms/dstack-app/builder/README.md index 093b8b21c..8e4709125 100644 --- a/dstack/kms/dstack-app/builder/README.md +++ b/dstack/kms/dstack-app/builder/README.md @@ -18,7 +18,7 @@ The builder creates a Docker image that includes: To build the KMS Docker image, use the provided `build-image.sh` script: ```bash -./build-image.sh <image-name>[:<tag>] +./build-image.sh <image-name>[:<tag>]... ``` For example: @@ -26,6 +26,52 @@ For example: ./build-image.sh kvin/kms ``` +Optional environment variables: + +| Variable | Purpose | +| --- | --- | +| `GIT_REV` | Revision to build (default `HEAD`) | +| `IMAGE_VERSION` | Version recorded in the image metadata (default `dev`) | +| `IMAGE_SOURCE_URL` | Repository URL recorded in the image metadata | +| `NO_CACHE` | Set to any value to build without the layer cache | +| `OCI_TAR` | Also write an OCI archive here, for digest comparison | +| `PUSH` | Set to any value to push the tags instead of only loading them | + +## Reproducing a released image + +Release CI runs this same script, so a published image can be rebuilt and +checked digest-for-digest. Pass the release tag's revision and version: + +```bash +GIT_REV=kms-v0.6.0 \ +IMAGE_VERSION=0.6.0 \ +IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack \ +OCI_TAR=/tmp/kms.oci.tar \ + ./build-image.sh dstacktee/dstack-kms:0.6.0 + +python3 -c 'import json,tarfile;t=tarfile.open("/tmp/kms.oci.tar");print(json.load(t.extractfile("index.json"))["manifests"][0]["digest"])' +``` + +`IMAGE_VERSION` is part of the image metadata, so it must match the release for +the digests to match. The printed digest is what the registry reports for +`dstacktee/dstack-kms:0.6.0`; compare it with: + +```bash +docker buildx imagetools inspect dstacktee/dstack-kms:0.6.0 --format '{{.Manifest.Digest}}' +``` + +This is the digest that `deploy-to-vmm.sh` pins in `KMS_IMAGE`, and that in turn +feeds the compose hash registered on chain. + +## Image metadata + +The image carries its provenance as OCI metadata in three places, all generated +from one definition in `dstack/build/shared/build-lib.sh`: + +- config labels — `docker inspect -f '{{json .Config.Labels}}' <image>` +- manifest annotations — `docker buildx imagetools inspect <image>` +- `/etc/dstack-kms/build-info` inside the image, readable from within the CVM + ## Running the Built Image ### Using Docker Compose diff --git a/dstack/verifier/builder/README.md b/dstack/verifier/builder/README.md new file mode 100644 index 000000000..74d2b813d --- /dev/null +++ b/dstack/verifier/builder/README.md @@ -0,0 +1,78 @@ +# dstack Verifier Builder + +This directory contains the files needed to build the dstack-verifier Docker +image reproducibly. + +## Overview + +The builder produces a Docker image containing the dstack-verifier service +compiled from Rust source. The build pins its base images by digest, pins the +Debian package set (`shared/pinned-packages.txt`), and normalizes timestamps, so +the same revision always yields the same image. + +## Prerequisites + +- Docker with BuildKit support (v20.10.0+) +- Git + +## Building the Image + +```bash +./build-image.sh <image-name>[:<tag>]... +``` + +For example: + +```bash +./build-image.sh my-org/dstack-verifier:dev +``` + +Optional environment variables: + +| Variable | Purpose | +| --- | --- | +| `GIT_REV` | Revision to build (default `HEAD`) | +| `IMAGE_VERSION` | Version recorded in the image metadata (default `dev`) | +| `IMAGE_SOURCE_URL` | Repository URL recorded in the image metadata | +| `NO_CACHE` | Set to any value to build without the layer cache | +| `OCI_TAR` | Also write an OCI archive here, for digest comparison | +| `PUSH` | Set to any value to push the tags instead of only loading them | + +## Reproducing a released image + +Release CI runs this same script, so a published image can be rebuilt and +checked digest-for-digest. Pass the release tag's revision and version: + +```bash +GIT_REV=verifier-v0.6.0 \ +IMAGE_VERSION=0.6.0 \ +IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack \ +OCI_TAR=/tmp/verifier.oci.tar \ + ./build-image.sh dstacktee/dstack-verifier:0.6.0 + +python3 -c 'import json,tarfile;t=tarfile.open("/tmp/verifier.oci.tar");print(json.load(t.extractfile("index.json"))["manifests"][0]["digest"])' +``` + +`IMAGE_VERSION` is part of the image metadata, so it must match the release for +the digests to match. The printed digest is what the registry reports for +`dstacktee/dstack-verifier:0.6.0`; compare it with: + +```bash +docker buildx imagetools inspect dstacktee/dstack-verifier:0.6.0 --format '{{.Manifest.Digest}}' +``` + +Note that the release also publishes a `:latest` tag pointing at the same +digest. + +## Image metadata + +The image carries its provenance as OCI metadata in three places, all generated +from one definition in `dstack/build/shared/build-lib.sh`: + +- config labels — `docker inspect -f '{{json .Config.Labels}}' <image>` +- manifest annotations — `docker buildx imagetools inspect <image>` +- `/etc/dstack-verifier/build-info` inside the image, readable from within the CVM + +## Running the Built Image + +See the [verifier README](../README.md) for configuration and the HTTP API. From c42903027970db4cee671b4d6a2a7353884507a7 Mon Sep 17 00:00:00 2001 From: Kevin Wang <wy721@qq.com> Date: Mon, 7 Sep 2026 20:44:32 -0700 Subject: [PATCH 5/5] fix(images): validate builds before publishing OCI manifests --- .github/workflows/docker-build-check.yml | 13 + .github/workflows/gateway-release.yml | 36 ++- .github/workflows/kms-release.yml | 44 +-- .github/workflows/verifier-release.yml | 39 ++- dstack/build/shared/build-lib.sh | 100 +++++-- dstack/build/shared/tests/test_build_lib.py | 268 ++++++++++++++++++ dstack/gateway/dstack-app/builder/README.md | 17 +- .../gateway/dstack-app/builder/build-image.sh | 7 +- dstack/kms/dstack-app/builder/README.md | 17 +- dstack/kms/dstack-app/builder/build-image.sh | 7 +- dstack/verifier/builder/README.md | 17 +- dstack/verifier/builder/build-image.sh | 7 +- 12 files changed, 477 insertions(+), 95 deletions(-) create mode 100644 dstack/build/shared/tests/test_build_lib.py diff --git a/.github/workflows/docker-build-check.yml b/.github/workflows/docker-build-check.yml index 34da220fc..967e66a45 100644 --- a/.github/workflows/docker-build-check.yml +++ b/.github/workflows/docker-build-check.yml @@ -16,18 +16,31 @@ on: - 'dstack/**' - 'rust-toolchain.toml' - '.github/workflows/docker-build-check.yml' + - '.github/workflows/kms-release.yml' + - '.github/workflows/gateway-release.yml' + - '.github/workflows/verifier-release.yml' pull_request: branches: [ next, 'release/**' ] paths: - 'dstack/**' - 'rust-toolchain.toml' - '.github/workflows/docker-build-check.yml' + - '.github/workflows/kms-release.yml' + - '.github/workflows/gateway-release.yml' + - '.github/workflows/verifier-release.yml' env: DSTACK_REV: ${{ github.event.pull_request.head.sha || github.sha }} DSTACK_SRC_URL: ${{ github.event.pull_request.head.repo.clone_url || format('{0}/{1}', github.server_url, github.repository) }} jobs: + build-script-tests: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v5 + - name: Test release build orchestration + run: python3 -m unittest discover -s dstack/build/shared/tests -v + gateway: runs-on: ubuntu-latest steps: diff --git a/.github/workflows/gateway-release.yml b/.github/workflows/gateway-release.yml index 524284b0d..998209bc5 100644 --- a/.github/workflows/gateway-release.yml +++ b/.github/workflows/gateway-release.yml @@ -8,7 +8,7 @@ on: workflow_dispatch: inputs: version: - description: Release version without the gateway-v prefix + description: Existing release tag version without the gateway-v prefix required: true type: string push: @@ -24,23 +24,30 @@ jobs: build-and-release: runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v5 - - name: Parse and check version env: DISPATCH_VERSION: ${{ inputs.version }} run: | - case "${GITHUB_REF}" in - refs/tags/gateway-v*) VERSION=${GITHUB_REF#refs/tags/gateway-v} ;; - *) VERSION=${DISPATCH_VERSION} ;; - esac - if [ -z "${VERSION}" ]; then - echo "This release workflow needs a gateway-v* tag or a version input; got ${GITHUB_REF}." >&2 + if [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then + VERSION=$DISPATCH_VERSION + else + case "$GITHUB_REF" in + refs/tags/gateway-v*) VERSION=${GITHUB_REF#refs/tags/gateway-v} ;; + *) echo "Expected a gateway-v* tag" >&2; exit 1 ;; + esac + fi + if [[ ! "$VERSION" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then + echo "Version must be a valid Docker tag" >&2 exit 1 fi - echo "VERSION=${VERSION}" >> "$GITHUB_ENV" - echo "Parsed version: ${VERSION}" + echo "VERSION=$VERSION" >> "$GITHUB_ENV" + echo "RELEASE_TAG=gateway-v$VERSION" >> "$GITHUB_ENV" + + # Manual runs rebuild an existing release tag, not the selected branch. + - name: Checkout repository + uses: actions/checkout@v5 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && env.RELEASE_TAG || github.sha }} - name: Log in to Docker Hub uses: docker/login-action@v3 @@ -64,11 +71,12 @@ jobs: METADATA_FILE: ${{ runner.temp }}/build-metadata.json GIT_REV: ${{ env.GIT_REV }} IMAGE_VERSION: ${{ env.VERSION }} + IMAGE_NAME: ${{ vars.DOCKERHUB_ORG }}/dstack-gateway:${{ env.VERSION }} IMAGE_SOURCE_URL: ${{ github.server_url }}/${{ github.repository }} DSTACK_SRC_URL: ${{ github.server_url }}/${{ github.repository }}.git run: | dstack/gateway/dstack-app/builder/build-image.sh \ - ${{ vars.DOCKERHUB_ORG }}/dstack-gateway:${{ env.VERSION }} + "$IMAGE_NAME" digest=$(jq -r '."containerimage.digest"' "$METADATA_FILE") if [ -z "$digest" ] || [ "$digest" = null ]; then echo "the build did not report an image digest" >&2 @@ -86,6 +94,8 @@ jobs: - name: GitHub Release uses: softprops/action-gh-release@v2 with: + tag_name: ${{ env.RELEASE_TAG }} + target_commitish: ${{ env.GIT_REV }} name: "Gateway Release v${{ env.VERSION }}" body: | ## Docker Image Information diff --git a/.github/workflows/kms-release.yml b/.github/workflows/kms-release.yml index 6ed9deb5f..122d886b6 100644 --- a/.github/workflows/kms-release.yml +++ b/.github/workflows/kms-release.yml @@ -8,7 +8,7 @@ on: workflow_dispatch: inputs: version: - description: Release version without the kms-v prefix + description: Existing release tag version without the kms-v prefix required: true type: string push: @@ -24,8 +24,30 @@ jobs: build-and-release: runs-on: ubuntu-latest steps: + - name: Parse and check version + env: + DISPATCH_VERSION: ${{ inputs.version }} + run: | + if [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then + VERSION=$DISPATCH_VERSION + else + case "$GITHUB_REF" in + refs/tags/kms-v*) VERSION=${GITHUB_REF#refs/tags/kms-v} ;; + *) echo "Expected a kms-v* tag" >&2; exit 1 ;; + esac + fi + if [[ ! "$VERSION" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then + echo "Version must be a valid Docker tag" >&2 + exit 1 + fi + echo "VERSION=$VERSION" >> "$GITHUB_ENV" + echo "RELEASE_TAG=kms-v$VERSION" >> "$GITHUB_ENV" + + # Manual runs rebuild an existing release tag, not the selected branch. - name: Checkout repository uses: actions/checkout@v5 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && env.RELEASE_TAG || github.sha }} - name: Checkout contract dependencies run: | @@ -34,21 +56,6 @@ jobs: dstack/kms/auth-eth/lib/openzeppelin-contracts-upgradeable \ dstack/kms/auth-eth/lib/openzeppelin-foundry-upgrades - - name: Parse and check version - env: - DISPATCH_VERSION: ${{ inputs.version }} - run: | - case "${GITHUB_REF}" in - refs/tags/kms-v*) VERSION=${GITHUB_REF#refs/tags/kms-v} ;; - *) VERSION=${DISPATCH_VERSION} ;; - esac - if [ -z "${VERSION}" ]; then - echo "This release workflow needs a kms-v* tag or a version input; got ${GITHUB_REF}." >&2 - exit 1 - fi - echo "VERSION=${VERSION}" >> "$GITHUB_ENV" - echo "Parsed version: ${VERSION}" - - name: Log in to Docker Hub uses: docker/login-action@v3 with: @@ -71,11 +78,12 @@ jobs: METADATA_FILE: ${{ runner.temp }}/build-metadata.json GIT_REV: ${{ env.GIT_REV }} IMAGE_VERSION: ${{ env.VERSION }} + IMAGE_NAME: ${{ vars.DOCKERHUB_ORG }}/dstack-kms:${{ env.VERSION }} IMAGE_SOURCE_URL: ${{ github.server_url }}/${{ github.repository }} DSTACK_SRC_URL: ${{ github.server_url }}/${{ github.repository }}.git run: | dstack/kms/dstack-app/builder/build-image.sh \ - ${{ vars.DOCKERHUB_ORG }}/dstack-kms:${{ env.VERSION }} + "$IMAGE_NAME" digest=$(jq -r '."containerimage.digest"' "$METADATA_FILE") if [ -z "$digest" ] || [ "$digest" = null ]; then echo "the build did not report an image digest" >&2 @@ -102,6 +110,8 @@ jobs: - name: GitHub Release uses: softprops/action-gh-release@v2 with: + tag_name: ${{ env.RELEASE_TAG }} + target_commitish: ${{ env.GIT_REV }} name: "KMS Release v${{ env.VERSION }}" files: | dstack/kms/auth-eth/out/DstackKms.sol/DstackKms.json diff --git a/.github/workflows/verifier-release.yml b/.github/workflows/verifier-release.yml index b242c6d5a..857a6382c 100644 --- a/.github/workflows/verifier-release.yml +++ b/.github/workflows/verifier-release.yml @@ -8,7 +8,7 @@ on: workflow_dispatch: inputs: version: - description: Release version without the verifier-v prefix + description: Existing release tag version without the verifier-v prefix required: true type: string push: @@ -24,23 +24,30 @@ jobs: build-and-release: runs-on: ubuntu-latest steps: - - name: Checkout repository - uses: actions/checkout@v5 - - name: Parse and check version env: DISPATCH_VERSION: ${{ inputs.version }} run: | - case "${GITHUB_REF}" in - refs/tags/verifier-v*) VERSION=${GITHUB_REF#refs/tags/verifier-v} ;; - *) VERSION=${DISPATCH_VERSION} ;; - esac - if [ -z "${VERSION}" ]; then - echo "This release workflow needs a verifier-v* tag or a version input; got ${GITHUB_REF}." >&2 + if [ "$GITHUB_EVENT_NAME" = workflow_dispatch ]; then + VERSION=$DISPATCH_VERSION + else + case "$GITHUB_REF" in + refs/tags/verifier-v*) VERSION=${GITHUB_REF#refs/tags/verifier-v} ;; + *) echo "Expected a verifier-v* tag" >&2; exit 1 ;; + esac + fi + if [[ ! "$VERSION" =~ ^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$ ]]; then + echo "Version must be a valid Docker tag" >&2 exit 1 fi - echo "VERSION=${VERSION}" >> "$GITHUB_ENV" - echo "Parsed version: ${VERSION}" + echo "VERSION=$VERSION" >> "$GITHUB_ENV" + echo "RELEASE_TAG=verifier-v$VERSION" >> "$GITHUB_ENV" + + # Manual runs rebuild an existing release tag, not the selected branch. + - name: Checkout repository + uses: actions/checkout@v5 + with: + ref: ${{ github.event_name == 'workflow_dispatch' && env.RELEASE_TAG || github.sha }} - name: Log in to Docker Hub uses: docker/login-action@v3 @@ -64,12 +71,14 @@ jobs: METADATA_FILE: ${{ runner.temp }}/build-metadata.json GIT_REV: ${{ env.GIT_REV }} IMAGE_VERSION: ${{ env.VERSION }} + IMAGE_NAME: ${{ vars.DOCKERHUB_ORG }}/dstack-verifier:${{ env.VERSION }} + LATEST_IMAGE_NAME: ${{ vars.DOCKERHUB_ORG }}/dstack-verifier:latest IMAGE_SOURCE_URL: ${{ github.server_url }}/${{ github.repository }} DSTACK_SRC_URL: ${{ github.server_url }}/${{ github.repository }}.git run: | dstack/verifier/builder/build-image.sh \ - ${{ vars.DOCKERHUB_ORG }}/dstack-verifier:${{ env.VERSION }} \ - ${{ vars.DOCKERHUB_ORG }}/dstack-verifier:latest + "$IMAGE_NAME" \ + "$LATEST_IMAGE_NAME" digest=$(jq -r '."containerimage.digest"' "$METADATA_FILE") if [ -z "$digest" ] || [ "$digest" = null ]; then echo "the build did not report an image digest" >&2 @@ -87,6 +96,8 @@ jobs: - name: GitHub Release uses: softprops/action-gh-release@v2 with: + tag_name: ${{ env.RELEASE_TAG }} + target_commitish: ${{ env.GIT_REV }} name: "Verifier Release v${{ env.VERSION }}" body: | ## Docker Image Information diff --git a/dstack/build/shared/build-lib.sh b/dstack/build/shared/build-lib.sh index cc9e8fc07..51c7dcb50 100755 --- a/dstack/build/shared/build-lib.sh +++ b/dstack/build/shared/build-lib.sh @@ -22,7 +22,7 @@ # Release CI passes the tag version; pass the same value to # reproduce a published image. # IMAGE_SOURCE_URL - repository URL recorded in the image metadata -# PUSH - non-empty to push the image instead of loading it +# PUSH - non-empty to publish after local validation # OCI_TAR - path to also write an OCI archive to. The manifest digest # of that archive is what a registry reports, so this is the # way to check a local rebuild against a published digest. @@ -98,17 +98,19 @@ image_metadata() { "org.opencontainers.image.base.digest=$base_digest" } -# docker_build <tags> [target] [pkg_list_file] [metadata] +# docker_build <tags> [target] [pkg_list_file] [metadata] [mode] # # tags - newline-separated list of image tags # target - build stage to stop at, empty for the final image # pkg_list_file - where to record the installed Debian packages # metadata - "key=value" lines from image_metadata, empty to record none +# mode - load (default), or export after package validation docker_build() { local tags=$1 local target=${2:-} local pkg_list_file=${3:-} local metadata=${4:-} + local mode=${5:-load} local commit_timestamp commit_timestamp=$(git -C "$REPO_ROOT" show -s --format=%ct "$GIT_REV") @@ -145,36 +147,53 @@ docker_build() { args+=(--build-arg "IMAGE_METADATA=$metadata") fi - # Only the final image is publishable. Intermediate stages are built purely - # to extract their package lists and must never reach a registry. - # - # oci-mediatypes keeps the pushed manifest byte-identical to the OCI archive - # a local rebuild produces, which is what makes the digests comparable. - if [ -n "${PUSH:-}" ] && [ -z "$target" ]; then - args+=(--output "type=image,push=true,oci-mediatypes=true,rewrite-timestamp=true") - fi - if [ -n "${OCI_TAR:-}" ] && [ -z "$target" ]; then - args+=(--output "type=oci,oci-mediatypes=true,rewrite-timestamp=true,dest=$OCI_TAR") - fi - # Always load locally as well: extract_packages inspects the built image. - args+=(--output "type=docker,rewrite-timestamp=true") - # Guard this like the outputs above: the intermediate stage builds run after - # the final one and would otherwise overwrite its reported digest. - if [ -n "${METADATA_FILE:-}" ] && [ -z "$target" ]; then - args+=(--metadata-file "$METADATA_FILE") - fi - - if [ -n "${NO_CACHE:-}" ]; then - args+=(--no-cache) - fi + # A validation build has only a Docker exporter. Never mix it with an OCI + # exporter: on classic Docker stores its digest overwrites the OCI digest in + # BuildKit's metadata response. Export the validated, cached result separately. + local outputs=() + case "$mode" in + load) + outputs+=("type=docker,rewrite-timestamp=true") + if [ -n "${NO_CACHE:-}" ]; then + args+=(--no-cache) + fi + ;; + export) + if [ -n "$target" ] || [ -n "$pkg_list_file" ]; then + echo "only validated final images may be exported" >&2 + return 1 + fi + if [ -n "${OCI_TAR:-}" ]; then + outputs+=("type=oci,oci-mediatypes=true,rewrite-timestamp=true,dest=$OCI_TAR") + fi + if [ -n "${PUSH:-}" ]; then + outputs+=("type=image,push=true,oci-mediatypes=true,rewrite-timestamp=true") + fi + if [ "${#outputs[@]}" -eq 0 ]; then + # METADATA_FILE alone still reports the registry-compatible OCI + # digest, not the manifest produced by a classic Docker exporter. + outputs+=("type=image,push=false,oci-mediatypes=true,rewrite-timestamp=true") + fi + if [ -n "${METADATA_FILE:-}" ]; then + args+=(--metadata-file "$METADATA_FILE") + fi + ;; + *) + echo "unknown build mode: $mode" >&2 + return 1 + ;; + esac if [ -n "$target" ]; then args+=(--target "$target") fi - docker buildx build "${args[@]}" \ - --file "$DOCKERFILE" \ - "$CONTEXT_DIR" + local output + for output in "${outputs[@]}"; do + docker buildx build "${args[@]}" --output "$output" \ + --file "$DOCKERFILE" \ + "$CONTEXT_DIR" + done extract_packages "$(head -n1 <<<"$tags")" "$pkg_list_file" } @@ -191,3 +210,30 @@ check_clean_tree() { exit 1 fi } + +# Build and validate all package lists before exporting or updating remote tags. +# The export reuses the validated cache even when NO_CACHE was requested for the +# validation builds. check_clean_tree ensures extraction did not change inputs. +build_component() { + local tags=$1 + local builder_tag=$2 + local builder_target=$3 + local shared_dir=$4 + local metadata=$5 + + if [ -n "${PUSH:-}${OCI_TAR:-}${METADATA_FILE:-}" ]; then + # Do not let package extraction erase pre-existing changes and make an + # unvalidated set of build inputs appear clean before publication. + check_clean_tree "$shared_dir" + fi + + # Build the intermediate stage first: with NO_CACHE it must not replace the + # final image's cached dependencies between validation and export. + docker_build "$builder_tag" "$builder_target" "$shared_dir/builder-pinned-packages.txt" + docker_build "$tags" "" "$shared_dir/pinned-packages.txt" "$metadata" + check_clean_tree "$shared_dir" + + if [ -n "${PUSH:-}${OCI_TAR:-}${METADATA_FILE:-}" ]; then + docker_build "$tags" "" "" "$metadata" export + fi +} diff --git a/dstack/build/shared/tests/test_build_lib.py b/dstack/build/shared/tests/test_build_lib.py new file mode 100644 index 000000000..4e907a619 --- /dev/null +++ b/dstack/build/shared/tests/test_build_lib.py @@ -0,0 +1,268 @@ +# SPDX-FileCopyrightText: © 2026 Phala Network <dstack@phala.network> +# +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise release orchestration without Docker or network access. + +Run with: python3 -m unittest discover -s dstack/build/shared/tests -v +""" + +import json +import os +import shutil +import subprocess +import sys +import tempfile +import textwrap +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[4] +COMPONENTS = ("kms/dstack-app", "gateway/dstack-app", "verifier") +MOCK_DOCKER = r""" +import json +import os +import sys + +args = sys.argv[1:] +with open(os.environ["CALLS"], "a") as log: + log.write(json.dumps(args) + "\n") +if args[:2] == ["buildx", "build"]: + stage = "builder" if "--target" in args else "final" + if os.environ.get("FAIL_BUILD") == stage: + sys.exit(1) + if "--metadata-file" in args: + with open(args[args.index("--metadata-file") + 1], "w") as out: + json.dump({"containerimage.digest": "sha256:" + "1" * 64}, out) +elif args[0] == "run": + stage = "builder" if args[4].endswith("-builder-temp") else "final" + if os.environ.get("FAIL_EXTRACT") == stage: + sys.exit(1) + print("package=" + ("2" if os.environ.get("DRIFT") == stage else "1")) +""" + + +class BuildTests(unittest.TestCase): + """Check the real component entry points with a recording Docker executable.""" + + def setUp(self): + """Create a minimal tracked checkout and an annotated release tag.""" + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + shared = self.root / "dstack/build/shared" + shared.mkdir(parents=True) + shutil.copy(ROOT / "dstack/build/shared/build-lib.sh", shared) + for component in COMPONENTS: + builder = self.root / "dstack" / component / "builder" + (builder / "shared").mkdir(parents=True) + shutil.copy(ROOT / "dstack" / component / "builder/build-image.sh", builder) + (builder / "Dockerfile").write_text( + "FROM debian:bookworm@sha256:" + "0" * 64 + "\n" + ) + for name in ("pinned-packages.txt", "builder-pinned-packages.txt"): + (builder / "shared" / name).write_text("package=1\n") + self.git("init", "-q") + self.git("add", ".") + tree = self.git("write-tree") + # Synthetic objects avoid depending on or overriding any Git identity. + identity = "Build Fixture <fixture@example.invalid> 1700000000 +0000" + self.commit = self.git( + "hash-object", + "-t", + "commit", + "-w", + "--stdin", + input=f"tree {tree}\nauthor {identity}\ncommitter {identity}\n\nFixture\n", + ) + self.git("update-ref", "HEAD", self.commit) + tag = self.git( + "hash-object", + "-t", + "tag", + "-w", + "--stdin", + input=f"object {self.commit}\ntype commit\ntag release\ntagger {identity}\n\nRelease\n", + ) + self.git("update-ref", "refs/tags/release", tag) + bin_dir = self.root / "bin" + bin_dir.mkdir() + docker = bin_dir / "docker" + docker.write_text(f"#!{sys.executable}\n" + MOCK_DOCKER) + docker.chmod(0o755) + self.calls = self.root / "calls.jsonl" + self.metadata = self.root / "metadata.json" + self.env = { + **os.environ, + "PATH": f"{bin_dir}:{os.environ['PATH']}", + "CALLS": str(self.calls), + "GIT_REV": "release", + "IMAGE_VERSION": "1.0", + "PUSH": "", + "OCI_TAR": "", + "METADATA_FILE": "", + "NO_CACHE": "", + "FAIL_BUILD": "", + "FAIL_EXTRACT": "", + "DRIFT": "", + } + + def git(self, *args, input=None): + """Run Git only against the disposable fixture repository.""" + return subprocess.check_output( + ["git", "-C", str(self.root), *args], + input=input, + text=True, + ).strip() + + def run_build(self, component="kms/dstack-app", **env): + """Run a real entry point and return its result and recorded build calls.""" + self.calls.unlink(missing_ok=True) + result = subprocess.run( + [ + "bash", + str(self.root / "dstack" / component / "builder/build-image.sh"), + "example/probe:1.0", + "example/probe:latest", + ], + env={**self.env, **env}, + text=True, + capture_output=True, + ) + calls = [json.loads(line) for line in self.calls.read_text().splitlines()] + builds = [call for call in calls if call[:2] == ["buildx", "build"]] + return result, builds + + def test_local_build_peels_annotated_tags(self): + """All components resolve annotated tags and load only the two stages.""" + for component in COMPONENTS: + with self.subTest(component=component): + result, builds = self.run_build(component) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(builds), 2) + for build in builds: + self.assertIn(f"DSTACK_REV={self.commit}", build) + self.assertIn("SOURCE_DATE_EPOCH=1700000000", build) + self.assertEqual(build.count("--output"), 1) + self.assertIn("type=docker,rewrite-timestamp=true", build) + self.assertNotIn("--metadata-file", build) + + def test_exports_are_separate_and_follow_validation(self): + """A no-cache request does not force a fresh build after validation.""" + result, builds = self.run_build( + PUSH="1", + OCI_TAR=str(self.root / "image.tar"), + METADATA_FILE=str(self.metadata), + NO_CACHE="1", + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(builds), 4) + self.assertIn("--target", builds[0]) + self.assertNotIn("--target", builds[1]) + for build in builds[:2]: + self.assertIn("--no-cache", build) + self.assertNotIn("--metadata-file", build) + for build in builds[2:]: + self.assertEqual(build.count("--output"), 1) + self.assertNotIn("--no-cache", build) + self.assertNotIn("--target", build) + self.assertNotIn("type=docker,rewrite-timestamp=true", build) + self.assertIn("--metadata-file", build) + self.assertIn("example/probe:1.0", build) + self.assertIn("example/probe:latest", build) + self.assertTrue( + builds[2][builds[2].index("--output") + 1].startswith("type=oci,") + ) + self.assertIn( + "type=image,push=true,oci-mediatypes=true,rewrite-timestamp=true", builds[3] + ) + self.assertTrue(self.metadata.exists()) + + def test_failures_never_publish(self): + """Build errors, extraction errors and either drifting list block export.""" + for variable in ("FAIL_BUILD", "FAIL_EXTRACT", "DRIFT"): + for stage in ("final", "builder"): + with self.subTest(variable=variable, stage=stage): + self.git("restore", ".") + result, builds = self.run_build( + PUSH="1", + METADATA_FILE=str(self.metadata), + **{variable: stage}, + ) + self.assertNotEqual(result.returncode, 0) + for build in builds: + self.assertIn("type=docker,rewrite-timestamp=true", build) + self.assertNotIn("--metadata-file", build) + self.assertFalse(self.metadata.exists()) + + def test_dirty_inputs_block_publication_before_building(self): + """Extraction must not be allowed to erase dirty publication inputs.""" + pins = self.root / "dstack/kms/dstack-app/builder/shared/pinned-packages.txt" + pins.write_text("package=2\n") + result, builds = self.run_build(PUSH="1") + self.assertNotEqual(result.returncode, 0) + self.assertEqual(builds, []) + self.assertEqual(pins.read_text(), "package=2\n") + + def test_metadata_only_uses_oci_image_exporter(self): + """Requesting metadata alone never reports a Docker schema manifest.""" + result, builds = self.run_build(METADATA_FILE=str(self.metadata)) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(len(builds), 3) + self.assertIn( + "type=image,push=false,oci-mediatypes=true,rewrite-timestamp=true", + builds[-1], + ) + + +class WorkflowTests(unittest.TestCase): + """Exercise the shell version parser from each release workflow.""" + + def test_version_validation_and_release_refs(self): + """Accept release versions, reject shell syntax, and bind checkout/release tags.""" + for component in ("kms", "gateway", "verifier"): + workflow = (ROOT / f".github/workflows/{component}-release.yml").read_text() + parser = textwrap.dedent( + workflow.split(" run: |\n", 1)[1].split("\n\n", 1)[0] + ) + self.assertIn("tag_name: ${{ env.RELEASE_TAG }}", workflow) + self.assertIn("target_commitish: ${{ env.GIT_REV }}", workflow) + self.assertIn("&& env.RELEASE_TAG || github.sha", workflow) + for event in ("push", "workflow_dispatch"): + for version in ( + "1.0-rc1", + "", + "bad;echo injected", + "bad\nNAME=value", + "x" * 129, + ): + with self.subTest( + component=component, event=event, version=version + ): + with tempfile.NamedTemporaryFile() as output: + result = subprocess.run( + ["bash", "-euc", parser], + capture_output=True, + text=True, + env={ + **os.environ, + "GITHUB_EVENT_NAME": event, + "GITHUB_REF": f"refs/tags/{component}-v{version}" + if event == "push" + else "refs/heads/next", + "DISPATCH_VERSION": version, + "GITHUB_ENV": output.name, + }, + ) + self.assertEqual( + result.returncode == 0, version == "1.0-rc1" + ) + if result.returncode == 0: + self.assertIn( + f"RELEASE_TAG={component}-v{version}\n", + Path(output.name).read_text(), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/dstack/gateway/dstack-app/builder/README.md b/dstack/gateway/dstack-app/builder/README.md index b4f63bf56..18e074452 100644 --- a/dstack/gateway/dstack-app/builder/README.md +++ b/dstack/gateway/dstack-app/builder/README.md @@ -12,7 +12,7 @@ the same revision always yields the same image. ## Prerequisites -- Docker with BuildKit support (v20.10.0+) +- Docker with Buildx v0.13.0+ (the script creates a BuildKit v0.20.2 builder) - Git ## Building the Image @@ -36,15 +36,26 @@ Optional environment variables: | `IMAGE_SOURCE_URL` | Repository URL recorded in the image metadata | | `NO_CACHE` | Set to any value to build without the layer cache | | `OCI_TAR` | Also write an OCI archive here, for digest comparison | +| `METADATA_FILE` | Write the validated OCI manifest digest and build metadata here | | `PUSH` | Set to any value to push the tags instead of only loading them | +Publication and OCI export happen only after both package lists pass validation. +`NO_CACHE` applies to the validation builds; export then reuses their cached result. +Manual release workflows require an existing component release tag and check out +that tag, rather than building the branch selected in the workflow UI. + ## Reproducing a released image Release CI runs this same script, so a published image can be rebuilt and -checked digest-for-digest. Pass the release tag's revision and version: +checked digest-for-digest. Use a clean checkout of the release commit so the +Dockerfile, package lists, shared scripts and copied files also match the release. +From the repository root (fetch the tag first if it is not available locally): ```bash -GIT_REV=gateway-v0.6.0 \ +git switch --detach "gateway-v0.6.0^{commit}" +cd dstack/gateway/dstack-app/builder + +GIT_REV=HEAD \ IMAGE_VERSION=0.6.0 \ IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack \ OCI_TAR=/tmp/gateway.oci.tar \ diff --git a/dstack/gateway/dstack-app/builder/build-image.sh b/dstack/gateway/dstack-app/builder/build-image.sh index 612cac220..1b1992c57 100755 --- a/dstack/gateway/dstack-app/builder/build-image.sh +++ b/dstack/gateway/dstack-app/builder/build-image.sh @@ -24,7 +24,7 @@ TAGS=$(printf '%s\n' "$@") NO_CACHE=${NO_CACHE:-} GIT_REV=${GIT_REV:-HEAD} -GIT_REV=$(git -C "$REPO_ROOT" rev-parse "$GIT_REV") +GIT_REV=$(git -C "$REPO_ROOT" rev-parse --verify "${GIT_REV}^{commit}") DSTACK_SRC_URL=${DSTACK_SRC_URL:-https://github.com/Dstack-TEE/dstack.git} ensure_buildkit @@ -38,7 +38,4 @@ METADATA=$(image_metadata \ "dstack/gateway" \ "dstack/gateway/README.md") -docker_build "$TAGS" "" "$SHARED_DIR/pinned-packages.txt" "$METADATA" -docker_build "gateway-builder-temp" "gateway-builder" "$SHARED_DIR/builder-pinned-packages.txt" - -check_clean_tree "$SHARED_DIR" +build_component "$TAGS" "gateway-builder-temp" "gateway-builder" "$SHARED_DIR" "$METADATA" diff --git a/dstack/kms/dstack-app/builder/README.md b/dstack/kms/dstack-app/builder/README.md index 8e4709125..923b34ebc 100644 --- a/dstack/kms/dstack-app/builder/README.md +++ b/dstack/kms/dstack-app/builder/README.md @@ -10,7 +10,7 @@ The builder creates a Docker image that includes: ## Prerequisites -- Docker with BuildKit support (v20.10.0+) +- Docker with Buildx v0.13.0+ (the script creates a BuildKit v0.20.2 builder) - Git ## Building the Image @@ -35,15 +35,26 @@ Optional environment variables: | `IMAGE_SOURCE_URL` | Repository URL recorded in the image metadata | | `NO_CACHE` | Set to any value to build without the layer cache | | `OCI_TAR` | Also write an OCI archive here, for digest comparison | +| `METADATA_FILE` | Write the validated OCI manifest digest and build metadata here | | `PUSH` | Set to any value to push the tags instead of only loading them | +Publication and OCI export happen only after both package lists pass validation. +`NO_CACHE` applies to the validation builds; export then reuses their cached result. +Manual release workflows require an existing component release tag and check out +that tag, rather than building the branch selected in the workflow UI. + ## Reproducing a released image Release CI runs this same script, so a published image can be rebuilt and -checked digest-for-digest. Pass the release tag's revision and version: +checked digest-for-digest. Use a clean checkout of the release commit so the +Dockerfile, package lists, shared scripts and copied files also match the release. +From the repository root (fetch the tag first if it is not available locally): ```bash -GIT_REV=kms-v0.6.0 \ +git switch --detach "kms-v0.6.0^{commit}" +cd dstack/kms/dstack-app/builder + +GIT_REV=HEAD \ IMAGE_VERSION=0.6.0 \ IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack \ OCI_TAR=/tmp/kms.oci.tar \ diff --git a/dstack/kms/dstack-app/builder/build-image.sh b/dstack/kms/dstack-app/builder/build-image.sh index e5e3ec476..c04c54f1d 100755 --- a/dstack/kms/dstack-app/builder/build-image.sh +++ b/dstack/kms/dstack-app/builder/build-image.sh @@ -24,7 +24,7 @@ TAGS=$(printf '%s\n' "$@") NO_CACHE=${NO_CACHE:-} GIT_REV=${GIT_REV:-HEAD} -GIT_REV=$(git -C "$REPO_ROOT" rev-parse "$GIT_REV") +GIT_REV=$(git -C "$REPO_ROOT" rev-parse --verify "${GIT_REV}^{commit}") DSTACK_SRC_URL=${DSTACK_SRC_URL:-https://github.com/Dstack-TEE/dstack.git} ensure_buildkit @@ -38,7 +38,4 @@ METADATA=$(image_metadata \ "dstack/kms" \ "dstack/kms/README.md") -docker_build "$TAGS" "" "$SHARED_DIR/pinned-packages.txt" "$METADATA" -docker_build "kms-builder-temp" "kms-builder" "$SHARED_DIR/builder-pinned-packages.txt" - -check_clean_tree "$SHARED_DIR" +build_component "$TAGS" "kms-builder-temp" "kms-builder" "$SHARED_DIR" "$METADATA" diff --git a/dstack/verifier/builder/README.md b/dstack/verifier/builder/README.md index 74d2b813d..9e409e925 100644 --- a/dstack/verifier/builder/README.md +++ b/dstack/verifier/builder/README.md @@ -12,7 +12,7 @@ the same revision always yields the same image. ## Prerequisites -- Docker with BuildKit support (v20.10.0+) +- Docker with Buildx v0.13.0+ (the script creates a BuildKit v0.20.2 builder) - Git ## Building the Image @@ -36,15 +36,26 @@ Optional environment variables: | `IMAGE_SOURCE_URL` | Repository URL recorded in the image metadata | | `NO_CACHE` | Set to any value to build without the layer cache | | `OCI_TAR` | Also write an OCI archive here, for digest comparison | +| `METADATA_FILE` | Write the validated OCI manifest digest and build metadata here | | `PUSH` | Set to any value to push the tags instead of only loading them | +Publication and OCI export happen only after both package lists pass validation. +`NO_CACHE` applies to the validation builds; export then reuses their cached result. +Manual release workflows require an existing component release tag and check out +that tag, rather than building the branch selected in the workflow UI. + ## Reproducing a released image Release CI runs this same script, so a published image can be rebuilt and -checked digest-for-digest. Pass the release tag's revision and version: +checked digest-for-digest. Use a clean checkout of the release commit so the +Dockerfile, package lists, shared scripts and copied files also match the release. +From the repository root (fetch the tag first if it is not available locally): ```bash -GIT_REV=verifier-v0.6.0 \ +git switch --detach "verifier-v0.6.0^{commit}" +cd dstack/verifier/builder + +GIT_REV=HEAD \ IMAGE_VERSION=0.6.0 \ IMAGE_SOURCE_URL=https://github.com/Dstack-TEE/dstack \ OCI_TAR=/tmp/verifier.oci.tar \ diff --git a/dstack/verifier/builder/build-image.sh b/dstack/verifier/builder/build-image.sh index dfe1ebddc..3530a80b0 100755 --- a/dstack/verifier/builder/build-image.sh +++ b/dstack/verifier/builder/build-image.sh @@ -24,7 +24,7 @@ TAGS=$(printf '%s\n' "$@") NO_CACHE=${NO_CACHE:-} GIT_REV=${GIT_REV:-HEAD} -GIT_REV=$(git -C "$REPO_ROOT" rev-parse "$GIT_REV") +GIT_REV=$(git -C "$REPO_ROOT" rev-parse --verify "${GIT_REV}^{commit}") DSTACK_SRC_URL=${DSTACK_SRC_URL:-https://github.com/Dstack-TEE/dstack.git} ensure_buildkit @@ -39,7 +39,4 @@ METADATA=$(image_metadata \ "dstack/verifier" \ "dstack/verifier/README.md") -docker_build "$TAGS" "" "$SHARED_DIR/pinned-packages.txt" "$METADATA" -docker_build "verifier-builder-temp" "verifier-builder" "$SHARED_DIR/builder-pinned-packages.txt" - -check_clean_tree "$SHARED_DIR" +build_component "$TAGS" "verifier-builder-temp" "verifier-builder" "$SHARED_DIR" "$METADATA"