fix: harden release builds and helmfile hydration

.github/workflows/internal--image-builder.yaml

@@ -12,6 +12,11 @@ on:
         required: true
         type: string
         description: 'Directory containing the Dockerfile'
+      ref:
+        required: false
+        type: string
+        default: ''
+        description: 'Git ref to build (empty uses the default ref of the triggering event)'
       app-id:
         required: true
         type: string
@@ -49,6 +54,7 @@ jobs:
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
+          ref: ${{ inputs.ref }}
 
       - name: Check for Dockerfile
         id: dockerfile
@@ -80,7 +86,7 @@ jobs:
           tags: |
             type=sha
             type=ref,event=pr
-            type=raw,value=latest,enable={{is_default_branch}}
+            type=raw,value=latest,enable=${{ inputs.ref == '' && '{{is_default_branch}}' || 'false' }}
             type=raw,value=${{ github.actor }}
             type=semver,pattern={{raw}},value=${{ inputs.semver-tag }},enable=${{ inputs.semver-tag != '' }}
             type=raw,value=${{ inputs.release-tag }},enable=${{ inputs.release-tag != '' }}

.github/workflows/reusable--label-base-deploy--resolver.yaml

@@ -122,29 +122,45 @@ jobs:
   helmfile-targets-group:
     name: 'Group Helmfile Targets by Environment'
     needs: resolve
-    if: |
-      needs.resolve.outputs.has-targets == 'true' &&
-      contains(needs.resolve.outputs.targets, '"type":"helmfile"')
+    if: needs.resolve.result == 'success'
     runs-on: ubuntu-latest
     outputs:
       environments: ${{ steps.group.outputs.environments }}
     steps:
+      - name: Checkout
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          persist-credentials: false
+
+      - name: Checkout shared workflows
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: ${{ job.workflow_repository }}
+          ref: ${{ job.workflow_sha }}
+          path: .shared-workflows
+          persist-credentials: false
+
       - name: Group by environment
         id: group
         env:
           TARGETS: ${{ needs.resolve.outputs.targets }}
         run: |
           set -euo pipefail
-          grouped=$(echo "$TARGETS" | jq -c '
+          orphaned=$(bash .shared-workflows/scripts/kubernetes-hydrate/orphan-environments.sh)
+          # TARGETS is empty when no pull request was found, and jq emits nothing for empty input.
+          grouped=$(jq -c --argjson orphaned "$orphaned" '
             [.[] | select(.type == "helmfile")]
             | group_by(.environment)
             | map({environment: .[0].environment, services: [.[].service]})
-          ')
+            | . + (($orphaned - map(.environment)) | map({environment: ., services: []}))
+            | sort_by(.environment)
+          ' <<<"${TARGETS:-[]}")
           echo "environments=$grouped" >> "$GITHUB_OUTPUT"
 
   deploy-helmfile-hydrate:
     name: 'Hydrate Helmfile (${{ matrix.env.environment }})'
     needs: helmfile-targets-group
+    if: needs.helmfile-targets-group.outputs.environments != '[]'
     strategy:
       matrix:
         env: ${{ fromJson(needs.helmfile-targets-group.outputs.environments) }}

.github/workflows/reusable--release--build.yaml

@@ -22,22 +22,25 @@ jobs:
       tag: ${{ steps.parse.outputs.tag }}
       working-directory: ${{ steps.resolve-dir.outputs.working-directory }}
     steps:
+      - name: Checkout shared workflows
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: ${{ job.workflow_repository }}
+          ref: ${{ job.workflow_sha }}
+          path: .shared-workflows
+          persist-credentials: false
+
       - name: Parse tag
         id: parse
         env:
           # `inputs` in a reusable workflow holds only its own workflow_call inputs, so the caller's dispatch input `tag` is read from the event.
           TAG: ${{ github.event.release.tag_name || github.event.inputs.tag }}
         run: |
           set -euo pipefail
-          if [[ ! "$TAG" =~ ^([a-z-]+)-v([0-9]+\.[0-9]+\.[0-9]+)$ ]]; then
-            echo "::error::Tag '$TAG' does not match expected pattern <service>-v<X.Y.Z>"
-            exit 1
-          fi
-          SERVICE="${BASH_REMATCH[1]}"
-          VERSION="v${BASH_REMATCH[2]}"
+          service=$(bash .shared-workflows/scripts/terragrunt-release/release-service.sh --tag "$TAG")
           {
-            echo "service=$SERVICE"
-            echo "version=$VERSION"
+            echo "service=$service"
+            echo "version=${TAG#"$service"-}"
             echo "tag=$TAG"
           } >> "$GITHUB_OUTPUT"
       # Container targets carry no environment, so environments is not passed.
@@ -71,6 +74,8 @@ jobs:
     with:
       image-name: ${{ needs.detect-component.outputs.service }}
       working-directory: ${{ needs.detect-component.outputs.working-directory }}
+      # The ref of a dispatched run is not the source of the release.
+      ref: ${{ needs.detect-component.outputs.tag }}
       app-id: ${{ inputs.app-id }}
       semver-tag: ${{ needs.detect-component.outputs.version }}
       release-tag: ${{ needs.detect-component.outputs.tag }}

scripts/kubernetes-hydrate/hydrate-component.sh

@@ -9,6 +9,12 @@ cd "$(git rev-parse --show-toplevel)"
 component_dir="kubernetes/components/${component}/${env}"
 out_dir="kubernetes/manifests/${env}/${component}"
 
+# Flux with prune: true would delete the live resources of an empty manifest.
+if [ ! -f "${component_dir}/helmfile.yaml" ] && [ ! -d "${component_dir}/kustomization" ]; then
+    echo "hydrate-component.sh: ${component_dir} has neither helmfile.yaml nor kustomization/; refusing to write an empty manifest" >&2
+    exit 1
+fi
+
 # Explicit --kube-version prevents template divergence caused by varying default helm binary versions.
 env_hcl="aws/eks/${env}/env.hcl"
 if [ ! -f "${env_hcl}" ]; then

scripts/kubernetes-hydrate/orphan-environments.sh

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+cd "$(git rev-parse --show-toplevel)"
+
+# An orphan is what hydrate-index.sh removes, so the two loops must agree.
+for env_dir in kubernetes/manifests/*/; do
+    [ -d "${env_dir}" ] || continue
+    env=$(basename "${env_dir}")
+    for dir in "${env_dir}"*/; do
+        [ -d "${dir}" ] || continue
+        name=$(basename "${dir}")
+        [ "${name}" = "00-namespaces" ] && continue
+        if [ ! -d "kubernetes/components/${name}/${env}" ]; then
+            printf '%s\n' "${env}"
+            break
+        fi
+    done
+done | jq -Rsc 'split("\n") | map(select(length > 0))'

scripts/kubernetes-hydrate/render-diff.sh

@@ -29,6 +29,11 @@ git worktree add -q --detach "${work}/head" HEAD
 
 render() {
     local side="$1" svc="$2" env="$3" dir="$4"
+    # hydrate-component.sh refuses a missing component, which here means the side before it was added or after it was deleted.
+    if [ ! -d "${work}/${side}/kubernetes/components/${svc}/${env}" ]; then
+        : > "${dir}/${side}.yaml"
+        return 0
+    fi
     (cd "${work}/${side}" && bash "${script_dir}/hydrate-component.sh" "${svc}" "${env}") > "${dir}/${side}.log" 2>&1 &&
         cp "${work}/${side}/kubernetes/manifests/${env}/${svc}/manifest.yaml" "${dir}/${side}.yaml"
 }

scripts/kubernetes-hydrate/tests/hydrate-component.test.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+here="$(cd "$(dirname "$0")" && pwd)"
+scripts="${here}/.."
+work="$(mktemp -d)"
+trap 'rm -rf "${work}"' EXIT
+failures=0
+
+pass() { echo "ok - $1"; }
+fail() { echo "not ok - $1"; failures=$((failures + 1)); }
+expect_contains() { if grep -qF -- "$2" "$3"; then pass "$1"; else fail "$1"; sed 's/^/# /' "$3"; fi; }
+expect_failure() { if [ "$2" -ne 0 ]; then pass "$1"; else fail "$1"; fi; }
+expect_success() { if [ "$2" -eq 0 ]; then pass "$1"; else fail "$1"; sed 's/^/# /' "$3"; fi; }
+expect_file() { if [ "$(cat "$2")" = "$3" ]; then pass "$1"; else fail "$1"; sed 's/^/# /' "$2"; fi; }
+expect_missing() { if [ ! -e "$2" ]; then pass "$1"; else fail "$1"; fi; }
+
+repo="${work}/repo"
+live="${repo}/kubernetes/manifests/production/namespace-only/manifest.yaml"
+mkdir -p "${repo}/aws/eks/production" "$(dirname "${live}")" "${repo}/kubernetes/components/namespace-only/production"
+printf 'locals {\n  cluster_version = "1.33"\n}\n' > "${repo}/aws/eks/production/env.hcl"
+printf 'kind: Namespace\n' > "${repo}/kubernetes/components/namespace-only/production/namespace.yaml"
+printf 'kind: Deployment\n' > "${live}"
+git -C "${repo}" init -q -b main
+git -C "${repo}" -c user.name=test -c user.email=test@example.com add -A
+git -C "${repo}" -c user.name=test -c user.email=test@example.com commit -q -m base
+
+hydrate() {
+    local status=0
+    (cd "${repo}" && bash "${scripts}/hydrate-component.sh" "$@") > "${work}/stdout" 2>&1 || status=$?
+    echo "${status}"
+}
+
+status="$(hydrate namespace-only production)"
+expect_failure "a component with neither input fails" "${status}"
+expect_contains "the failure names the directory" "kubernetes/components/namespace-only/production" "${work}/stdout"
+expect_file "the failure keeps the existing manifest" "${live}" "kind: Deployment"
+
+status="$(hydrate missing production)"
+expect_failure "a component without a directory fails" "${status}"
+expect_contains "the failure names the missing directory" "kubernetes/components/missing/production" "${work}/stdout"
+expect_missing "the failure writes no manifest directory" "${repo}/kubernetes/manifests/production/missing"
+
+bin="${work}/bin"
+mkdir -p "${bin}"
+printf '#!/usr/bin/env bash\necho "kind: HelmRendered"\n' > "${bin}/helmfile"
+chmod +x "${bin}/helmfile"
+mkdir -p "${repo}/kubernetes/components/chart/production"
+touch "${repo}/kubernetes/components/chart/production/helmfile.yaml"
+status="$(PATH="${bin}:${PATH}" hydrate chart production)"
+expect_success "a component with helmfile.yaml hydrates" "${status}" "${work}/stdout"
+expect_contains "the helmfile output reaches the manifest" "kind: HelmRendered" "${repo}/kubernetes/manifests/production/chart/manifest.yaml"
+
+mkdir -p "${repo}/kubernetes/components/plain/production/kustomization"
+printf 'resources:\n  - configmap.yaml\n' > "${repo}/kubernetes/components/plain/production/kustomization/kustomization.yaml"
+printf 'apiVersion: v1\nkind: ConfigMap\nmetadata:\n  name: plain\n' > "${repo}/kubernetes/components/plain/production/kustomization/configmap.yaml"
+status="$(hydrate plain production)"
+expect_success "a component with kustomization/ hydrates" "${status}" "${work}/stdout"
+expect_contains "the kustomize output reaches the manifest" "kind: ConfigMap" "${repo}/kubernetes/manifests/production/plain/manifest.yaml"
+
+exit "${failures}"

scripts/kubernetes-hydrate/tests/orphan-environments.test.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+here="$(cd "$(dirname "$0")" && pwd)"
+scripts="${here}/.."
+work="$(mktemp -d)"
+trap 'rm -rf "${work}"' EXIT
+failures=0
+
+pass() { echo "ok - $1"; }
+fail() { echo "not ok - $1"; failures=$((failures + 1)); }
+expect_eq() { if [ "$2" = "$3" ]; then pass "$1"; else fail "$1"; echo "# expected: $2"; echo "# actual:   $3"; fi; }
+
+new_repo() {
+    local repo="${work}/$1"
+    mkdir -p "${repo}"
+    git -C "${repo}" init -q -b main
+    echo "${repo}"
+}
+
+add_dirs() {
+    local repo="$1" dir
+    shift
+    for dir in "$@"; do
+        mkdir -p "${repo}/kubernetes/${dir}"
+        touch "${repo}/kubernetes/${dir}/file.yaml"
+    done
+}
+
+orphans() {
+    (cd "$1" && bash "${scripts}/orphan-environments.sh")
+}
+
+repo="$(new_repo no-kubernetes)"
+expect_eq "a repository without kubernetes/manifests yields an empty list" "[]" "$(orphans "${repo}")"
+
+repo="$(new_repo components-only)"
+add_dirs "${repo}" components/app/production
+expect_eq "components without manifests yield an empty list" "[]" "$(orphans "${repo}")"
+
+repo="$(new_repo clean)"
+add_dirs "${repo}" components/app/production manifests/production/app manifests/production/00-namespaces
+expect_eq "matching manifests yield an empty list" "[]" "$(orphans "${repo}")"
+
+repo="$(new_repo namespaces-only)"
+add_dirs "${repo}" manifests/production/00-namespaces
+expect_eq "00-namespaces is not an orphan" "[]" "$(orphans "${repo}")"
+
+repo="$(new_repo file-in-environment)"
+add_dirs "${repo}" components/app/production manifests/production/app
+touch "${repo}/kubernetes/manifests/production/kustomization.yaml"
+expect_eq "a file in the environment directory is not an orphan" "[]" "$(orphans "${repo}")"
+
+repo="$(new_repo one-orphan)"
+add_dirs "${repo}" components/app/production manifests/production/app manifests/production/gone
+expect_eq "a manifest without a component is an orphan" '["production"]' "$(orphans "${repo}")"
+
+repo="$(new_repo several-orphans)"
+add_dirs "${repo}" manifests/production/a manifests/production/b manifests/develop/a manifests/staging/00-namespaces
+expect_eq "each environment with an orphan appears once" '["develop","production"]' "$(orphans "${repo}")"
+
+repo="$(new_repo other-environment)"
+add_dirs "${repo}" components/app/develop manifests/production/app
+expect_eq "a component in another environment does not cover the manifest" '["production"]' "$(orphans "${repo}")"
+
+expect_eq "the result does not depend on the working directory" '["production"]' "$(mkdir -p "${repo}/nested" && cd "${repo}/nested" && bash "${scripts}/orphan-environments.sh")"
+
+repo="$(new_repo matches-hydrate-index)"
+add_dirs "${repo}" components/app/production manifests/production/app manifests/production/gone components/app/develop manifests/develop/app
+(cd "${repo}" && bash "${scripts}/hydrate-index.sh" production)
+if [ -d "${repo}/kubernetes/manifests/production/gone" ]; then fail "hydrate-index removes the reported orphan"; else pass "hydrate-index removes the reported orphan"; fi
+expect_eq "hydrate-index leaves no orphan to report" "[]" "$(orphans "${repo}")"
+
+exit "${failures}"

scripts/kubernetes-hydrate/tests/render-diff.test.sh

@@ -44,6 +44,7 @@ git -C "${repo}" -c user.name=test -c user.email=test@example.com commit -q -m b
 write_deployment "${repo}/kubernetes/components/demo/production" demo nginx:1.1
 echo "# changed" >> "${repo}/kubernetes/components/demo/namespace.yaml"
 git -C "${repo}" rm -q -r kubernetes/components/gone
+write_deployment "${repo}/kubernetes/components/added/production" added alpine:3
 write_deployment "${repo}/kubernetes/components/pending-only/pending" pending-only busybox:1.36
 write_deployment "${repo}/kubernetes/components/broken/production" broken busybox:1.36
 printf 'resources:\n  - missing.yaml\n' > "${repo}/kubernetes/components/broken/production/kustomization/kustomization.yaml"
@@ -60,6 +61,7 @@ expect_contains "links the full diff" "production/demo/manifest.diff" "${summary
 expect_contains "writes the full diff" "+      - image: nginx:1.1" "${out}/production/demo/manifest.diff"
 expect_contains "lists a deleted component" "## gone (production)" "${summary}"
 expect_contains "reports resources of a deleted component as removed" "- Deployment gone" "${summary}"
+expect_contains "reports resources of an added component as added" "- Deployment added" "${summary}"
 expect_contains "reports a render failure instead of aborting" "Rendering failed (head)" "${summary}"
 expect_contains "skips directories that are not hydrated environments" "Not rendered:" "${summary}"
 if [ -d "${out}/pending" ]; then fail "does not render directories that are not hydrated environments"; else pass "does not render directories that are not hydrated environments"; fi

scripts/terragrunt-release/release-service.sh

@@ -4,13 +4,16 @@ set -euo pipefail
 mode="${1:-}"
 value="${2:-}"
 
+# The service name rule of deploy-actions' Entities::Service.name_format.
+service_rule='[a-zA-Z0-9_-]+'
+
 case "${mode}" in
     --tag)
-        pattern='^([a-z0-9-]+)-v[0-9]+\.[0-9]+\.[0-9]+$'
+        pattern="^(${service_rule})-v[0-9]+\.[0-9]+\.[0-9]+\$"
         expected='<service>-vX.Y.Z'
         ;;
     --branch)
-        pattern='^release-please--branches--main--components--([a-z0-9-]+)$'
+        pattern="^release-please--branches--main--components--(${service_rule})\$"
         expected='release-please--branches--main--components--<service>'
         ;;
     *)

scripts/terragrunt-release/test.sh

@@ -38,6 +38,26 @@ test_release_service_parses_tag() {
     assert_eq "monolith" "$(bash "${script_dir}/release-service.sh" --tag monolith-v1.2.3)"
 }
 
+test_release_service_keeps_the_hyphens_of_a_tag_service() {
+    assert_eq "github-oidc-auth" "$(bash "${script_dir}/release-service.sh" --tag github-oidc-auth-v1.2.3)"
+    assert_eq "api-v2" "$(bash "${script_dir}/release-service.sh" --tag api-v2-v1.2.3)"
+}
+
+test_release_service_accepts_every_service_name_character() {
+    local service
+    for service in route53 oauth2-proxy my_service MyService 2fa; do
+        assert_eq "${service}" "$(bash "${script_dir}/release-service.sh" --tag "${service}-v1.2.3")"
+        assert_eq "${service}" "$(bash "${script_dir}/release-service.sh" --branch "release-please--branches--main--components--${service}")"
+    done
+}
+
+test_release_service_rejects_other_service_name_characters() {
+    assert_fails bash "${script_dir}/release-service.sh" --tag a.b-v1.2.3
+    assert_stderr_contains "::error::"
+    assert_fails bash "${script_dir}/release-service.sh" --branch release-please--branches--main--components--a.b
+    assert_stderr_contains "::error::"
+}
+
 test_release_service_rejects_root_tag() {
     assert_fails bash "${script_dir}/release-service.sh" --tag v0.1.0
     assert_stderr_contains "::error::"