From 43cbc73108908a074f32746ce49245500f7bcfa2 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 19 Nov 2026 10:42:45 +0100 Subject: [PATCH 01/15] feat(pulp): add shared Pulp delivery actions Import package-delivery-pulp, promote-to-stable-pulp, pulp-repository-properties, setup-pulp-cli and the pulp script helpers from centreon/centreon-collect/centreon-modules (identical copies) under pulp/, made self-contained so they can be called from another repository. Refs: MON-215510 --- README.md | 59 +- pulp/lib/api.sh | 292 ++++++++ pulp/lib/check-common.sh | 198 +++++ pulp/lib/check-deb.sh | 123 ++++ pulp/lib/check-rpm.sh | 150 ++++ pulp/lib/manifest.sh | 57 ++ pulp/package-delivery/action.yml | 222 ++++++ pulp/package-delivery/deliver-deb.sh | 679 ++++++++++++++++++ pulp/package-delivery/deliver-rpm.sh | 256 +++++++ .../ensure-feature-repository.sh | 80 +++ pulp/promote-to-stable/action.yml | 168 +++++ pulp/promote-to-stable/promote-deb.sh | 488 +++++++++++++ pulp/promote-to-stable/promote-rpm.sh | 138 ++++ pulp/repository-properties/action.yml | 146 ++++ pulp/repository-properties/parse-distrib.sh | 26 + pulp/repository-properties/properties.sh | 281 ++++++++ pulp/setup-cli/action.yml | 29 + pulp/setup-cli/setup.sh | 78 ++ 18 files changed, 3469 insertions(+), 1 deletion(-) create mode 100644 pulp/lib/api.sh create mode 100644 pulp/lib/check-common.sh create mode 100755 pulp/lib/check-deb.sh create mode 100755 pulp/lib/check-rpm.sh create mode 100644 pulp/lib/manifest.sh create mode 100644 pulp/package-delivery/action.yml create mode 100755 pulp/package-delivery/deliver-deb.sh create mode 100755 pulp/package-delivery/deliver-rpm.sh create mode 100755 pulp/package-delivery/ensure-feature-repository.sh create mode 100644 pulp/promote-to-stable/action.yml create mode 100755 pulp/promote-to-stable/promote-deb.sh create mode 100755 pulp/promote-to-stable/promote-rpm.sh create mode 100644 pulp/repository-properties/action.yml create mode 100644 pulp/repository-properties/parse-distrib.sh create mode 100755 pulp/repository-properties/properties.sh create mode 100644 pulp/setup-cli/action.yml create mode 100755 pulp/setup-cli/setup.sh diff --git a/README.md b/README.md index f0b7c9b..18f672b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,59 @@ # github-actions -Shared GitHub Actions used by Centreon CI workflows + +Shared GitHub Actions used by Centreon CI workflows. + +## Usage + +Reference an action by its path in this repository, pinned to a full commit SHA: + +```yaml +- uses: centreon/github-actions/pulp/package-delivery@ # +``` + +Each action is self-contained: it never calls a sibling action through `uses: ./...` +(that path would resolve in the caller's workspace). Shared logic lives in scripts +called through `${{ github.action_path }}`, so every file comes from the same commit. + +## Pulp delivery + +| Action | Purpose | +| ---------------------------- | --------------------------------------------------------------------------------------------- | +| `pulp/package-delivery` | Deliver rpm/deb packages from the actions cache to an unstable, testing or feature repository | +| `pulp/promote-to-stable` | Promote testing packages to the stable repository | +| `pulp/repository-properties` | Compute repository names, base paths, suites and domains of a delivery | +| `pulp/setup-cli` | Install pulp-cli and authenticate with a GitHub OIDC token | +| `pulp/lib` | Shell helpers (REST api, manifest, post-delivery checks), not an action | + +Requirements of the calling job: + +- `permissions: id-token: write` (OIDC authentication to Pulp). +- An internal runner: the Pulp api is not reachable from GitHub-hosted runners. +- An OIDC rule for the calling repository in the Pulp configuration. +- Inputs `pulp_url`, `pulp_content_url` and `audience` wired from the `PULP_API_URL`, + `PULP_CONTENT_URL` and `PULP_OIDC_AUDIENCE` organization variables. + +`repository_name` selects the repository family: `standard`, `standard-internal`, +`business`, `business-internal`, `plugins` or `connectors`. + +Example: + +```yaml +deliver-pulp: + runs-on: centreon-ubuntu-26.04-small + permissions: + contents: read + id-token: write + steps: + - uses: centreon/github-actions/pulp/package-delivery@ # main + with: + module_name: centreon-web + distrib: el9 + version: "26.10" + cache_key: ${{ github.sha }}-${{ github.run_id }}-rpm-el9 + stability: testing + release_type: release + is_cloud: "false" + pulp_url: ${{ vars.PULP_API_URL }} + pulp_content_url: ${{ vars.PULP_CONTENT_URL }} + audience: ${{ vars.PULP_OIDC_AUDIENCE }} +``` diff --git a/pulp/lib/api.sh b/pulp/lib/api.sh new file mode 100644 index 0000000..5c4dddb --- /dev/null +++ b/pulp/lib/api.sh @@ -0,0 +1,292 @@ +#!/usr/bin/env bash +# Shared pulp api helpers for the delivery/promotion scripts. Source this file +# like manifest.sh; PULP_URL and PULP_TOKEN must be set by the caller. +# +# DOMAIN_ENABLED requires every viewset URL to carry a domain segment, no +# unprefixed fallback; every domain-scoped REST call in this file is prefixed with "/$PULP_DOMAIN" +PULP_DOMAIN="${PULP_DOMAIN:-default}" + +# forces pulp-cli's config to update immediately, unlike a plain PULP_DOMAIN +# reassignment: refresh_pulp_token only re-runs "pulp config create" once the +# token itself goes stale (>240s), which could leave pulp-cli on the old domain that long +switch_pulp_domain() { + PULP_DOMAIN="$1" + if command -v pulp >/dev/null 2>&1; then + if ! pulp config create --overwrite --base-url "$PULP_URL" --api-root "/" \ + --domain "$PULP_DOMAIN" \ + --header "Authorization:Bearer $PULP_TOKEN" --timeout 0 >/dev/null 2>&1; then + echo "::error::Cannot switch the pulp-cli profile to domain $PULP_DOMAIN" >&2 + return 1 + fi + fi +} + +# the github actions oidc token expires ~5 minutes after issuance, which is +# shorter than a large delivery: refresh it before it goes stale whenever the +# job can mint tokens (id-token: write context + audience exported by +# setup-pulp-cli). The fresh token is propagated to the following steps +# (GITHUB_ENV) and to the pulp-cli config so the pulp commands keep working. +refresh_pulp_token() { + local now token + now=$(date +%s) + if ((now - ${PULP_TOKEN_ISSUED_AT:-0} < 240)); then + return 0 + fi + if [[ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" || -z "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:-}" || -z "${PULP_OIDC_AUDIENCE:-}" ]]; then + return 0 + fi + token=$( + curl -fsSL --retry 5 --retry-delay 3 --retry-all-errors \ + -H "Authorization: bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + -G --data-urlencode "audience=$PULP_OIDC_AUDIENCE" \ + "$ACTIONS_ID_TOKEN_REQUEST_URL" 2>/dev/null | jq -r '.value' 2>/dev/null + ) || token="" + if [[ -z "$token" || "$token" == "null" ]]; then + return 0 + fi + PULP_TOKEN="$token" + PULP_TOKEN_ISSUED_AT="$now" + # written to stderr: this function also runs inside command substitutions + # (pulp_upload), where a stdout echo would corrupt the captured output + echo "::add-mask::$token" >&2 + if [[ -n "${GITHUB_ENV:-}" ]]; then + { + echo "PULP_TOKEN=$token" + echo "PULP_TOKEN_ISSUED_AT=$now" + } >> "$GITHUB_ENV" + fi + if command -v pulp >/dev/null 2>&1; then + pulp config create --overwrite --base-url "$PULP_URL" --api-root "/" \ + --domain "$PULP_DOMAIN" \ + --header "Authorization:Bearer $token" --timeout 0 >/dev/null 2>&1 || true + fi +} + +# wait for a pulp api task to complete. a failed poll request (network blip, +# api 5xx) is retried like an unexpected state instead of aborting the caller +# under set -e; the 600-attempt cap bounds the total wait (~30 min): under a +# full-matrix delivery the publication of a large repository can sit several +# minutes in the worker queue before its ~5 min of actual execution. +wait_task() { + local task_href=$1 + local state attempt + for ((attempt = 0; attempt < 600; attempt++)); do + refresh_pulp_token + state=$(curl -fsSL -H "Authorization: Bearer $PULP_TOKEN" "$PULP_URL$task_href" 2>/dev/null | jq -r '.state' 2>/dev/null) || state="" + case "$state" in + completed) + return 0 + ;; + failed|canceled) + echo "::error::Task $task_href $state: $(curl -fsSL -H "Authorization: Bearer $PULP_TOKEN" "$PULP_URL$task_href" | jq -c '.error')" + return 1 + ;; + *) + sleep 3 + ;; + esac + done + echo "::error::Task $task_href did not complete in time (~30 min)" + return 1 +} + +# upload a package through the pulp api, with retry on transient failures +# (concurrent deliveries can race on artifact creation), echoes the task href +pulp_upload() { + local attempt response http_code body + for attempt in 1 2 3 4 5; do + refresh_pulp_token + response=$(curl -sS -H "Authorization: Bearer $PULP_TOKEN" -w $'\n%{http_code}' "$@" 2>/dev/null) || response="" + http_code=${response##*$'\n'} + body=${response%$'\n'*} + if [[ "$http_code" == "202" ]]; then + echo "$body" | jq -r '.task' + return 0 + fi + echo "[WARN] upload attempt $attempt/5 failed (HTTP ${http_code:-network-error}), retrying..." >&2 + sleep $((attempt * 5)) + done + echo "::error::Upload failed after 5 attempts (HTTP ${http_code:-network-error})" >&2 + return 1 +} + +# wait for a task, distinguishing a RETRYABLE failure from a genuine one: +# - lost repository-version race: concurrent legs of a SHARED deb repository +# collide on the version bookkeeping (duplicate (repository_id, number) +# insert, or the retention cleanup deleting a version another transaction +# still references) +# - "Worker has gone missing": the task worker pod was terminated mid-task +# (scale-down, node consolidation) +# Re-running the operation converges, so callers retry on rc 2. +# 0=completed, 2=retryable failure, 1=failed or timed out. +wait_task_race() { + local task_href=$1 body state error attempt + for ((attempt = 0; attempt < 600; attempt++)); do + refresh_pulp_token + body=$(curl -fsSL -H "Authorization: Bearer $PULP_TOKEN" "$PULP_URL$task_href" 2>/dev/null) || body="" + state=$(echo "$body" | jq -r '.state' 2>/dev/null) || state="" + case "$state" in + completed) + return 0 + ;; + failed | canceled) + error=$(echo "$body" | jq -c '.error // empty' 2>/dev/null) || error="" + if echo "$error" | grep -qE 'core_repositoryversion|Worker has gone missing'; then + return 2 + fi + # the "gone missing" reason lands in a separate field on some tasks + if echo "$body" | jq -r '.reason // empty' 2>/dev/null | grep -q 'Worker has gone missing'; then + return 2 + fi + echo "::error::Task $task_href $state: $error" + return 1 + ;; + *) + sleep 3 + ;; + esac + done + echo "::error::Task $task_href did not complete in time (~30 min)" + return 1 +} + +# check that a named pulp resource exists through the REST list api, +# distinguishing a genuine absence from an api failure: a failed "pulp show" +# reads as "does not exist" and turned an api saturation window into +# "nothing to promote". 0=exists, 1=absent; an unanswerable api aborts the +# caller (set -e) after retries: better fail loud than act on a guess. +# usage: pulp_resource_exists e.g. repositories/deb/apt +pulp_resource_exists() { + local type_path=$1 name=$2 attempt response http_code count + for attempt in 1 2 3 4; do + refresh_pulp_token + response=$(curl -sS -H "Authorization: Bearer $PULP_TOKEN" -w $'\n%{http_code}' -G \ + --data-urlencode "name=$name" --data-urlencode "limit=1" \ + "$PULP_URL/$PULP_DOMAIN/api/v3/$type_path/" 2>/dev/null) || response=$'\n000' + http_code="${response##*$'\n'}" + if [[ "$http_code" == "200" ]]; then + count=$(printf '%s' "${response%$'\n'*}" | jq -r '.count' 2>/dev/null) || count="" + [[ "$count" == "0" ]] && return 1 + [[ -n "$count" ]] && return 0 + fi + sleep $((attempt * 5)) + done + echo "::error::Cannot check the existence of $name (HTTP ${http_code:-network-error}); refusing to guess." >&2 + exit 70 +} + +# POST a json body with MANUAL retry on transient gateway failures: curl +# --retry concatenates the bodies of failed attempts, which corrupts a +# captured json response (a 502 html page glued before the 201 body). A 500 +# is NOT retried: on this api it means a duplicate synchronous content +# create, and the caller's lookup fallback is the correct answer. +# Echoes the final body followed by a newline and the final status code (the +# function runs inside command substitutions, so it cannot hand the code back +# through a variable); the manual retry guarantees a single body, making the +# last-line split safe. +post_json() { + local url=$1 json=$2 attempt response code + response=$'\n000' + for attempt in 1 2 3 4; do + refresh_pulp_token + response=$(curl -sS -H "Authorization: Bearer $PULP_TOKEN" -w $'\n%{http_code}' \ + -X POST -H "Content-Type: application/json" -d "$json" "$url" 2>/dev/null) || response=$'\n000' + code="${response##*$'\n'}" + case "$code" in + 2* | 4* | 500) break ;; + *) sleep $((attempt * 3)) ;; + esac + done + printf '%s\n%s' "${response%$'\n'*}" "$code" +} + +# start a repository modify task, with retry on transient gateway failures: +# it is the single most critical call of the flow (all uploaded content lands +# in the repository through it) and add_content_units is idempotent, so +# retrying is always safe. Echoes the task href. +start_modify_task() { + local url=$1 body_file=$2 attempt response http_code body + for attempt in 1 2 3 4 5; do + refresh_pulp_token + response=$(curl -sS -H "Authorization: Bearer $PULP_TOKEN" -w $'\n%{http_code}' \ + -X POST -H "Content-Type: application/json" \ + -d @"$body_file" "$url" 2>/dev/null) || response="" + http_code=${response##*$'\n'} + body=${response%$'\n'*} + if [[ "$http_code" == "202" ]]; then + echo "$body" | jq -r '.task' + return 0 + fi + echo "[WARN] modify attempt $attempt/5 on $url failed (HTTP ${http_code:-network-error}), retrying..." >&2 + sleep $((attempt * 5)) + done + echo "::error::Repository modify failed after 5 attempts on $url (HTTP ${http_code:-network-error}): $(echo "$body" | head -c 300)" >&2 + return 1 +} + +# create a publication in the background and wait for it with re-authenticating +# polling: pulp-cli reads its token once at startup, so its built-in wait fails +# with "Authentication failed for tasks_read" as soon as the publication of a +# large repository outlives the OIDC token validity (~5 minutes) +# curl against the content endpoint with the CI credentials: the guarded +# *business-internal distributions authorize the download through the same +# GitHub OIDC authentication as the api (the CI user holds the guard +# downloader role); unguarded distributions ignore the header. +content_curl() { + refresh_pulp_token + curl -H "Authorization: Bearer $PULP_TOKEN" "$@" +} + +create_publication() { + local plugin=$1 repository=$2 + shift 2 + # pulp-cli's own task re-poll is domain-unaware and 404s under Domains ("No + # Task matches the given query"), so this goes straight through + # post_json/wait_task_race instead, both already domain-aware + local publication_path + case "$plugin" in + rpm) publication_path="rpm/rpm" ;; + deb) publication_path="deb/apt" ;; + *) echo "::error::create_publication: unsupported plugin $plugin" >&2; return 1 ;; + esac + local repo_href body_json response code body task attempt outer rc + # outer retry: the publication task itself can die server-side for + # retryable reasons (task worker terminated mid-task, version race) + for outer in 1 2 3; do + task="" + for attempt in 1 2 3; do + refresh_pulp_token + repo_href=$(pulp "$plugin" repository show --name "$repository" 2>/dev/null | jq -r '.pulp_href // empty') || repo_href="" + if [[ -n "$repo_href" ]]; then + if [[ "$*" == *"--structured"* ]]; then + body_json=$(jq -nc --arg repo "$repo_href" '{repository: $repo, structured: true, simple: false}') + else + body_json=$(jq -nc --arg repo "$repo_href" '{repository: $repo}') + fi + response=$(post_json "$PULP_URL/$PULP_DOMAIN/api/v3/publications/$publication_path/" "$body_json") + code="${response##*$'\n'}" + body="${response%$'\n'*}" + if [[ "$code" == "202" ]]; then + task=$(echo "$body" | jq -r '.task // empty') + [[ -n "$task" ]] && break + fi + fi + echo "[WARN] publication start attempt $attempt/3 failed for $repository, retrying..." >&2 + sleep $((attempt * 10)) + done + if [[ -z "$task" ]]; then + echo "::error::Cannot start the publication of repository $repository" + return 1 + fi + wait_task_race "$task" && rc=0 || rc=$? + if [[ $rc -eq 0 ]]; then + return 0 + elif [[ $rc -eq 2 && $outer -lt 3 ]]; then + echo "[WARN] Publication of $repository was interrupted server-side, retrying" + sleep $((outer * 15)) + else + echo "::error::Publication of repository $repository failed" + return 1 + fi + done +} diff --git a/pulp/lib/check-common.sh b/pulp/lib/check-common.sh new file mode 100644 index 0000000..3fa79a1 --- /dev/null +++ b/pulp/lib/check-common.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +# Shared helpers for the pulp delivery/promotion verification scripts +# (check-rpm.sh, check-deb.sh). +# +# The check never aborts mid-verification: every expected package is checked, its +# per-package result is recorded, the full table is rendered to the step summary, +# and only then is a non-zero status returned if anything failed. Source this +# file, call load_expected, verify the packages, then call render_summary. + +# authenticated content fetches (content_curl) and token refresh +# shellcheck source=pulp/lib/api.sh +source "$(dirname "${BASH_SOURCE[0]}")/api.sh" + +PULP_URL="${PULP_URL:-https://pulp-api.int.centreon.com}" +PULP_CONTENT_URL="${PULP_CONTENT_URL:-https://packages.int.centreon.com}" + +switch_pulp_domain "$PULP_DOMAIN" || exit 1 +# 900s: the published indexes can lag a finished publication by up to the +# content-app cache TTL (600s), the window must survive that worst case +METADATA_TIMEOUT="${METADATA_TIMEOUT:-900}" +METADATA_INTERVAL="${METADATA_INTERVAL:-15}" + +# latest_version_href : print the repository's latest version +# href (empty when it has none). The lookup is retried: a transient API error must +# not read as a missing repository, which fails the whole verification. +latest_version_href() { + local type=$1 name=$2 attempt out + for attempt in 1 2 3; do + if out=$(pulp "$type" repository show --name "$name" 2>/dev/null); then + jq -r '.latest_version_href // empty' <<< "$out" + return 0 + fi + ((attempt < 3)) && sleep 5 + done + return 0 +} + +# accumulated per-package result rows and the aggregate failure flag +ROWS=() +PRESENT_OK=0 +META_OK=0 +TOTAL=0 +FAILED=0 + +# load_expected — load the manifest emitted by the delivery or +# promotion step into PACKAGES_JSON/COUNT; the manifest is required. +load_expected() { + local package_type=$1 + if [[ -z "${MANIFEST:-}" || ! -s "${MANIFEST:-}" ]]; then + echo "::error::pulp verification requires the manifest emitted by the delivery/promote step" + exit 1 + fi + echo "[INFO] Reading expected ${package_type} packages from manifest ${MANIFEST}" + PACKAGES_JSON=$(jq -c '.packages' "$MANIFEST") + COUNT=$(echo "$PACKAGES_JSON" | jq 'length') + if [[ "$COUNT" -eq 0 ]]; then + echo "::error::No expected ${package_type} package to verify" + exit 1 + fi + echo "[INFO] ${COUNT} expected ${package_type} package(s) to verify" +} + +# wait_for_metadata — repeatedly call the sourcing script's resolve_pending (one +# resolution round over the still-unresolved packages, returning 0 once none +# remain) until everything resolves, METADATA_TIMEOUT is reached, or the +# resolution stalls. The retry window only covers publication propagation: once +# a round reads the published metadata and resolves nothing new while some +# packages already resolved, the remaining ones are not in the publication at +# all (e.g. evicted by the retention policy) and no amount of waiting will +# surface them - report them right away instead of burning the whole window. +wait_for_metadata() { + local deadline=$(( SECONDS + METADATA_TIMEOUT )) + local resolved previous_resolved=-1 stall_rounds=0 + until resolve_pending; do + resolved=0 + for i in "${!E_FILENAME[@]}"; do + [[ "${META_IDX[$i]}" == "true" ]] && resolved=$((resolved + 1)) + done + if ((resolved > 0 && resolved == previous_resolved)); then + stall_rounds=$((stall_rounds + 1)) + else + stall_rounds=0 + fi + # several consecutive stalled rounds before giving up: a single one is + # not enough, the published index can lag the publication by minutes + # (content-app cache TTL), which read as false negatives on freshly + # delivered builds + if ((stall_rounds >= 6)); then + echo "[WARN] ${resolved}/${#E_FILENAME[@]} package(s) resolvable in the published metadata and no progress in the last 6 rounds; the remaining ones are not part of the publication, giving up early" + break + fi + previous_resolved=$resolved + if [[ "$SECONDS" -ge "$deadline" ]]; then + echo "[WARN] Metadata resolution timed out after ${METADATA_TIMEOUT}s (${resolved}/${#E_FILENAME[@]} resolvable)" + break + fi + echo "[INFO] ${resolved}/${#E_FILENAME[@]} package(s) resolvable, waiting ${METADATA_INTERVAL}s for the metadata to propagate..." + sleep "$METADATA_INTERVAL" + done +} + +# record_row +record_row() { + local filename=$1 arch=$2 present=$3 in_meta=$4 fetchable=$5 + local p m f + TOTAL=$((TOTAL + 1)) + if [[ "$present" == "true" ]]; then p="✅"; PRESENT_OK=$((PRESENT_OK + 1)); else p="❌"; FAILED=1; fi + if [[ "$in_meta" == "true" ]]; then m="✅"; META_OK=$((META_OK + 1)); else m="❌"; FAILED=1; fi + if [[ "$fetchable" == "true" ]]; then f="✅"; else f="❌"; FAILED=1; fi + ROWS+=("| \`$filename\` | $arch | $p | $m | $f |") + echo "[CHECK] $filename ($arch): present=$p metadata=$m fetchable=$f" +} + +# served_artifact_sha256 — print the sha256 of the artifact the +# content app redirected to (storage path artifact///), +# nothing when the url doesn't follow that layout +served_artifact_sha256() { + if [[ "$1" =~ /artifact/[^/]+/([0-9a-f]{2})/([0-9a-f]{62})([?]|$) ]]; then + echo "${BASH_REMATCH[1]}${BASH_REMATCH[2]}" + fi +} + +# check_fetchable_and_record — HEAD each resolved package on the content url and +# record every package's result row. Uses the E_FILENAME/E_ARCH/E_BASEPATH/E_SHA256, +# PRESENT_IDX, META_IDX and RESOLVED_IDX arrays filled by the sourcing script. +check_fetchable_and_record() { + local i url out code served retry_served fetchable attempt digest_deadline="" + for i in "${!E_FILENAME[@]}"; do + fetchable=false + if [[ "${META_IDX[$i]}" == "true" ]]; then + url="${PULP_CONTENT_URL}/${E_BASEPATH[$i]}/${RESOLVED_IDX[$i]}" + # retry: one flaky HEAD out of hundreds (content-app/S3 hiccup) must not + # fail the whole verification + for attempt in 1 2 3; do + out=$(content_curl -fsSL -o /dev/null -w '%{http_code} %{url_effective}' -I "$url" 2>/dev/null || echo 000) + code=${out%% *} + [[ "$code" == "200" ]] && { fetchable=true; break; } + sleep 2 + done + # the legacy front resolves pool paths stable first: a same-named build + # published elsewhere is served instead of the delivered one + served=$(served_artifact_sha256 "${out#* }") + if [[ "$fetchable" == "true" && -n "$served" && "$served" != "${E_SHA256[$i]}" ]]; then + # a replaced build keeps being served until the content-app cache + # expires (600s TTL): one shared window for the whole verification + digest_deadline=${digest_deadline:-$(( SECONDS + METADATA_TIMEOUT ))} + while [[ "$served" != "${E_SHA256[$i]}" && "$SECONDS" -lt "$digest_deadline" ]]; do + echo "[INFO] ${E_FILENAME[$i]}: served sha256 ${served} differs from the delivered one, waiting ${METADATA_INTERVAL}s for the content cache..." + sleep "$METADATA_INTERVAL" + out=$(content_curl -fsSL -o /dev/null -w '%{http_code} %{url_effective}' -I "$url" 2>/dev/null || echo 000) + retry_served=$(served_artifact_sha256 "${out#* }") + # a failed retry (no digest) keeps the known mismatch + [[ -n "$retry_served" ]] && served=$retry_served + done + if [[ "$served" != "${E_SHA256[$i]}" ]]; then + if [[ "${STABILITY:-}" == "unstable" ]]; then + echo "::warning::${url} serves sha256 ${served}, not the delivered ${E_SHA256[$i]}: another build with the same file name masks it." + else + echo "::error::${url} serves sha256 ${served}, not the delivered ${E_SHA256[$i]}: another build with the same file name masks it. Bump the package version." + fetchable=false + fi + fi + fi + fi + record_row "${E_FILENAME[$i]}" "${E_ARCH[$i]}" "${PRESENT_IDX[$i]}" "${META_IDX[$i]}" "$fetchable" + done +} + +# render_summary — write the per-OS table to the step summary (and stdout), then +# return non-zero if any package failed any check. Called once, after all checks. +render_summary() { + local title + title="### ${DISTRIB:-?} — ${STABILITY:-?} (${CHECK_MODE:-delivery}, pulp)" + + { + echo "$title" + echo "" + echo "| Package | Arch | On repository | In metadata | Fetchable |" + echo "|---|---|---|---|---|" + if ((${#ROWS[@]})); then + printf '%s\n' "${ROWS[@]}" + fi + echo "" + echo "Result: ${PRESENT_OK}/${TOTAL} present · ${META_OK}/${TOTAL} resolvable" + if [[ "$FAILED" -ne 0 ]]; then + echo "" + echo "> ❌ Some packages are missing on the repository or unresolvable through metadata." + fi + echo "" + } | tee -a "${GITHUB_STEP_SUMMARY:-/dev/stdout}" + + if [[ "$FAILED" -ne 0 ]]; then + echo "::error::pulp verification: ${DISTRIB:-?} verification failed (see the table above)" + return 1 + fi + echo "[INFO] All ${TOTAL} package(s) verified on ${DISTRIB:-?}" + return 0 +} diff --git a/pulp/lib/check-deb.sh b/pulp/lib/check-deb.sh new file mode 100755 index 0000000..43672b2 --- /dev/null +++ b/pulp/lib/check-deb.sh @@ -0,0 +1,123 @@ +#!/usr/bin/env bash +# Verify that the delivered/promoted DEB packages are (1) physically present as +# content units in their target pulp repository and (2) resolvable through the +# published apt metadata (and actually fetchable from the content url). +# +# Metadata resolution mirrors promote-deb.sh: the structured publication serves +# packages under the canonical pool layout, so the published Filename is resolved +# from the suite's Packages index by sha256. Architecture "all" packages are +# listed under every binary-, so they are looked up across the suite's +# architectures. +# +# The expected list is the manifest emitted by the delivery/promote step. The +# script never exits early — see check-common.sh. +set -uo pipefail + +# shellcheck source=pulp/lib/check-common.sh +source "$(dirname "$0")/check-common.sh" + +load_expected "DEB" + +# --- load expected packages into parallel arrays --------------------------- +mapfile -t E_FILENAME < <(echo "$PACKAGES_JSON" | jq -r '.[].filename') +mapfile -t E_ARCH < <(echo "$PACKAGES_JSON" | jq -r '.[].arch') +mapfile -t E_SHA256 < <(echo "$PACKAGES_JSON" | jq -r '.[].sha256') +mapfile -t E_REPOSITORY < <(echo "$PACKAGES_JSON" | jq -r '.[].repository') +mapfile -t E_BASEPATH < <(echo "$PACKAGES_JSON" | jq -r '.[].base_path') +mapfile -t E_SUITE < <(echo "$PACKAGES_JSON" | jq -r '.[].suite') + +# --- physical presence: content units in the repository's latest version ---- +# one lookup per expected package, by sha256 within the repository version: the +# shared repositories hold 10k+ packages and listing them, even newest first, +# is slow enough to hit the gateway timeout, which then read as "absent" while +# the package was published and fetchable. --retry covers those transient 5xx. +declare -A VERSION_BY_REPO # repo -> latest_version_href, empty when missing +for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do + VERSION_BY_REPO[$repo]=$(latest_version_href deb "$repo") + [[ -n "${VERSION_BY_REPO[$repo]}" ]] || echo "[WARN] Repository $repo does not exist or has no version" +done + +declare -A PRESENT_IDX +for i in "${!E_FILENAME[@]}"; do + PRESENT_IDX[$i]=false + version_href=${VERSION_BY_REPO[${E_REPOSITORY[$i]}]} + [[ -n "$version_href" ]] || continue + refresh_pulp_token + url="$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/packages/?$( + printf 'repository_version=%s&sha256=%s&fields=pulp_href&limit=1' \ + "$(jq -rn --arg v "$version_href" '$v | @uri')" "${E_SHA256[$i]}" + )" + count=$(curl -fsSL --retry 3 --retry-delay 5 \ + -H "Authorization: Bearer $PULP_TOKEN" "$url" | jq -r '.count // 0') || { + echo "[WARN] presence lookup failed for ${E_FILENAME[$i]} in ${E_REPOSITORY[$i]} ($url)" >&2 + continue + } + [[ "${count:-0}" -gt 0 ]] && PRESENT_IDX[$i]=true +done + +# --- metadata resolvability + fetchability, with a bounded retry window ----- +# resolve a package's published Filename from a suite Packages index by sha256 +resolve_filename() { + # read the Packages index from a file: awk exits on the first match, and a + # pipe writer would take a SIGPIPE ("printf: write error: Broken pipe" noise) + local packages_file=$1 sha=$2 + [[ -s "$packages_file" ]] || return 0 + awk -v sha="$sha" ' + BEGIN { RS = ""; FS = "\n" } + index($0, "SHA256: " sha) { + for (i = 1; i <= NF; i++) if ($i ~ /^Filename: /) { sub(/^Filename: /, "", $i); print $i; exit } + }' "$packages_file" +} + +declare -A META_IDX # idx -> true|false +declare -A RESOLVED_IDX # idx -> published Filename +for i in "${!E_FILENAME[@]}"; do META_IDX[$i]=false; done + +# one resolution round: fetch each suite's Packages indexes once, then resolve +# each pending package's published Filename by sha256 +resolve_pending() { + local -A pkg_cache=() # key: base_path|suite|arch -> Packages index file + local -A arches_cache=() # key: base_path|suite -> space separated arches + local all_resolved=true i base_path suite arch search_arches sk ck a filename cache_file + for i in "${!E_FILENAME[@]}"; do + [[ "${META_IDX[$i]}" == "true" ]] && continue + base_path=${E_BASEPATH[$i]}; suite=${E_SUITE[$i]}; arch=${E_ARCH[$i]} + + # architectures to search: the package arch, plus every arch of the suite + # for "all" packages (which are duplicated across each binary-) + search_arches="$arch" + if [[ "$arch" == "all" ]]; then + sk="$base_path|$suite" + if [[ -z "${arches_cache[$sk]+set}" ]]; then + arches_cache[$sk]=$(content_curl -fsSL "$PULP_CONTENT_URL/$base_path/dists/$suite/Release" 2>/dev/null \ + | awk -F': ' '/^Architectures:/ { print $2; exit }') + fi + search_arches="${arches_cache[$sk]:-amd64 arm64 all}" + fi + + filename="" + for a in $search_arches; do + ck="$base_path|$suite|$a" + if [[ -z "${pkg_cache[$ck]+set}" ]]; then + cache_file=$(mktemp) + content_curl -fsSL "$PULP_CONTENT_URL/$base_path/dists/$suite/main/binary-$a/Packages" 2>/dev/null > "$cache_file" || true + pkg_cache[$ck]=$cache_file + fi + filename=$(resolve_filename "${pkg_cache[$ck]}" "${E_SHA256[$i]}") + [[ -n "$filename" ]] && break + done + + if [[ -n "$filename" ]]; then + META_IDX[$i]=true + RESOLVED_IDX[$i]=$filename + else + all_resolved=false + fi + done + rm -f "${pkg_cache[@]}" + [[ "$all_resolved" == "true" ]] +} + +wait_for_metadata +check_fetchable_and_record +render_summary diff --git a/pulp/lib/check-rpm.sh b/pulp/lib/check-rpm.sh new file mode 100755 index 0000000..2369da2 --- /dev/null +++ b/pulp/lib/check-rpm.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash +# Verify that the delivered/promoted RPM packages are (1) physically present as +# content units in their target pulp repository and (2) resolvable through the +# published repodata (and actually fetchable from the content url). +# +# The expected list is the manifest emitted by the delivery/promote step. The +# script never exits early — see check-common.sh. +set -uo pipefail + +# shellcheck source=pulp/lib/check-common.sh +source "$(dirname "$0")/check-common.sh" + +load_expected "RPM" + +# --- load expected packages into parallel arrays --------------------------- +mapfile -t E_FILENAME < <(echo "$PACKAGES_JSON" | jq -r '.[].filename') +mapfile -t E_ARCH < <(echo "$PACKAGES_JSON" | jq -r '.[].arch') +mapfile -t E_SHA256 < <(echo "$PACKAGES_JSON" | jq -r '.[].sha256') +mapfile -t E_REPOSITORY < <(echo "$PACKAGES_JSON" | jq -r '.[].repository') +mapfile -t E_BASEPATH < <(echo "$PACKAGES_JSON" | jq -r '.[].base_path') + +# --- physical presence: content units in each repository's latest version -- +# newest first, stopping as soon as every expected package of the repository +# has been seen: a large repository makes deep offset pagination costly and +# flaky, while the freshly delivered packages are the newest content by +# construction. +# The listing goes to a file and grep reads the file: piping it into grep -q +# would SIGPIPE the writer on the (early-exiting) first match, and pipefail +# then turns every successful match into a false negative. +declare -A PRESENT_BY_REPO # repo -> file holding one filename per line +declare -A EXPECTED_COUNT_BY_REPO +declare -A TOTAL_COUNT_BY_REPO + +for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do + PRESENT_BY_REPO[$repo]=$(mktemp) + version_href=$(latest_version_href rpm "$repo") + if [[ -z "$version_href" ]]; then + echo "[WARN] Repository $repo does not exist or has no version" + continue + fi + url="$PULP_URL/$PULP_DOMAIN/api/v3/content/rpm/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&ordering=-pulp_created&limit=1000' \ + "$(jq -rn --arg v "$version_href" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" + )" + pages=0 + while [[ -n "$url" ]] && ((pages < 20)); do + page=$(curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Bearer $PULP_TOKEN" "$url") || { + echo "[WARN] presence page fetch failed for $repo ($url)" >&2 + break + } + if ((pages == 0)); then + TOTAL_COUNT_BY_REPO[$repo]=$(echo "$page" | jq -r '.count') + echo "[INFO] Repository $repo holds ${TOTAL_COUNT_BY_REPO[$repo]} module package(s) in its latest version" + fi + echo "$page" | jq -r '.results[].location_href' | awk -F/ '{print $NF}' >> "${PRESENT_BY_REPO[$repo]}" + pages=$((pages + 1)) + all_found=true + for i in "${!E_FILENAME[@]}"; do + [[ "${E_REPOSITORY[$i]}" == "$repo" ]] || continue + if ! grep -Fxq "${E_FILENAME[$i]}" "${PRESENT_BY_REPO[$repo]}"; then + all_found=false + break + fi + done + [[ "$all_found" == "true" ]] && break + url=$(echo "$page" | jq -r '.next // empty') + done +done + +declare -A PRESENT_IDX # idx -> true|false +for i in "${!E_FILENAME[@]}"; do + repo=${E_REPOSITORY[$i]} + EXPECTED_COUNT_BY_REPO[$repo]=$(( ${EXPECTED_COUNT_BY_REPO[$repo]:-0} + 1 )) + if grep -Fxq "${E_FILENAME[$i]}" "${PRESENT_BY_REPO[$repo]}"; then + PRESENT_IDX[$i]=true + else + PRESENT_IDX[$i]=false + fi +done + +# right number (delivery only): warn when the repository holds more module +# packages than expected. Count-based only: with shared repositories and the +# daily develop deliveries, listing every extra name is noise (and the full +# listing is what the bounded pagination above avoids). +[[ "${CHECK_MODE:-delivery}" == "delivery" ]] && for repo in "${!EXPECTED_COUNT_BY_REPO[@]}"; do + total=${TOTAL_COUNT_BY_REPO[$repo]:-0} + if [[ "$total" -gt "${EXPECTED_COUNT_BY_REPO[$repo]}" ]]; then + echo "::warning::Repository $repo holds $total module packages, ${EXPECTED_COUNT_BY_REPO[$repo]} delivered by this run (older builds and other modules accumulate until cleanup)" + fi +done + +# --- metadata resolvability + fetchability, with a bounded retry window ----- +declare -A META_IDX # idx -> true|false +declare -A RESOLVED_IDX # idx -> published location href +for i in "${!E_FILENAME[@]}"; do META_IDX[$i]=false; done + +# resolve the published location href of the package whose sha256 checksum +# matches, from a primary.xml body: matching by checksum (not filename) binds +# the verification to the exact delivered content, like the DEB by-sha256 match +resolve_href() { + # read the primary.xml from a file: awk exits on the first match, and a pipe + # writer would take a SIGPIPE ("printf: write error: Broken pipe" noise) + local primary_file=$1 sha=$2 + [[ -s "$primary_file" ]] || return 0 + awk -v sha="$sha" ' + BEGIN { RS = "" } + index($0, ">" sha "") { + if (match($0, /]*href="[^"]+"/)) { + s = substr($0, RSTART, RLENGTH) + sub(/.*href="/, "", s); sub(/"$/, "", s) + print s; exit + } + }' "$primary_file" +} + +# one resolution round: fetch each base_path's primary.xml once, then resolve +# each pending package's published href by sha256 +resolve_pending() { + local -A primary_cache=() + local all_resolved=true i base_path repomd primary_href href cache_file + for i in "${!E_FILENAME[@]}"; do + [[ "${META_IDX[$i]}" == "true" ]] && continue + base_path=${E_BASEPATH[$i]} + + if [[ -z "${primary_cache[$base_path]+set}" ]]; then + cache_file=$(mktemp) + repomd=$(content_curl -fsSL "$PULP_CONTENT_URL/$base_path/repodata/repomd.xml" 2>/dev/null || true) + primary_href=$(printf '%s' "$repomd" | grep -oP '/dev/null | gunzip -c 2>/dev/null > "$cache_file" || true + fi + primary_cache[$base_path]=$cache_file + fi + + href=$(resolve_href "${primary_cache[$base_path]}" "${E_SHA256[$i]}") + if [[ -n "$href" ]]; then + META_IDX[$i]=true + RESOLVED_IDX[$i]=$href + else + all_resolved=false + fi + done + rm -f "${primary_cache[@]}" + [[ "$all_resolved" == "true" ]] +} + +wait_for_metadata +check_fetchable_and_record +render_summary diff --git a/pulp/lib/manifest.sh b/pulp/lib/manifest.sh new file mode 100644 index 0000000..242f5d7 --- /dev/null +++ b/pulp/lib/manifest.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Shared helpers to emit the delivery/promotion manifest consumed by the +# verification scripts (check-rpm.sh, check-deb.sh). Source this file, append +# one compact JSON object per package with manifest_add, then finalize the +# manifest with manifest_write. +# +# The manifest is the authoritative "expected packages" list: it is captured at +# the moment of upload (the packages that were on the runner before/at upload), +# so the check step verifies exactly this set rather than re-reading the build +# cache. Each entry carries the package identity (for the content-api presence +# query), its sha256 (for the by-checksum metadata match) and the pulp target +# coordinates (so the check re-derives no path). + +MANIFEST_ENTRIES=() + +# manifest_add +manifest_add() { + MANIFEST_ENTRIES+=("$1") +} + +# manifest_write +# writes the manifest json to the workspace and exposes its path as the +# "manifest" step output when running inside a github action. +manifest_write() { + local module=$1 distrib=$2 package_type=$3 stability=$4 mode=$5 content_url=$6 + local file="${GITHUB_WORKSPACE:-$PWD}/pulp-delivery-manifest.json" + + # the packages array is slurped from stdin: with hundreds of packages an + # --argjson argument would exceed the kernel argv size limit (E2BIG). + # NB: the jq variable is named module_name, not module — `module` is a reserved + # jq keyword and `$module` fails to parse on stricter jq builds (the CI runner) + { + if ((${#MANIFEST_ENTRIES[@]})); then + printf '%s\n' "${MANIFEST_ENTRIES[@]}" + fi + } | jq -s \ + --arg module_name "$module" \ + --arg distrib "$distrib" \ + --arg package_type "$package_type" \ + --arg stability "$stability" \ + --arg mode "$mode" \ + --arg content_url "$content_url" \ + '{ + "module": $module_name, + distrib: $distrib, + package_type: $package_type, + stability: $stability, + mode: $mode, + pulp_content_url: $content_url, + packages: . + }' > "$file" + + echo "[INFO] Wrote delivery manifest ($(jq '.packages | length' "$file") package(s)) to $file" + if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo "manifest=$file" >> "$GITHUB_OUTPUT" + fi +} diff --git a/pulp/package-delivery/action.yml b/pulp/package-delivery/action.yml new file mode 100644 index 0000000..5877c42 --- /dev/null +++ b/pulp/package-delivery/action.yml @@ -0,0 +1,222 @@ +name: "package-delivery" +description: "Deliver rpm and deb packages to pulp" +inputs: + module_name: + description: "The package module name" + required: true + distrib: + description: "The distribution used for packaging" + required: true + version: + description: "Centreon packaged major version (not used by the plugins repository family)" + required: false + default: "" + cache_key: + description: "The cached package key" + required: true + stability: + description: "The package stability (stable, testing, unstable)" + required: true + release_type: + description: "Type of release (hotfix, release)" + required: false + default: "" + is_cloud: + description: "Release context (cloud or not cloud)" + required: true + delivery_type: + description: "Delivery to feature or standard repository" + required: false + default: "standard" + repository_name: + description: "Target repository family: standard, standard-internal, business, business-internal, plugins or connectors" + required: false + default: "standard" + verify: + description: "Verify the delivered packages afterwards (presence, metadata resolution, fetchability)" + required: false + default: "true" + pulp_url: + description: "The pulp server api url" + required: true + pulp_content_url: + description: "The pulp server content url" + required: true + audience: + description: "OIDC audience, must match the pulp server GITHUB_OIDC.audience" + required: true + centreon_technique_pat: + description: "Token used to request the feature repository creation (required when delivery_type is feature)" + required: false + default: "" + +outputs: + manifest: + description: "Path of the JSON manifest of the delivered packages, empty when nothing was delivered" + value: ${{ steps.deliver-rpm.outputs.manifest || steps.deliver-deb.outputs.manifest }} + manifest_artifact: + description: "Name of the artifact holding the manifest of a testing delivery, empty otherwise" + value: ${{ steps.upload-manifest.outcome == 'success' && steps.manifest-artifact.outputs.name || '' }} + +runs: + using: "composite" + steps: + - name: Validate inputs + shell: bash + env: + STABILITY: ${{ inputs.stability }} + DELIVERY_TYPE: ${{ inputs.delivery_type }} + run: | + set -euo pipefail + + if [[ "$DELIVERY_TYPE" != "feature" && "$STABILITY" != "testing" && "$STABILITY" != "unstable" ]]; then + echo "::error::stability must be one of: testing, unstable (got '$STABILITY')" + exit 1 + fi + + - name: Compute pulp repository properties + id: pulp-properties + shell: bash + env: + MODULE_NAME: ${{ inputs.module_name }} + DISTRIB: ${{ inputs.distrib }} + VERSION: ${{ inputs.version }} + STABILITY: ${{ inputs.stability }} + RELEASE_TYPE: ${{ inputs.release_type }} + IS_CLOUD: ${{ inputs.is_cloud }} + DELIVERY_TYPE: ${{ inputs.delivery_type }} + REPOSITORY_TYPE: ${{ inputs.repository_name }} + GH_HEAD_REF: ${{ github.head_ref || github.ref_name }} + run: ${{ github.action_path }}/../repository-properties/properties.sh + + - name: Restore packages from cache + if: steps.pulp-properties.outputs.skip_delivery == 'false' + uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5 + with: + path: ./*.${{ steps.pulp-properties.outputs.package_extension }} + key: ${{ inputs.cache_key }} + fail-on-cache-miss: true + + - name: Setup python + if: steps.pulp-properties.outputs.skip_delivery == 'false' + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Setup pulp-cli + if: steps.pulp-properties.outputs.skip_delivery == 'false' + shell: bash + env: + PULP_URL: ${{ inputs.pulp_url }} + AUDIENCE: ${{ inputs.audience }} + DOMAIN: ${{ steps.pulp-properties.outputs.domain }} + run: ${{ github.action_path }}/../setup-cli/setup.sh + + - name: Ensure the rpm feature repository exists + if: inputs.delivery_type == 'feature' && steps.pulp-properties.outputs.package_extension == 'rpm' && steps.pulp-properties.outputs.skip_delivery == 'false' + shell: bash + env: + VERSION: ${{ inputs.version }} + GH_HEAD_REF: ${{ github.head_ref || github.ref_name }} + REPOSITORY_PREFIX: ${{ steps.pulp-properties.outputs.repository_prefix }} + PULP_URL: ${{ inputs.pulp_url }} + PULP_DOMAIN: ${{ steps.pulp-properties.outputs.domain }} + PULP_TOKEN: ${{ env.PULP_TOKEN }} + CENTREON_TECHNIQUE_PAT: ${{ inputs.centreon_technique_pat }} + run: ${{ github.action_path }}/ensure-feature-repository.sh + + - name: Deliver RPM packages + id: deliver-rpm + if: steps.pulp-properties.outputs.package_extension == 'rpm' && steps.pulp-properties.outputs.skip_delivery == 'false' + shell: bash + env: + MODULE_NAME: ${{ inputs.module_name }} + DISTRIB: ${{ inputs.distrib }} + STABILITY: ${{ inputs.stability }} + REPOSITORY_PREFIX: ${{ steps.pulp-properties.outputs.repository_prefix }} + STABLE_REPOSITORY_PREFIX: ${{ steps.pulp-properties.outputs.stable_repository_prefix }} + BASE_PATH_PREFIX: ${{ steps.pulp-properties.outputs.base_path_prefix }} + LEGACY_BASE_PATH_PREFIX: ${{ steps.pulp-properties.outputs.legacy_base_path_prefix }} + PULP_URL: ${{ inputs.pulp_url }} + PULP_DOMAIN: ${{ steps.pulp-properties.outputs.domain }} + PULP_STABLE_DOMAIN: ${{ steps.pulp-properties.outputs.stable_domain }} + PULP_TOKEN: ${{ env.PULP_TOKEN }} + PULP_CONTENT_URL: ${{ inputs.pulp_content_url }} + run: ${{ github.action_path }}/deliver-rpm.sh + + - name: Deliver DEB packages + id: deliver-deb + if: steps.pulp-properties.outputs.package_extension == 'deb' && steps.pulp-properties.outputs.skip_delivery == 'false' + shell: bash + env: + MODULE_NAME: ${{ inputs.module_name }} + DISTRIB: ${{ inputs.distrib }} + STABILITY: ${{ inputs.stability }} + REPOSITORY_NAME: ${{ steps.pulp-properties.outputs.repository_name }} + BASE_PATH: ${{ steps.pulp-properties.outputs.base_path }} + LEGACY_BASE_PATH: ${{ steps.pulp-properties.outputs.legacy_base_path }} + LEGACY_STABLE_BASE_PATH: ${{ steps.pulp-properties.outputs.legacy_stable_base_path }} + SUITE: ${{ steps.pulp-properties.outputs.suite }} + STABLE_SUITE: ${{ steps.pulp-properties.outputs.stable_suite }} + STABLE_BASE_PATH: ${{ steps.pulp-properties.outputs.stable_base_path }} + POOL_PATH: ${{ steps.pulp-properties.outputs.pool_path }} + PULP_URL: ${{ inputs.pulp_url }} + PULP_DOMAIN: ${{ steps.pulp-properties.outputs.domain }} + PULP_STABLE_DOMAIN: ${{ steps.pulp-properties.outputs.stable_domain }} + PULP_TOKEN: ${{ env.PULP_TOKEN }} + PULP_CONTENT_URL: ${{ inputs.pulp_content_url }} + run: ${{ github.action_path }}/deliver-deb.sh + + - name: Verify RPM delivery + if: inputs.verify == 'true' && steps.pulp-properties.outputs.package_extension == 'rpm' && steps.pulp-properties.outputs.skip_delivery == 'false' + shell: bash + env: + MANIFEST: ${{ steps.deliver-rpm.outputs.manifest }} + CHECK_MODE: delivery + MODULE_NAME: ${{ inputs.module_name }} + DISTRIB: ${{ inputs.distrib }} + STABILITY: ${{ inputs.stability }} + PULP_URL: ${{ inputs.pulp_url }} + PULP_DOMAIN: ${{ steps.pulp-properties.outputs.domain }} + PULP_TOKEN: ${{ env.PULP_TOKEN }} + PULP_CONTENT_URL: ${{ inputs.pulp_content_url }} + run: ${{ github.action_path }}/../lib/check-rpm.sh + + - name: Verify DEB delivery + if: inputs.verify == 'true' && steps.pulp-properties.outputs.package_extension == 'deb' && steps.pulp-properties.outputs.skip_delivery == 'false' + shell: bash + env: + MANIFEST: ${{ steps.deliver-deb.outputs.manifest }} + CHECK_MODE: delivery + MODULE_NAME: ${{ inputs.module_name }} + DISTRIB: ${{ inputs.distrib }} + STABILITY: ${{ inputs.stability }} + PULP_URL: ${{ inputs.pulp_url }} + PULP_DOMAIN: ${{ steps.pulp-properties.outputs.domain }} + PULP_TOKEN: ${{ env.PULP_TOKEN }} + PULP_CONTENT_URL: ${{ inputs.pulp_content_url }} + run: ${{ github.action_path }}/../lib/check-deb.sh + + # Testing deliveries keep their manifest as an artifact for later jobs of the run. + - name: Prepare delivery manifest artifact + id: manifest-artifact + if: inputs.stability == 'testing' && inputs.delivery_type != 'feature' && (steps.deliver-rpm.outputs.manifest != '' || steps.deliver-deb.outputs.manifest != '') + shell: bash + env: + CACHE_KEY: ${{ inputs.cache_key }} + run: | + # artifact names only keep [A-Za-z0-9._-]: cache keys can carry a branch name with a / + echo "name=pulp-manifest-${CACHE_KEY//[^A-Za-z0-9._-]/-}" >> "$GITHUB_OUTPUT" + + - name: Upload delivery manifest + id: upload-manifest + if: steps.manifest-artifact.outcome == 'success' + # the packages are already delivered: a missing manifest is reported by its consumer, not here + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: ${{ steps.manifest-artifact.outputs.name }} + path: ${{ steps.deliver-rpm.outputs.manifest || steps.deliver-deb.outputs.manifest }} + retention-days: 30 + overwrite: true + if-no-files-found: error diff --git a/pulp/package-delivery/deliver-deb.sh b/pulp/package-delivery/deliver-deb.sh new file mode 100755 index 0000000..b38e6a3 --- /dev/null +++ b/pulp/package-delivery/deliver-deb.sh @@ -0,0 +1,679 @@ +#!/usr/bin/env bash +set -euo pipefail +shopt -s nullglob + +# shellcheck source=pulp/lib/manifest.sh +source "$(dirname "$0")/../lib/manifest.sh" +# shellcheck source=pulp/lib/api.sh +source "$(dirname "$0")/../lib/api.sh" + +# an unset org variable is forwarded as an empty string, overriding the default +PULP_URL="${PULP_URL:-https://pulp-api.int.centreon.com}" +PULP_CONTENT_URL="${PULP_CONTENT_URL:-https://packages.int.centreon.com}" +PULP_DOMAIN="${PULP_DOMAIN:-default}" +# read through the public content-app, no auth/grant needed (see api.sh's content_curl) +PULP_STABLE_DOMAIN="${PULP_STABLE_DOMAIN:-default}" + +STABLE_ROOT="${LEGACY_STABLE_BASE_PATH:-$PULP_STABLE_DOMAIN/${STABLE_BASE_PATH:-$BASE_PATH}}" + +# per-stability repositories sharing the legacy front's pool, in the order it +# resolves a pool path (stable -> testing-release -> testing-hotfix -> unstable): +# the first one publishing a file name serves it to every suite of the family +FAMILY_STABLE_BASE_PATH="${STABLE_BASE_PATH:-$BASE_PATH}" +FAMILY_REPOSITORIES=("$FAMILY_STABLE_BASE_PATH") +if [[ "$FAMILY_STABLE_BASE_PATH" == *stable ]]; then + FAMILY_REPOSITORIES+=( + "${FAMILY_STABLE_BASE_PATH%stable}testing-release" + "${FAMILY_STABLE_BASE_PATH%stable}testing-hotfix" + "${FAMILY_STABLE_BASE_PATH%stable}unstable" + ) +fi +# a target outside the front cascade still shares its pool between its suites +TARGET_RANK="" +for rank in "${!FAMILY_REPOSITORIES[@]}"; do + [[ "${FAMILY_REPOSITORIES[$rank]}" == "$BASE_PATH" ]] && TARGET_RANK=$rank +done +if [[ -z "$TARGET_RANK" ]]; then + TARGET_RANK=${#FAMILY_REPOSITORIES[@]} + FAMILY_REPOSITORIES+=("$BASE_PATH") +fi +MAX_PARALLEL_FETCHES=8 + +# print a published repository file (empty output = 404, i.e. not published). +# Fails closed (non-zero) on anything but a clean 404. +fetch_published_file() { + local url=$1 out_file http_code + out_file=$(mktemp) + http_code=$(content_curl -sSL --retry 3 --retry-delay 5 -o "$out_file" -w '%{http_code}' "$url" 2>/dev/null || echo 000) + case "$http_code" in + 404) rm -f "$out_file"; return 0 ;; + 200) cat "$out_file"; rm -f "$out_file" ;; + *) + rm -f "$out_file" + echo "::error::Cannot read $url (HTTP $http_code) to guard the published packages; refusing to deliver. Retry once the content endpoint is reachable." >&2 + return 1 + ;; + esac +} + +# print the load_published_packages lines of one suite +load_suite_packages() { + local rank=$1 repository=$2 suite=$3 root=$4 release arches a packages + release=$(fetch_published_file "$PULP_CONTENT_URL/$root/dists/$suite/Release") || return 1 + # no Release yet (fresh suite) -> nothing published + [[ -z "$release" ]] && return 0 + # "Architecture: all" packages are listed in every binary-/Packages + # index the suite advertises; there is no binary-all index (404, skipped) + arches=$(awk -F': ' '/^Architectures:/ {print $2}' <<< "$release") + for a in $arches; do + packages=$(fetch_published_file "$PULP_CONTENT_URL/$root/dists/$suite/main/binary-$a/Packages") || return 1 + [[ -z "$packages" ]] && continue + awk -v r="$rank" -v repo="$repository" -v s="$suite" ' + BEGIN { RS = ""; FS = "\n" } + { + n = v = ar = h = f = "" + for (i = 1; i <= NF; i++) { + if ($i ~ /^Package: /) n = substr($i, 10) + else if ($i ~ /^Version: /) v = substr($i, 10) + else if ($i ~ /^Architecture: /) ar = substr($i, 15) + else if ($i ~ /^SHA256: /) h = substr($i, 9) + else if ($i ~ /^Filename: /) f = substr($i, 11) + } + if (n != "") print r, repo, s, n, v, ar, h, f + } + ' <<< "$packages" + done +} + +# print one "rank repository suite package version architecture sha256 filename" +# line per package the family's repositories publish (rank = position in the +# front cascade). The publication lays every suite out in one flat pool +# (pool/main///), so every suite publishing a package +# version shares one pool path: pulp keeps a single build per (package, +# version, arch) per repository, and the legacy front serves the first +# repository's file to the whole family. Suites load in parallel; any failed +# one fails the whole load. +load_published_packages() { + local rank repository domain listing listed suites suite root out_dir count=0 i + out_dir=$(mktemp -d) + for rank in "${!FAMILY_REPOSITORIES[@]}"; do + repository=${FAMILY_REPOSITORIES[$rank]} + domain=$PULP_DOMAIN + [[ "$rank" -eq 0 ]] && domain=$PULP_STABLE_DOMAIN + # the dists listing is only served on the domain path, not through the legacy front + if ! listing=$(fetch_published_file "$PULP_CONTENT_URL/$domain/$repository/dists/"); then + wait || true + rm -rf "$out_dir" + return 1 + fi + listed=$(grep -oE 'href="(\./)?[^"/?]+/"' <<< "$listing" | sed -E 's|href="(\./)?([^"/?]+)/"|\2|' | grep -vx '\.\.' || true) + # a listing without any link isn't the expected directory index: guarding + # the target suite only would let a shared-pool collision through + if [[ -n "$listing" ]] && ! grep -q 'href="' <<< "$listing"; then + echo "::error::Cannot parse the suites from $domain/$repository/dists/; refusing to deliver." >&2 + wait || true + rm -rf "$out_dir" + return 1 + fi + suites=$listed + [[ "$rank" -eq 0 ]] && suites=$(printf '%s\n' "$STABLE_SUITE" "$listed" | awk 'NF && !seen[$0]++') + for suite in $suites; do + root=$domain/$repository + [[ "$rank" -eq 0 && "$suite" == "$STABLE_SUITE" ]] && root=$STABLE_ROOT + # one background job per suite; only a completed one leaves its marker + ( + refresh_pulp_token + load_suite_packages "$rank" "$repository" "$suite" "$root" > "$out_dir/$count.entries" && + : > "$out_dir/$count.done" + ) & + count=$((count + 1)) + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_FETCHES)); do + wait -n || true + done + done + done + wait || true + for ((i = 0; i < count; i++)); do + if [[ ! -e "$out_dir/$i.done" ]]; then + rm -rf "$out_dir" + return 1 + fi + done + for ((i = 0; i < count; i++)); do + cat "$out_dir/$i.entries" + done + rm -rf "$out_dir" +} + +# print a digest of what a deb installs, blind to what a rebuild of unchanged +# sources changes: mtimes, the description (it embeds the commit) and the owner +# spelling (root/root or 0/0 depending on the nfpm version) +deb_fingerprint() { + local file=$1 dir + dir=$(mktemp -d) + if ! { + dpkg-deb -e "$file" "$dir/control" && + dpkg-deb -x "$file" "$dir/data" && + awk '/^Description:/ { skip = 1; next } skip && /^[ \t]/ { next } { skip = 0; print }' "$dir/control/control" && + (cd "$dir/control" && find . -type f ! -path ./control -print0 | LC_ALL=C sort -z | xargs -0r sha256sum) && + dpkg-deb -c "$file" | awk '{ split($2, o, "/"); for (i = 1; i <= 2; i++) if (o[i] == "root") o[i] = 0; $2 = o[1] "/" o[2]; $4 = $5 = ""; print }' | LC_ALL=C sort && + (cd "$dir/data" && find . -type f -print0 | LC_ALL=C sort -z | xargs -0r sha256sum) + } > "$dir/manifest"; then + rm -rf "$dir" + return 1 + fi + sha256sum "$dir/manifest" | cut -d' ' -f1 + rm -rf "$dir" +} + +# when another suite of the family publishes another build of this package +# version, deliver the build the legacy front serves first instead of the local +# one if both install the same thing (a rebuild), so every index matches the +# served file. A build differing in content needs a version bump: testing +# refuses it where the other build would mask it or be evicted by it; unstable, +# whose unversioned packages are rebuilt on every push, delivers it with a warning. +# Testing never adopts from a lower stability: its own build is served first, +# and promote would ship the lower pipeline's build to stable. +adopt_published_build() { + local file=$1 name version arch sha256 max_rank entry rank repository suite published_sha256 filename domain tmp_file + local local_fingerprint published_fingerprint reason lower_suite + name=$(dpkg-deb -f "$file" Package) + version=$(dpkg-deb -f "$file" Version) + arch=$(dpkg-deb -f "$file" Architecture) + sha256=$(sha256sum "$file" | cut -d' ' -f1) + # already in this version's stable suite: assert_not_in_stable refuses it + if [[ "${STABILITY:-}" != "unstable" ]] && awk -v s="$STABLE_SUITE" -v n="$name" -v v="$version" -v a="$arch" \ + '$1 == 0 && $3 == s && $4 == n && $5 == v && $6 == a { found = 1 } END { exit found ? 0 : 1 }' "$PUBLISHED_PACKAGES"; then + return 0 + fi + max_rank=$TARGET_RANK + [[ "${STABILITY:-}" == "unstable" ]] && max_rank=${#FAMILY_REPOSITORIES[@]} + # the build the front serves first among the eligible ranks + entry=$(awk -v t="$BASE_PATH" -v s="$SUITE" -v n="$name" -v v="$version" -v a="$arch" -v m="$max_rank" ' + !($2 == t && $3 == s) && $1 <= m && $4 == n && $5 == v && $6 == a && (best == "" || $1 < best_rank) { best_rank = $1; best = $0 } + END { if (best != "") print best } + ' "$PUBLISHED_PACKAGES") + if [[ -z "$entry" ]]; then + lower_suite=$(awk -v n="$name" -v v="$version" -v a="$arch" -v h="$sha256" -v m="$max_rank" \ + '$1 > m && $4 == n && $5 == v && $6 == a && $7 != h { print $3; exit }' "$PUBLISHED_PACKAGES") + if [[ -n "$lower_suite" ]]; then + echo "::notice::$name $version ($arch) is also published in $lower_suite with another build: delivering the local build, which the legacy front serves for $lower_suite too until its next delivery." + fi + return 0 + fi + read -r rank repository suite _ _ _ published_sha256 filename <<< "$entry" + # nothing to do when the build served first is the local one + [[ "$published_sha256" == "$sha256" ]] && return 0 + domain=$PULP_DOMAIN + [[ "$rank" -eq 0 ]] && domain=$PULP_STABLE_DOMAIN + + tmp_file=$(mktemp) + if ! content_curl -fsSL --retry 3 --retry-delay 5 -o "$tmp_file" "$PULP_CONTENT_URL/$domain/$repository/$filename"; then + rm -f "$tmp_file" + echo "::error::Cannot download the $suite build of $name $version ($arch) to compare it with the local one; refusing to deliver." + return 1 + fi + if [[ "$(sha256sum "$tmp_file" | cut -d' ' -f1)" != "$published_sha256" \ + || "$(dpkg-deb -f "$tmp_file" Package)" != "$name" \ + || "$(dpkg-deb -f "$tmp_file" Version)" != "$version" \ + || "$(dpkg-deb -f "$tmp_file" Architecture)" != "$arch" ]]; then + rm -f "$tmp_file" + echo "::error::The downloaded $suite build of $name $version ($arch) doesn't match its index entry; refusing to deliver." + return 1 + fi + if ! local_fingerprint=$(deb_fingerprint "$file") || ! published_fingerprint=$(deb_fingerprint "$tmp_file"); then + rm -f "$tmp_file" + echo "::error::Cannot compare the content of $name $version ($arch) with its $suite build; refusing to deliver." + return 1 + fi + + if [[ "$local_fingerprint" == "$published_fingerprint" ]]; then + mv -f "$tmp_file" "$file" + chmod 644 "$file" + ADOPTED[$file]=1 + echo "::notice::$name $version ($arch) is already published in $suite (sha256 $published_sha256) with the same content: delivering that build instead of the local rebuild (sha256 $sha256), both share one pool path." + return 0 + fi + rm -f "$tmp_file" + + if ((rank < TARGET_RANK)); then + reason="the legacy front serves the $suite build for $SUITE too" + elif ((rank == TARGET_RANK)); then + reason="it replaces the $suite build in $repository" + else + reason="the legacy front serves it for $suite too" + fi + if [[ "${STABILITY:-}" != "unstable" ]] && ((rank <= TARGET_RANK)); then + echo "::error::$name $version ($arch) differs in content from the build published in $suite (sha256 $published_sha256) and shares its pool path: $reason. Refusing to deliver it to $SUITE; bump the package version." + return 1 + fi + echo "::warning::$name $version ($arch) differs in content from the build published in $suite (sha256 $published_sha256) and shares its pool path: delivering the local build (sha256 $sha256), $reason. Bump the package version." +} + +# refuse delivering a package version already published in the stable suite +# (pulp dedupes by (package, version, arch) repository-wide, so re-delivering +# would silently evict the stable one), or another build of a version published +# in any other stable suite (same pool path, the stable file would be served; +# adopt_published_build normally replaced it beforehand). The release_components +# api isn't listable by the OIDC ci-user (403), so this reads the served +# Packages indexes loaded into STABLE_PACKAGES. +assert_not_in_stable() { + local file=$1 name version arch sha256 conflict + # unstable is never promoted to stable: a rebuilt package may reuse a version + # already published there (unversioned plugins/connectors packages) + [[ "${STABILITY:-}" == "unstable" ]] && return 0 + name=$(dpkg-deb -f "$file" Package) + version=$(dpkg-deb -f "$file" Version) + arch=$(dpkg-deb -f "$file" Architecture) + sha256=$(sha256sum "$file" | cut -d' ' -f1) + + if awk -v s="$STABLE_SUITE" -v n="$name" -v v="$version" -v a="$arch" \ + '$1 == s && $2 == n && $3 == v && $4 == a { found = 1 } END { exit found ? 0 : 1 }' "$STABLE_PACKAGES"; then + echo "::error::$name $version ($arch) is already published in the stable suite $STABLE_SUITE; refusing to deliver it to $SUITE. Bump the package version for a new build." + return 1 + fi + + conflict=$(awk -v n="$name" -v v="$version" -v a="$arch" -v h="$sha256" \ + '$2 == n && $3 == v && $4 == a && $5 != h { print $1; exit }' "$STABLE_PACKAGES") + if [[ -n "$conflict" ]]; then + echo "::error::Another build of $name $version ($arch) is published in the stable suite $conflict and shares its pool path; refusing to deliver it to $SUITE, the stable file would be served instead. Bump the package version." + return 1 + fi +} + +FILES=(*.deb) +if [[ ${#FILES[@]} -eq 0 ]]; then + echo "::error::No deb package found to deliver" + exit 1 +fi + +if ! pulp_resource_exists "repositories/deb/apt" "$REPOSITORY_NAME"; then + echo "::error::deb repository $REPOSITORY_NAME does not exist. Pulp repositories and distributions are provisioned centrally by delivery-tooling create-repos; run create-repos for this version before delivering." + exit 1 +fi + +if ! pulp_resource_exists "distributions/deb/apt" "$REPOSITORY_NAME"; then + echo "::error::deb distribution $REPOSITORY_NAME does not exist. Pulp distributions are provisioned centrally by delivery-tooling create-repos; run create-repos for this version before delivering. Refusing to create it here to avoid an unguarded distribution." + exit 1 +fi + +REPOSITORY_HREF=$(pulp deb repository show --name "$REPOSITORY_NAME" | jq -r '.pulp_href') + +# loaded once, read by every adopt_published_build and assert_not_in_stable +# (parallel uploads included); adoptions happen before anything reads the local +# files' checksums +PUBLISHED_PACKAGES=$(mktemp) +STABLE_PACKAGES=$(mktemp) +declare -A ADOPTED=() +refresh_pulp_token +load_published_packages > "$PUBLISHED_PACKAGES" +awk '$1 == 0 { print $3, $4, $5, $6, $7, $8 }' "$PUBLISHED_PACKAGES" > "$STABLE_PACKAGES" +for FILE in "${FILES[@]}"; do + adopt_published_build "$FILE" +done + +# record every delivered package in the manifest for the verification step, +# publish, and write the manifest +record_and_publish() { + local file name version arch sha256 + for file in "${FILES[@]}"; do + name=$(dpkg-deb -f "$file" Package) + version=$(dpkg-deb -f "$file" Version) + arch=$(dpkg-deb -f "$file" Architecture) + sha256=$(sha256sum "$file" | cut -d' ' -f1) + manifest_add "$(jq -cn \ + --arg filename "$file" --arg name "$name" --arg version "$version" \ + --arg arch "$arch" --arg sha256 "$sha256" --arg repository "$REPOSITORY_NAME" \ + --arg base_path "${LEGACY_BASE_PATH:-$PULP_DOMAIN/$BASE_PATH}" --arg suite "$SUITE" --arg relative_path "$POOL_PATH/$file" \ + '{filename:$filename,name:$name,version:$version,arch:$arch,sha256:$sha256,repository:$repository,base_path:$base_path,suite:$suite,relative_path:$relative_path}')" + done + + echo "[INFO] Publishing repository $REPOSITORY_NAME" + create_publication deb "$REPOSITORY_NAME" --structured + + echo "::notice::Packages are available with: deb $PULP_CONTENT_URL/${LEGACY_BASE_PATH:-$PULP_DOMAIN/$BASE_PATH}/ $SUITE main" + + manifest_write "$MODULE_NAME" "${DISTRIB:-}" "deb" "${STABILITY:-}" "delivery" "$PULP_CONTENT_URL" +} + +# an adopted build already published in the target suite (a previous delivery +# of the same version) needs no upload: it is reused content whose suite +# association pre-exists, from which no release component can be deduced +DELIVER_FILES=("${FILES[@]}") +if ((${#ADOPTED[@]} > 0)); then + TARGET_ROOT="${LEGACY_BASE_PATH:-$PULP_DOMAIN/$BASE_PATH}" + TARGET_SHA256S="" + TARGET_RELEASE=$(fetch_published_file "$PULP_CONTENT_URL/$TARGET_ROOT/dists/$SUITE/Release") + read -ra TARGET_ARCHES <<< "$(awk -F': ' '/^Architectures:/ {print $2}' <<< "$TARGET_RELEASE")" + for a in "${TARGET_ARCHES[@]}"; do + TARGET_SHA256S+=$(fetch_published_file "$PULP_CONTENT_URL/$TARGET_ROOT/dists/$SUITE/main/binary-$a/Packages" | awk -F': ' '/^SHA256:/ {print $2}') + TARGET_SHA256S+=$'\n' + done + DELIVER_FILES=() + for FILE in "${FILES[@]}"; do + if [[ -n "${ADOPTED[$FILE]+set}" ]] && grep -qxF "$(sha256sum "$FILE" | cut -d' ' -f1)" <<< "$TARGET_SHA256S"; then + echo "[INFO] $FILE: the adopted build is already published in $SUITE, nothing to upload" + else + DELIVER_FILES+=("$FILE") + fi + done + if ((${#DELIVER_FILES[@]} == 0)); then + record_and_publish + exit 0 + fi +fi + +PULP_LABELS=$(jq -cn \ + --arg mod "$MODULE_NAME" \ + --arg git_commit "${GITHUB_SHA:-}" \ + --arg git_ref "${GITHUB_REF:-}" \ + --arg run_id "${GITHUB_RUN_ID:-}" \ + --arg actor "${GITHUB_ACTOR:-}" \ + --arg workflow "${GITHUB_WORKFLOW:-}" \ + '{"module": $mod, "git_commit": $git_commit, "git_ref": $git_ref, "github_run_id": $run_id, "github_actor": $actor, "github_workflow": $workflow}') + +# a repository-less deb upload ignores distribution/component, so the FIRST +# package of each arch goes through the legacy repository path instead to +# establish the suite's release component; the rest upload unassociated and +# get associated explicitly, then all of it is added in a single modify. +lookup_deb_content() { + # emit the href of a matching deb content unit, empty if absent. Retried: + # the api has been observed answering an empty page for content committed + # seconds earlier (stale read) + local endpoint=$1 query=$2 out attempt + for attempt in 1 2 3 4 5; do + out=$(curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Bearer $PULP_TOKEN" -G \ + --data-urlencode "limit=1" \ + $query \ + "$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/$endpoint/" | jq -r '.results[0].pulp_href // empty') || out="" + if [[ -n "$out" ]]; then + echo "$out" + return 0 + fi + sleep $((attempt * 3)) + refresh_pulp_token + done +} + +resolve_task_content() { + # wait for a content-create task and emit its created content href; fall + # back to a lookup (content already existing on a job re-run) + local task_href=$1 endpoint=$2 fallback_query=$3 + local body state content attempt + for ((attempt = 0; attempt < 200; attempt++)); do + refresh_pulp_token + body=$(curl -fsSL -H "Authorization: Bearer $PULP_TOKEN" "$PULP_URL$task_href" 2>/dev/null) || body="" + state=$(echo "$body" | jq -r '.state' 2>/dev/null) || state="" + case "$state" in + completed) + content=$(echo "$body" | jq -r '.created_resources[0] // empty') + if [[ -n "$content" ]]; then + echo "$content" + return 0 + fi + break + ;; + failed | canceled) + break + ;; + *) + sleep 3 + ;; + esac + done + content=$(lookup_deb_content "$endpoint" "$fallback_query") + if [[ -z "$content" ]]; then + echo "::error::Cannot resolve the created deb content for task $task_href" >&2 + return 1 + fi + echo "$content" +} + +# emit the release-component hrefs a package is associated with +lookup_prcs() { + local package_href=$1 + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Bearer $PULP_TOKEN" -G \ + --data-urlencode "package=$package_href" \ + --data-urlencode "limit=100" \ + "$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/package_release_components/" \ + | jq -r '.results[].release_component' +} + +# pick one file per arch for the legacy path. Prefer content that doesn't +# exist in pulp yet: a fresh upload carries exactly one suite association +# afterwards, identifying the release component unambiguously. For a reused +# representative (identical bytes delivered before), capture its pre-upload +# associations to diff against later. Content is unique on (relative_path, +# sha256): the same bytes under another pool path (an adopted build) are a +# distinct unit, so every lookup filters on both. +declare -A LEGACY_FOR_ARCH=() +declare -A LEGACY_FRESH=() +declare -A LEGACY_BEFORE=() +refresh_pulp_token +for i in "${!DELIVER_FILES[@]}"; do + FILE=${DELIVER_FILES[$i]} + arch=$(dpkg-deb -f "$FILE" Architecture) + if [[ -n "${LEGACY_FRESH[$arch]+set}" ]]; then + continue + fi + refresh_pulp_token + sha=$(sha256sum "$FILE" | cut -d' ' -f1) + existing=$(lookup_deb_content "packages" "--data-urlencode sha256=$sha --data-urlencode relative_path=$POOL_PATH/$FILE") + if [[ -z "$existing" ]]; then + LEGACY_FOR_ARCH[$arch]="$FILE" + LEGACY_FRESH[$arch]=1 + elif [[ -z "${LEGACY_FOR_ARCH[$arch]+set}" ]]; then + LEGACY_FOR_ARCH[$arch]="$FILE" + LEGACY_BEFORE[$arch]=$(lookup_prcs "$existing" | sort) + fi +done + +declare -A IS_LEGACY=() +LEGACY_FILES=() +LEGACY_ARCHS=() +for arch in "${!LEGACY_FOR_ARCH[@]}"; do + LEGACY_FILES+=("${LEGACY_FOR_ARCH[$arch]}") + LEGACY_ARCHS+=("$arch") + IS_LEGACY["${LEGACY_FOR_ARCH[$arch]}"]=1 +done +ORPHAN_FILES=() +for FILE in "${DELIVER_FILES[@]}"; do + [[ -n "${IS_LEGACY[$FILE]+set}" ]] || ORPHAN_FILES+=("$FILE") +done + +refresh_pulp_token +for FILE in "${LEGACY_FILES[@]}"; do + assert_not_in_stable "$FILE" +done +# sequential with retry: the legacy path creates a repository version, and +# concurrent legs of the shared repository can lose the version race (rc 2) +for FILE in "${LEGACY_FILES[@]}"; do + for legacy_attempt in 1 2 3; do + echo "[INFO] Uploading $FILE to $POOL_PATH/ ($SUITE/main, module $MODULE_NAME) [legacy path, attempt $legacy_attempt]" + LEGACY_TASK=$( + pulp_upload \ + -F "file=@\"$FILE\"" \ + -F "relative_path=$POOL_PATH/$FILE" \ + -F "distribution=$SUITE" \ + -F "component=main" \ + -F "repository=$REPOSITORY_HREF" \ + -F "pulp_labels=$PULP_LABELS" \ + "$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/packages/" + ) + wait_task_race "$LEGACY_TASK" && rc=0 || rc=$? + if [[ $rc -eq 0 ]]; then + break + elif [[ $rc -eq 2 && $legacy_attempt -lt 3 ]]; then + echo "[WARN] Legacy upload of $FILE lost the repository-version race against a concurrent delivery, retrying" + sleep $((legacy_attempt * 15)) + else + echo "::error::Legacy upload of $FILE failed" + exit 1 + fi + done +done + +# deduce the release component href from the association the legacy uploads +# just created: a fresh representative now carries exactly one association; a +# reused one carries its prior ones too, so diff post- against pre-upload +refresh_pulp_token +RELEASE_COMPONENT_HREF="" +for i in "${!LEGACY_FILES[@]}"; do + FILE=${LEGACY_FILES[$i]} + arch=${LEGACY_ARCHS[$i]} + sha=$(sha256sum "$FILE" | cut -d' ' -f1) + href=$(lookup_deb_content "packages" "--data-urlencode sha256=$sha --data-urlencode relative_path=$POOL_PATH/$FILE") + if [[ -z "$href" ]]; then + echo "::error::Cannot find the legacy-delivered package $FILE (sha256 $sha)" + exit 1 + fi + after=$(lookup_prcs "$href" | sort) + if [[ -n "${LEGACY_FRESH[$arch]+set}" ]]; then + if [[ $(echo "$after" | grep -c .) -eq 1 ]]; then + RELEASE_COMPONENT_HREF="$after" + break + fi + else + new_rcs=$(comm -13 <(echo "${LEGACY_BEFORE[$arch]}") <(echo "$after") | grep . || true) + if [[ $(echo "$new_rcs" | grep -c .) -eq 1 ]]; then + RELEASE_COMPONENT_HREF="$new_rcs" + break + elif [[ $(echo "$after" | grep -c .) -eq 1 ]]; then + # rerun of a partially delivered leg: empty diff, but it's the only association + RELEASE_COMPONENT_HREF=$(echo "$after" | grep .) + break + fi + fi +done +if [[ -z "$RELEASE_COMPONENT_HREF" ]]; then + echo "::error::Cannot deduce the $SUITE/main release component: every legacy representative is reused content whose suite association pre-exists. Deliver a rebuilt (fresh) package or clean up the previous partial delivery." + exit 1 +fi +echo "[INFO] Release component of $SUITE/main: $RELEASE_COMPONENT_HREF" + +# parallel repository-less uploads (pool of 8), marker-file based like rpm +UPLOAD_DIR=$(mktemp -d) +MAX_PARALLEL_UPLOADS=8 +for i in "${!ORPHAN_FILES[@]}"; do + FILE=${ORPHAN_FILES[$i]} + refresh_pulp_token + ( + # subshell-local: the inherited token can go stale between parent refreshes + refresh_pulp_token + assert_not_in_stable "$FILE" + echo "[INFO] Uploading $FILE to $POOL_PATH/ ($SUITE/main, module $MODULE_NAME)" + pulp_upload \ + -F "file=@\"$FILE\"" \ + -F "relative_path=$POOL_PATH/$FILE" \ + -F "pulp_labels=$PULP_LABELS" \ + "$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/packages/" > "$UPLOAD_DIR/$i.task" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done +done +wait || true + +echo "[INFO] Resolving ${#ORPHAN_FILES[@]} uploaded package(s)" +ORPHAN_SHA256S=() +for i in "${!ORPHAN_FILES[@]}"; do + FILE=${ORPHAN_FILES[$i]} + if [[ ! -s "$UPLOAD_DIR/$i.task" ]]; then + echo "::error::Upload failed for $FILE (no task href, see the worker error above)" + exit 1 + fi + ORPHAN_SHA256S+=("$(sha256sum "$FILE" | cut -d' ' -f1)") +done +for i in "${!ORPHAN_FILES[@]}"; do + refresh_pulp_token + ( + resolve_task_content "$(cat "$UPLOAD_DIR/$i.task")" "packages" \ + "--data-urlencode sha256=${ORPHAN_SHA256S[$i]} --data-urlencode relative_path=$POOL_PATH/${ORPHAN_FILES[$i]}" > "$UPLOAD_DIR/$i.content" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done +done +wait || true +PACKAGE_HREFS=() +for i in "${!ORPHAN_FILES[@]}"; do + if [[ ! -s "$UPLOAD_DIR/$i.content" ]]; then + echo "::error::Cannot resolve the uploaded content for ${ORPHAN_FILES[$i]} (see the worker error above)" + exit 1 + fi + PACKAGE_HREFS+=("$(cat "$UPLOAD_DIR/$i.content")") +done + +# package_release_components create is synchronous (201 with the unit, no +# task); a failed create (already existing on a job re-run) falls back to a lookup +echo "[INFO] Associating ${#PACKAGE_HREFS[@]} package(s) with $SUITE/main" +for i in "${!PACKAGE_HREFS[@]}"; do + refresh_pulp_token + ( + refresh_pulp_token + out=$(post_json "$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/package_release_components/" \ + "{\"package\": \"${PACKAGE_HREFS[$i]}\", \"release_component\": \"$RELEASE_COMPONENT_HREF\"}") || out=$'\n000' + code="${out##*$'\n'}" + body="${out%$'\n'*}" + href="" + if [[ "$code" == 2* ]]; then + # tolerate a non-json body (gateway error page behind a 2xx) + href=$(echo "$body" | jq -r '.pulp_href // .task // empty' 2>/dev/null) || href="" + fi + if [[ -z "$href" ]]; then + href=$(lookup_deb_content "package_release_components" \ + "--data-urlencode package=${PACKAGE_HREFS[$i]} --data-urlencode release_component=$RELEASE_COMPONENT_HREF") + if [[ -n "$href" ]]; then + # api answers 500 on a duplicate synchronous create; expected on a rerun + echo "[INFO] ${ORPHAN_FILES[$i]}: suite association already exists (HTTP $code on create, expected on a rerun), reusing it" + else + echo "[WARN] Suite association create for ${ORPHAN_FILES[$i]} returned HTTP $code: $(echo "$body" | head -c 300)" >&2 + fi + fi + printf '%s' "$href" > "$UPLOAD_DIR/$i.prc" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done +done +wait || true + +PRC_HREFS=() +for i in "${!PACKAGE_HREFS[@]}"; do + href="" + [[ -s "$UPLOAD_DIR/$i.prc" ]] && href=$(cat "$UPLOAD_DIR/$i.prc") + if [[ "$href" == */tasks/* ]]; then + href=$(resolve_task_content "$href" "package_release_components" \ + "--data-urlencode package=${PACKAGE_HREFS[$i]} --data-urlencode release_component=$RELEASE_COMPONENT_HREF") + fi + if [[ -z "$href" ]]; then + echo "::error::Suite association failed for ${ORPHAN_FILES[$i]} (see the worker error above)" + exit 1 + fi + PRC_HREFS+=("$href") +done +rm -rf "$UPLOAD_DIR" + +if ((${#PACKAGE_HREFS[@]} > 0)); then + echo "[INFO] Adding ${#PACKAGE_HREFS[@]} package(s) and their suite associations to $REPOSITORY_NAME in a single task" + refresh_pulp_token + # body through a file: thousands of hrefs exceed the argv limit + ADD_BODY_FILE=$(mktemp) + printf '%s\n' "${PACKAGE_HREFS[@]}" "${PRC_HREFS[@]}" | jq -R . | jq -cs '{add_content_units: .}' > "$ADD_BODY_FILE" + # retried like the legacy uploads: same repository-version race + for modify_attempt in 1 2 3; do + MODIFY_TASK=$(start_modify_task "$PULP_URL${REPOSITORY_HREF}modify/" "$ADD_BODY_FILE") + wait_task_race "$MODIFY_TASK" && rc=0 || rc=$? + if [[ $rc -eq 0 ]]; then + break + elif [[ $rc -eq 2 && $modify_attempt -lt 3 ]]; then + echo "[WARN] Repository modify lost the repository-version race against a concurrent delivery, retrying" + sleep $((modify_attempt * 15)) + else + echo "::error::Repository modify failed" + exit 1 + fi + done +fi + +record_and_publish diff --git a/pulp/package-delivery/deliver-rpm.sh b/pulp/package-delivery/deliver-rpm.sh new file mode 100755 index 0000000..b9165e1 --- /dev/null +++ b/pulp/package-delivery/deliver-rpm.sh @@ -0,0 +1,256 @@ +#!/usr/bin/env bash +set -euo pipefail +shopt -s nullglob + +# shellcheck source=pulp/lib/manifest.sh +source "$(dirname "$0")/../lib/manifest.sh" +# shellcheck source=pulp/lib/api.sh +source "$(dirname "$0")/../lib/api.sh" + +# an unset org variable is forwarded as an empty string, overriding the default +PULP_URL="${PULP_URL:-https://pulp-api.int.centreon.com}" +PULP_CONTENT_URL="${PULP_CONTENT_URL:-https://packages.int.centreon.com}" +PULP_DOMAIN="${PULP_DOMAIN:-default}" +# read-only grant into the stable domain, used by assert_not_in_stable below +PULP_STABLE_DOMAIN="${PULP_STABLE_DOMAIN:-default}" + +# refuse delivering a package version already published in the stable repository +assert_not_in_stable() { + local file=$1 arch=$2 base name version release stable_repository repository_version count + # unstable is never promoted to stable: a rebuilt package may reuse a version + # already published there (unversioned plugins/connectors packages) + [[ "${STABILITY:-}" == "unstable" ]] && return 0 + base=$(basename "$file" .rpm) + base=${base%.$arch} + release=${base##*-} + base=${base%-*} + version=${base##*-} + name=${base%-*} + + stable_repository="$STABLE_REPOSITORY_PREFIX-$arch" + # fail closed on an unreachable api, fail open only if the repo doesn't exist yet + local repo_page + repo_page=$( + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Bearer $PULP_TOKEN" -G \ + --data-urlencode "name=$stable_repository" \ + --data-urlencode "limit=1" \ + "$PULP_URL/$PULP_STABLE_DOMAIN/api/v3/repositories/rpm/rpm/" + ) || { + echo "::error::Cannot check the stable repository $stable_repository to guard $name $version-$release ($arch); refusing to deliver. Retry once the api is reachable." + return 1 + } + repository_version=$(echo "$repo_page" | jq -r '.results[0].latest_version_href // empty') + # no stable repository yet => nothing can be in stable + [[ -z "$repository_version" ]] && return 0 + + count=$( + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Bearer $PULP_TOKEN" -G \ + --data-urlencode "repository_version=$repository_version" \ + --data-urlencode "name=$name" \ + --data-urlencode "version=$version" \ + --data-urlencode "release=$release" \ + --data-urlencode "arch=$arch" \ + --data-urlencode "limit=1" \ + "$PULP_URL/$PULP_STABLE_DOMAIN/api/v3/content/rpm/packages/" | jq -r '.count' + ) || { + echo "::error::Cannot check the stable repository $stable_repository content to guard $name $version-$release ($arch); refusing to deliver. Retry once the api is reachable." + return 1 + } + if [[ "$count" -gt 0 ]]; then + echo "::error::$name $version-$release ($arch) is already published in the stable repository $stable_repository; refusing to deliver it to $REPOSITORY_NAME. Bump the package version for a new build." + return 1 + fi +} + +# wait for the upload task and emit its content href; fall back to a sha256 +# lookup if the task produced none (content already existing on a job re-run) +resolve_uploaded_content() { + local task_href=$1 sha256=$2 + local body state content attempt + for ((attempt = 0; attempt < 200; attempt++)); do + refresh_pulp_token + body=$(curl -fsSL -H "Authorization: Bearer $PULP_TOKEN" "$PULP_URL$task_href" 2>/dev/null) || body="" + state=$(echo "$body" | jq -r '.state' 2>/dev/null) || state="" + case "$state" in + completed) + content=$(echo "$body" | jq -r '.created_resources[0] // empty') + if [[ -n "$content" ]]; then + echo "$content" + return 0 + fi + break + ;; + failed | canceled) + break + ;; + *) + sleep 3 + ;; + esac + done + content=$( + curl -fsSL -H "Authorization: Bearer $PULP_TOKEN" -G \ + --data-urlencode "sha256=$sha256" \ + --data-urlencode "limit=1" \ + "$PULP_URL/$PULP_DOMAIN/api/v3/content/rpm/packages/" | jq -r '.results[0].pulp_href // empty' + ) + if [[ -z "$content" ]]; then + echo "::error::Cannot resolve the uploaded content for task $task_href (sha256 $sha256)" >&2 + return 1 + fi + echo "$content" +} + +FILES=(*.rpm) +if [[ ${#FILES[@]} -eq 0 ]]; then + echo "::error::No rpm package found to deliver" + exit 1 +fi + +mkdir -p noarch x86_64 +for FILE in "${FILES[@]}"; do + ARCH=$(echo "$FILE" | grep -oP '(x86_64|noarch)' | head -1 || true) + if [[ -z "$ARCH" ]]; then + echo "::error::Cannot find the architecture of package $FILE" + exit 1 + fi + mv "$FILE" "$ARCH" +done + +PULP_LABELS=$(jq -cn \ + --arg mod "$MODULE_NAME" \ + --arg git_commit "${GITHUB_SHA:-}" \ + --arg git_ref "${GITHUB_REF:-}" \ + --arg run_id "${GITHUB_RUN_ID:-}" \ + --arg actor "${GITHUB_ACTOR:-}" \ + --arg workflow "${GITHUB_WORKFLOW:-}" \ + '{"module": $mod, "git_commit": $git_commit, "git_ref": $git_ref, "github_run_id": $run_id, "github_actor": $actor, "github_workflow": $workflow}') + +for ARCH in noarch x86_64; do + ARCH_FILES=("$ARCH"/*.rpm) + if [[ ${#ARCH_FILES[@]} -eq 0 ]]; then + continue + fi + + REPOSITORY_NAME="$REPOSITORY_PREFIX-$ARCH" + BASE_PATH="$BASE_PATH_PREFIX/$ARCH" + LEGACY_BASE_PATH="" + [[ -n "$LEGACY_BASE_PATH_PREFIX" ]] && LEGACY_BASE_PATH="$LEGACY_BASE_PATH_PREFIX/$ARCH" + + if ! pulp_resource_exists "repositories/rpm/rpm" "$REPOSITORY_NAME"; then + echo "::error::rpm repository $REPOSITORY_NAME does not exist. Pulp repositories and distributions are provisioned centrally by delivery-tooling create-repos; run create-repos for this version before delivering." + exit 1 + fi + + if ! pulp_resource_exists "distributions/rpm/rpm" "$REPOSITORY_NAME"; then + echo "::error::rpm distribution $REPOSITORY_NAME does not exist. Pulp distributions are provisioned centrally by delivery-tooling create-repos; run create-repos for this version before delivering. Refusing to create it here to avoid an unguarded distribution." + exit 1 + fi + + REPOSITORY_HREF=$(pulp rpm repository show --name "$REPOSITORY_NAME" | jq -r '.pulp_href') + + # upload as unassociated (repository-less) content so creates parallelize + # across pulp workers, then add the whole batch in one modify task + TASK_HREFS=() + SHA256S=() + # bounded pool of background subshells upload in parallel, each writing its + # task href to a marker file; a missing/empty marker means that worker failed + UPLOAD_DIR=$(mktemp -d) + MAX_PARALLEL_UPLOADS=8 + for i in "${!ARCH_FILES[@]}"; do + FILE=${ARCH_FILES[$i]} + refresh_pulp_token + ( + # subshell-local: the inherited token can go stale between parent refreshes + refresh_pulp_token + assert_not_in_stable "$FILE" "$ARCH" + echo "[INFO] Uploading $(basename "$FILE") (module $MODULE_NAME)" + pulp_upload \ + -F "file=@\"$FILE\"" \ + -F "pulp_labels=$PULP_LABELS" \ + "$PULP_URL/$PULP_DOMAIN/api/v3/content/rpm/packages/" > "$UPLOAD_DIR/$i.task" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done + done + wait || true + + for i in "${!ARCH_FILES[@]}"; do + FILE=${ARCH_FILES[$i]} + if [[ ! -s "$UPLOAD_DIR/$i.task" ]]; then + echo "::error::Upload failed for $(basename "$FILE") (no task href, see the worker error above)" + exit 1 + fi + TASK_HREFS+=("$(cat "$UPLOAD_DIR/$i.task")") + sha256=$(sha256sum "$FILE" | cut -d' ' -f1) + SHA256S+=("$sha256") + + # name/version/release parsed the same way as assert_not_in_stable + FILENAME=$(basename "$FILE") + base=${FILENAME%.rpm} + base=${base%."$ARCH"} + release=${base##*-} + base=${base%-*} + version=${base##*-} + name=${base%-*} + manifest_add "$(jq -cn \ + --arg filename "$FILENAME" --arg name "$name" --arg version "$version" \ + --arg release "$release" --arg arch "$ARCH" --arg sha256 "$sha256" \ + --arg repository "$REPOSITORY_NAME" --arg base_path "${LEGACY_BASE_PATH:-$PULP_DOMAIN/$BASE_PATH}" \ + '{filename:$filename,name:$name,version:$version,release:$release,arch:$arch,sha256:$sha256,repository:$repository,base_path:$base_path}')" + done + rm -rf "$UPLOAD_DIR" + + echo "[INFO] Waiting for ${#TASK_HREFS[@]} upload task(s) and resolving the content" + # parallelized like the uploads: sequential took one task GET per package (~6 min at 675) + RESOLVE_DIR=$(mktemp -d) + for i in "${!TASK_HREFS[@]}"; do + refresh_pulp_token + ( + resolve_uploaded_content "${TASK_HREFS[$i]}" "${SHA256S[$i]}" > "$RESOLVE_DIR/$i.content" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL_UPLOADS)); do + wait -n || true + done + done + wait || true + + CONTENT_HREFS=() + for i in "${!TASK_HREFS[@]}"; do + if [[ ! -s "$RESOLVE_DIR/$i.content" ]]; then + echo "::error::Cannot resolve the uploaded content of task ${TASK_HREFS[$i]} (see the worker error above)" + exit 1 + fi + CONTENT_HREFS+=("$(cat "$RESOLVE_DIR/$i.content")") + done + rm -rf "$RESOLVE_DIR" + + echo "[INFO] Adding ${#CONTENT_HREFS[@]} package(s) to $REPOSITORY_NAME in a single task" + # the subshell refreshes above never updated this shell's token + refresh_pulp_token + # body through a file: thousands of hrefs exceed the argv limit + ADD_BODY_FILE=$(mktemp) + printf '%s\n' "${CONTENT_HREFS[@]}" | jq -R . | jq -cs '{add_content_units: .}' > "$ADD_BODY_FILE" + # several modules deliver into the same repository (version race) + for modify_attempt in 1 2 3; do + MODIFY_TASK=$(start_modify_task "$PULP_URL${REPOSITORY_HREF}modify/" "$ADD_BODY_FILE") + wait_task_race "$MODIFY_TASK" && rc=0 || rc=$? + if [[ $rc -eq 0 ]]; then + break + elif [[ $rc -eq 2 && $modify_attempt -lt 3 ]]; then + echo "[WARN] Repository modify was interrupted server-side, retrying" + sleep $((modify_attempt * 15)) + else + echo "::error::Repository modify failed" + exit 1 + fi + done + + echo "[INFO] Publishing repository $REPOSITORY_NAME" + create_publication rpm "$REPOSITORY_NAME" + + echo "::notice::Packages are available at $PULP_CONTENT_URL/${LEGACY_BASE_PATH:-$PULP_DOMAIN/$BASE_PATH}/" +done + +manifest_write "$MODULE_NAME" "${DISTRIB:-}" "rpm" "${STABILITY:-}" "delivery" "$PULP_CONTENT_URL" diff --git a/pulp/package-delivery/ensure-feature-repository.sh b/pulp/package-delivery/ensure-feature-repository.sh new file mode 100755 index 0000000..8d57f0e --- /dev/null +++ b/pulp/package-delivery/ensure-feature-repository.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +set -euo pipefail + +# shellcheck source=pulp/lib/api.sh +source "$(dirname "$0")/../lib/api.sh" + +# an unset org variable is forwarded as an empty string, overriding the default +PULP_URL="${PULP_URL:-https://pulp-api.int.centreon.com}" +PULP_DOMAIN="${PULP_DOMAIN:-default}" + +# feature repositories are provisioned on demand by delivery-tooling create-repos +# (repository_dispatch), which creates every distribution of the major version in +# both the standard and business domains: wait for the ones this delivery targets +MAX_ATTEMPTS=40 +POLL_INTERVAL=15 + +if [[ -z "${CENTREON_TECHNIQUE_PAT:-}" ]]; then + echo "::error::centreon_technique_pat input is required to request the feature repository creation" + exit 1 +fi + +if [[ -z "${VERSION:-}" ]]; then + echo "::error::version input is required for a feature delivery" + exit 1 +fi + +FEATURE_TICKET=$(echo "$GH_HEAD_REF" | grep -oE 'MON-[0-9]+' | head -1 || true) +if [[ -z "$FEATURE_TICKET" ]]; then + echo "::error::Cannot extract the feature ticket from branch name $GH_HEAD_REF" + exit 1 +fi + +# the delivery checks both the repository and the distribution; create-repos +# creates the distribution last (after the initial publication), so its presence +# means the repository is ready to receive packages +feature_repositories_exist() { + local arch + for arch in noarch x86_64; do + pulp_resource_exists "repositories/rpm/rpm" "$REPOSITORY_PREFIX-$arch" || return 1 + pulp_resource_exists "distributions/rpm/rpm" "$REPOSITORY_PREFIX-$arch" || return 1 + done +} + +if feature_repositories_exist; then + echo "[INFO] Feature repositories $REPOSITORY_PREFIX-{noarch,x86_64} already exist in domain $PULP_DOMAIN" + exit 0 +fi + +echo "[INFO] Requesting the creation of the $FEATURE_TICKET feature repositories for $VERSION" +curl --fail-with-body -sS -X POST \ + -H "Authorization: Bearer $CENTREON_TECHNIQUE_PAT" \ + -H "Accept: application/vnd.github.v3+json" \ + https://api.github.com/repos/centreon/delivery-tooling/dispatches \ + -d "$(jq -n \ + --arg major_version "$VERSION" \ + --arg repo_type "feature" \ + --arg feature_ticket "$FEATURE_TICKET" \ + --arg target "pulp" \ + '{ + event_type: "create-feature-repositories", + client_payload: { + major_version: $major_version, + repo_type: $repo_type, + feature_ticket: $feature_ticket, + target: $target + } + }' + )" + +for ((attempt = 1; attempt <= MAX_ATTEMPTS; attempt++)); do + sleep "$POLL_INTERVAL" + echo "[INFO] Checking if the feature repositories have been created... [ATTEMPT $attempt/$MAX_ATTEMPTS]" + if feature_repositories_exist; then + echo "[INFO] Feature repositories $REPOSITORY_PREFIX-{noarch,x86_64} are ready in domain $PULP_DOMAIN" + exit 0 + fi +done + +echo "::error::Timeout waiting for the feature repositories creation. Aborting delivery process. Please contact someone from the DevSecOps team." +exit 1 diff --git a/pulp/promote-to-stable/action.yml b/pulp/promote-to-stable/action.yml new file mode 100644 index 0000000..22f1597 --- /dev/null +++ b/pulp/promote-to-stable/action.yml @@ -0,0 +1,168 @@ +name: "promote testing to stable" +description: "Promote testing packages to stable." +inputs: + module_name: + description: "Module name" + required: true + distrib: + description: "The distribution used for packaging" + required: true + major_version: + description: "Centreon packaged major version (not used by the plugins repository family)" + required: false + default: "" + stability: + description: "The package stability (stable, testing, unstable)" + required: true + release_type: + description: "Type of release (hotfix, release)" + required: true + is_cloud: + description: "Release context (cloud or not cloud)" + required: true + repository_name: + description: "Target repository family: standard, standard-internal, business, business-internal, plugins or connectors" + required: false + default: "standard" + verify: + description: "Verify the promoted packages afterwards (presence, metadata resolution, fetchability)" + required: false + default: "true" + pulp_url: + description: "The pulp server api url" + required: true + pulp_content_url: + description: "The pulp server content url" + required: true + audience: + description: "OIDC audience, must match the pulp server GITHUB_OIDC.audience" + required: true + +runs: + using: "composite" + steps: + - name: Validate inputs + shell: bash + env: + RELEASE_TYPE: ${{ inputs.release_type }} + run: | + set -euo pipefail + + if [[ "$RELEASE_TYPE" != "release" && "$RELEASE_TYPE" != "hotfix" ]]; then + echo "::error::release_type input must be defined (release or hotfix)" + exit 1 + fi + + - name: Compute pulp repository properties + id: pulp-properties + shell: bash + env: + MODULE_NAME: ${{ inputs.module_name }} + DISTRIB: ${{ inputs.distrib }} + VERSION: ${{ inputs.major_version }} + STABILITY: ${{ inputs.stability }} + RELEASE_TYPE: ${{ inputs.release_type }} + IS_CLOUD: ${{ inputs.is_cloud }} + REPOSITORY_TYPE: ${{ inputs.repository_name }} + GH_HEAD_REF: ${{ github.head_ref || github.ref_name }} + run: ${{ github.action_path }}/../repository-properties/properties.sh + + - name: Setup python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Setup pulp-cli + shell: bash + env: + PULP_URL: ${{ inputs.pulp_url }} + AUDIENCE: ${{ inputs.audience }} + DOMAIN: ${{ steps.pulp-properties.outputs.domain }} + run: ${{ github.action_path }}/../setup-cli/setup.sh + + - name: Promote RPM packages to stable + id: promote-rpm + if: steps.pulp-properties.outputs.package_extension == 'rpm' + shell: bash + env: + MODULE_NAME: ${{ inputs.module_name }} + DISTRIB: ${{ inputs.distrib }} + TESTING_REPOSITORY_PREFIX: ${{ steps.pulp-properties.outputs.testing_repository_prefix }} + TESTING_BASE_PATH_PREFIX: ${{ steps.pulp-properties.outputs.testing_base_path_prefix }} + STABLE_REPOSITORY_PREFIX: ${{ steps.pulp-properties.outputs.stable_repository_prefix }} + STABLE_BASE_PATH_PREFIX: ${{ steps.pulp-properties.outputs.stable_base_path_prefix }} + LEGACY_STABLE_BASE_PATH_PREFIX: ${{ steps.pulp-properties.outputs.legacy_stable_base_path_prefix }} + STABILITY: ${{ inputs.stability }} + PULP_URL: ${{ inputs.pulp_url }} + PULP_TOKEN: ${{ env.PULP_TOKEN }} + PULP_CONTENT_URL: ${{ inputs.pulp_content_url }} + # stable shares the domain since the domain merge; kept distinct for the scripts that address the stable tier + PULP_DOMAIN: ${{ steps.pulp-properties.outputs.domain }} + PULP_STABLE_DOMAIN: ${{ steps.pulp-properties.outputs.stable_domain }} + run: ${{ github.action_path }}/promote-rpm.sh + + - name: Promote DEB packages to stable + id: promote-deb + if: steps.pulp-properties.outputs.package_extension == 'deb' + shell: bash + env: + MODULE_NAME: ${{ inputs.module_name }} + DISTRIB: ${{ inputs.distrib }} + # the promotion source is the testing repository (one deb repository per + # stability), not the repository matching the requested stability + REPOSITORY_NAME: ${{ steps.pulp-properties.outputs.testing_repository_name }} + BASE_PATH: ${{ steps.pulp-properties.outputs.testing_base_path }} + LEGACY_TESTING_BASE_PATH: ${{ steps.pulp-properties.outputs.legacy_testing_base_path }} + LEGACY_STABLE_BASE_PATH: ${{ steps.pulp-properties.outputs.legacy_stable_base_path }} + TESTING_POOL_PATH: ${{ steps.pulp-properties.outputs.testing_pool_path }} + TESTING_SUITE: ${{ steps.pulp-properties.outputs.testing_suite }} + STABLE_SUITE: ${{ steps.pulp-properties.outputs.stable_suite }} + STABLE_LEGACY_SUITE: ${{ steps.pulp-properties.outputs.stable_legacy_suite }} + STABLE_LEGACY_REPOSITORY_NAME: ${{ steps.pulp-properties.outputs.stable_legacy_repository_name }} + STABLE_REPOSITORY_NAME: ${{ steps.pulp-properties.outputs.stable_repository_name }} + STABLE_BASE_PATH: ${{ steps.pulp-properties.outputs.stable_base_path }} + PACKAGE_DISTRIB_NAME: ${{ steps.pulp-properties.outputs.package_distrib_name }} + STABILITY: ${{ inputs.stability }} + PULP_URL: ${{ inputs.pulp_url }} + PULP_TOKEN: ${{ env.PULP_TOKEN }} + PULP_CONTENT_URL: ${{ inputs.pulp_content_url }} + # stable shares the domain since the domain merge; kept distinct for the scripts that address the stable tier + PULP_DOMAIN: ${{ steps.pulp-properties.outputs.domain }} + PULP_STABLE_DOMAIN: ${{ steps.pulp-properties.outputs.stable_domain }} + run: ${{ github.action_path }}/promote-deb.sh + + - name: Verify RPM promotion + # skipped on dry runs (stability != stable): the promote script exits + # before promoting anything, so there is no manifest to verify + if: inputs.verify == 'true' && inputs.stability == 'stable' && steps.pulp-properties.outputs.package_extension == 'rpm' + shell: bash + env: + MANIFEST: ${{ steps.promote-rpm.outputs.manifest }} + CHECK_MODE: promote + MODULE_NAME: ${{ inputs.module_name }} + DISTRIB: ${{ inputs.distrib }} + STABILITY: ${{ inputs.stability }} + PULP_URL: ${{ inputs.pulp_url }} + PULP_TOKEN: ${{ env.PULP_TOKEN }} + PULP_CONTENT_URL: ${{ inputs.pulp_content_url }} + # verifying the just-promoted content, which now lives in the stable domain + PULP_DOMAIN: ${{ steps.pulp-properties.outputs.stable_domain }} + run: ${{ github.action_path }}/../lib/check-rpm.sh + + - name: Verify DEB promotion + # skipped on dry runs (stability != stable): the promote script exits + # before promoting anything, so there is no manifest to verify + if: inputs.verify == 'true' && inputs.stability == 'stable' && steps.pulp-properties.outputs.package_extension == 'deb' + shell: bash + env: + MANIFEST: ${{ steps.promote-deb.outputs.manifest }} + CHECK_MODE: promote + MODULE_NAME: ${{ inputs.module_name }} + DISTRIB: ${{ inputs.distrib }} + STABILITY: ${{ inputs.stability }} + PULP_URL: ${{ inputs.pulp_url }} + PULP_TOKEN: ${{ env.PULP_TOKEN }} + PULP_CONTENT_URL: ${{ inputs.pulp_content_url }} + # verifying the just-promoted content, which now lives in the stable domain + PULP_DOMAIN: ${{ steps.pulp-properties.outputs.stable_domain }} + run: ${{ github.action_path }}/../lib/check-deb.sh diff --git a/pulp/promote-to-stable/promote-deb.sh b/pulp/promote-to-stable/promote-deb.sh new file mode 100755 index 0000000..522c391 --- /dev/null +++ b/pulp/promote-to-stable/promote-deb.sh @@ -0,0 +1,488 @@ +#!/usr/bin/env bash +set -euo pipefail + +# shellcheck source=pulp/lib/manifest.sh +source "$(dirname "$0")/../lib/manifest.sh" +# shellcheck source=pulp/lib/api.sh +source "$(dirname "$0")/../lib/api.sh" + +# an unset org variable is forwarded as an empty string, overriding the default +PULP_URL="${PULP_URL:-https://pulp-api.int.centreon.com}" +PULP_CONTENT_URL="${PULP_CONTENT_URL:-https://packages.int.centreon.com}" +# stable shares its Domain with testing since the domain merge (PULP_STABLE_DOMAIN +# now equals PULP_DOMAIN); the read/write phase switch is kept as a no-op +PULP_DOMAIN="${PULP_DOMAIN:-default}" +PULP_STABLE_DOMAIN="${PULP_STABLE_DOMAIN:-default}" +# switch_pulp_domain overwrites PULP_DOMAIN itself once the write phase +# starts; download_testing_package always needs the original (testing-tier) +# domain, including after the switch, so keep our own copy +TESTING_DOMAIN="$PULP_DOMAIN" + +if ! pulp_resource_exists "repositories/deb/apt" "$REPOSITORY_NAME"; then + echo "::error::Nothing to promote, repository $REPOSITORY_NAME does not exist" + exit 1 +fi + +VERSION_HREF=$(pulp deb repository show --name "$REPOSITORY_NAME" | jq -r '.latest_version_href') + +# fetch a published index into a file: 0=ok, 2=absent (404), 1=error +fetch_index() { + local url=$1 out=$2 code + code=$(content_curl -sSL --retry 3 --retry-delay 5 -o "$out" -w '%{http_code}' "$url") || return 1 + [[ "$code" == "200" ]] && return 0 + [[ "$code" == "404" ]] && return 2 + return 1 +} + +# collect the sha256 set of every package published in a suite: the served +# dists/ indexes are the source of truth for suite membership (relative_path +# and the release_components api can't be trusted/used here, see deliver-deb.sh). +# A missing Release (404) is an empty suite; any other fetch failure aborts. +suite_sha_file() { + local base_path=$1 suite=$2 out release_file arches arch pkg_file rc + out=$(mktemp) + release_file=$(mktemp) + fetch_index "$PULP_CONTENT_URL/$base_path/dists/$suite/Release" "$release_file" && rc=0 || rc=$? + if [[ $rc -eq 2 ]]; then + echo "$out" + return 0 + elif [[ $rc -ne 0 ]]; then + echo "::error::Cannot fetch the $suite Release index; refusing to promote from a partial view." >&2 + return 1 + fi + arches=$(awk -F': ' '/^Architectures:/ {print $2}' "$release_file") + for arch in $arches; do + pkg_file=$(mktemp) + if ! fetch_index "$PULP_CONTENT_URL/$base_path/dists/$suite/main/binary-$arch/Packages" "$pkg_file"; then + echo "::error::Cannot fetch the $suite binary-$arch Packages index; refusing to promote from a partial view." >&2 + return 1 + fi + awk -F': ' '/^SHA256:/ {print $2}' "$pkg_file" >> "$out" + rm -f "$pkg_file" + done + rm -f "$release_file" + sort -u "$out" -o "$out" + echo "$out" +} + +# resolve the served filename from the testing suite's Packages file by +# sha256 (the published pool layout doesn't match the upload relative_path) +download_testing_package() { + local sha256=$1 arch=$2 dest=$3 filename packages_file + # read the index from a file: awk exits on the first match, which would + # SIGPIPE a piped curl on a large index (curl 23, fatal under pipefail) + packages_file=$(mktemp) + content_curl -fsSL --retry 3 --retry-delay 5 -o "$packages_file" "$PULP_CONTENT_URL/${LEGACY_TESTING_BASE_PATH:-$TESTING_DOMAIN/$BASE_PATH}/dists/$TESTING_SUITE/main/binary-$arch/Packages" + filename=$(awk -v sha="$sha256" 'BEGIN { RS = ""; FS = "\n" } index($0, "SHA256: " sha) { for (i = 1; i <= NF; i++) if ($i ~ /^Filename: /) { sub(/^Filename: /, "", $i); print $i; exit } }' "$packages_file") + rm -f "$packages_file" + if [[ -z "$filename" ]]; then + echo "::error::Cannot locate the published file for sha256 $sha256 in $TESTING_SUITE ($arch)" >&2 + return 1 + fi + content_curl -fsSL --retry 3 --retry-delay 5 -o "$dest" "$PULP_CONTENT_URL/${LEGACY_TESTING_BASE_PATH:-$TESTING_DOMAIN/$BASE_PATH}/$filename" +} + +TESTING_SHAS_FILE=$(suite_sha_file "${LEGACY_TESTING_BASE_PATH:-$TESTING_DOMAIN/$BASE_PATH}" "$TESTING_SUITE") + +# paginated: the repository keeps every delivered version across all suites, +# so the module listing can exceed a single page +RESULTS_FILE=$(mktemp) +url="$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&limit=1000' \ + "$(jq -rn --arg v "$VERSION_HREF" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" +)" +while [[ -n "$url" ]]; do + refresh_pulp_token + page=$(curl -fsSL -H "Authorization: Bearer $PULP_TOKEN" "$url") + echo "$page" | jq -c '.results[]' >> "$RESULTS_FILE" + url=$(echo "$page" | jq -r '.next // empty') +done + +# only the LATEST version of each package is promoted (deb has no retention +# mechanism, testing accumulates every build); compare segment by segment +# (numeric as numbers) since a plain string max ranks "0.9" above "0.10" +TESTING_SET_FILE=$(mktemp) +jq -R . "$TESTING_SHAS_FILE" | jq -s 'map({key: ., value: true}) | from_entries' > "$TESTING_SET_FILE" +PACKAGES=$( + jq -s --slurpfile testing_set "$TESTING_SET_FILE" \ + 'def vkey: [scan("[0-9]+|[^0-9]+") | (tonumber? // .)]; + [.[] | select($testing_set[0][.sha256])] + | group_by(.package, .architecture) | map(max_by(.version | vkey))' \ + "$RESULTS_FILE" +) +rm -f "$RESULTS_FILE" "$TESTING_SHAS_FILE" "$TESTING_SET_FILE" +PACKAGES_COUNT=$(echo "$PACKAGES" | jq 'length') + +if [[ "$PACKAGES_COUNT" -eq 0 ]]; then + echo "::error::Nothing to promote, no package of module $MODULE_NAME found in the $TESTING_SUITE suite of $REPOSITORY_NAME" + exit 1 +fi + +echo "[INFO] $PACKAGES_COUNT packages of module $MODULE_NAME found in the $TESTING_SUITE suite of $REPOSITORY_NAME" + +if [[ "$STABILITY" != "stable" ]]; then + echo "[INFO] Dry run, $PACKAGES_COUNT packages would be promoted to $STABLE_SUITE/main" + exit 0 +fi + +# everything from here on targets the stable-tier domain +refresh_pulp_token +switch_pulp_domain "$PULP_STABLE_DOMAIN" + +if ! pulp_resource_exists "repositories/deb/apt" "$STABLE_REPOSITORY_NAME"; then + echo "::error::Stable repository $STABLE_REPOSITORY_NAME does not exist. Pulp repositories are provisioned centrally by delivery-tooling create-repos; run create-repos before promoting." + exit 1 +fi +STABLE_REPOSITORY_HREF=$(pulp deb repository show --name "$STABLE_REPOSITORY_NAME" | jq -r '.pulp_href') + +# 24.x standard clients use the dedicated legacy repository form +# (apt-standard-24.10-stable/ with a plain-codename suite, provisioned by +# create-repos): mirror every candidate package association into that suite so +# both suites always list the same stable content. Idempotent (rerun safe). +ensure_legacy_suite_associations() { + [[ -z "${STABLE_LEGACY_REPOSITORY_NAME:-}" ]] && return 0 + refresh_pulp_token + if ! pulp_resource_exists "repositories/deb/apt" "$STABLE_LEGACY_REPOSITORY_NAME"; then + echo "::error::Dedicated legacy repository $STABLE_LEGACY_REPOSITORY_NAME does not exist. Pulp repositories are provisioned centrally by delivery-tooling create-repos; run create-repos before promoting." + return 1 + fi + local legacy_repo_href latest legacy_rc + legacy_repo_href=$(pulp deb repository show --name "$STABLE_LEGACY_REPOSITORY_NAME" | jq -r '.pulp_href') + latest=$(pulp deb repository show --name "$STABLE_LEGACY_REPOSITORY_NAME" | jq -r '.latest_version_href') + legacy_rc=$( + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Bearer $PULP_TOKEN" \ + "$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/release_components/?$( + printf 'repository_version=%s&distribution=%s&component=main&limit=1' \ + "$(jq -rn --arg v "$latest" '$v | @uri')" "$(jq -rn --arg v "$STABLE_LEGACY_SUITE" '$v | @uri')" + )" | jq -r '.results[0].pulp_href // empty' + ) + if [[ -z "$legacy_rc" ]]; then + echo "::error::Cannot resolve the $STABLE_LEGACY_SUITE/main release component of $STABLE_LEGACY_REPOSITORY_NAME (provisioned by create-repos)" + return 1 + fi + echo "[INFO] Mirroring $PACKAGES_COUNT package association(s) into $STABLE_LEGACY_REPOSITORY_NAME $STABLE_LEGACY_SUITE/main" + local units_file body_file sha href out code body prc + units_file=$(mktemp) + while read -r sha; do + refresh_pulp_token + href=$(lookup_deb_content "packages" "--data-urlencode sha256=$sha") + if [[ -z "$href" ]]; then + echo "::error::Cannot resolve the promoted package (sha256 $sha) for the $STABLE_LEGACY_SUITE mirror" + return 1 + fi + out=$(post_json "$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/package_release_components/" \ + "{\"package\": \"$href\", \"release_component\": \"$legacy_rc\"}") || out=$'\n000' + code="${out##*$'\n'}" + body="${out%$'\n'*}" + prc="" + if [[ "$code" == 2* ]]; then + prc=$(echo "$body" | jq -r '.pulp_href // empty' 2>/dev/null) || prc="" + fi + if [[ -z "$prc" ]]; then + # the api answers 500 on a duplicate synchronous create; expected on a rerun + prc=$(lookup_deb_content "package_release_components" \ + "--data-urlencode package=$href --data-urlencode release_component=$legacy_rc") + fi + if [[ -z "$prc" ]]; then + echo "::error::Legacy suite association failed for sha256 $sha (HTTP $code)" + return 1 + fi + printf '%s\n%s\n' "$href" "$prc" >> "$units_file" + done < <(echo "$PACKAGES" | jq -r '.[].sha256') + body_file=$(mktemp) + jq -R . "$units_file" | jq -cs '{add_content_units: ([.[] | select(. != "")] | unique)}' > "$body_file" + rm -f "$units_file" + local task rc + for _ in 1 2 3; do + task=$(start_modify_task "$PULP_URL${legacy_repo_href}modify/" "$body_file") + wait_task_race "$task" && rc=0 || rc=$? + if [[ $rc -eq 0 ]]; then + rm -f "$body_file" + create_publication deb "$STABLE_LEGACY_REPOSITORY_NAME" --structured + return 0 + fi + done + echo "::error::Cannot add the $STABLE_LEGACY_SUITE mirror associations to $STABLE_LEGACY_REPOSITORY_NAME" + return 1 +} + +# "already promoted" is decided against the stable SUITE: the stable repository +# is shared by every major version, so a package another version's stable suite +# publishes with the same bytes is in the repository without being in this suite +STABLE_SHAS_FILE=$(suite_sha_file "${LEGACY_STABLE_BASE_PATH:-$PULP_STABLE_DOMAIN/$STABLE_BASE_PATH}" "$STABLE_SUITE") || exit 1 + +mkdir -p promoted-packages + +# mirrors deliver-deb.sh's own PULP_LABELS, but this run's own git context +PULP_LABELS=$(jq -cn \ + --arg mod "$MODULE_NAME" \ + --arg git_commit "${GITHUB_SHA:-}" \ + --arg git_ref "${GITHUB_REF:-}" \ + --arg run_id "${GITHUB_RUN_ID:-}" \ + --arg actor "${GITHUB_ACTOR:-}" \ + --arg workflow "${GITHUB_WORKFLOW:-}" \ + '{"module": $mod, "git_commit": $git_commit, "git_ref": $git_ref, "github_run_id": $run_id, "github_actor": $actor, "github_workflow": $workflow}') + +# emit the href of a matching stable-domain deb content unit, empty if absent. +# Retried: the api has been observed answering an empty page for content +# committed seconds earlier (stale read) +lookup_deb_content() { + local endpoint=$1 query=$2 out attempt + for attempt in 1 2 3 4 5; do + out=$(curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Bearer $PULP_TOKEN" -G \ + --data-urlencode "limit=1" \ + $query \ + "$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/$endpoint/" | jq -r '.results[0].pulp_href // empty') || out="" + if [[ -n "$out" ]]; then + echo "$out" + return 0 + fi + sleep $((attempt * 3)) + refresh_pulp_token + done +} + +# emit the release-component hrefs a package is associated with +lookup_prcs() { + local package_href=$1 + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Bearer $PULP_TOKEN" -G \ + --data-urlencode "package=$package_href" \ + --data-urlencode "limit=100" \ + "$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/package_release_components/" \ + | jq -r '.results[].release_component' +} + +# candidates already published in the stable suite need no new association; +# when EVERYTHING already reached stable (a rerun after the modify), only the +# publication may be missing: republish and stop. +UNPROMOTED_HREFS=() +while read -r PACKAGE; do + sha=$(echo "$PACKAGE" | jq -r '.sha256') + if ! grep -qxF "$sha" "$STABLE_SHAS_FILE"; then + UNPROMOTED_HREFS+=("$(echo "$PACKAGE" | jq -r '.pulp_href')") + fi +done < <(echo "$PACKAGES" | jq -c '.[]') +rm -f "$STABLE_SHAS_FILE" +if ((${#UNPROMOTED_HREFS[@]} == 0)); then + echo "[INFO] All $PACKAGES_COUNT package(s) are already promoted to $STABLE_SUITE; republishing only" + while read -r PACKAGE; do + manifest_add "$(echo "$PACKAGE" | jq -c \ + --arg repository "$STABLE_REPOSITORY_NAME" --arg base_path "${LEGACY_STABLE_BASE_PATH:-$PULP_STABLE_DOMAIN/$STABLE_BASE_PATH}" \ + --arg suite "$STABLE_SUITE" \ + '{filename: (.relative_path | sub(".*/"; "")), name: .package, version, arch: .architecture, sha256, repository: $repository, base_path: $base_path, suite: $suite, relative_path}')" + done < <(echo "$PACKAGES" | jq -c '.[]') + ensure_legacy_suite_associations + create_publication deb "$STABLE_REPOSITORY_NAME" --structured + echo "::notice::Packages are available with: deb $PULP_CONTENT_URL/${LEGACY_STABLE_BASE_PATH:-$PULP_STABLE_DOMAIN/$STABLE_BASE_PATH}/ $STABLE_SUITE main" + manifest_write "$MODULE_NAME" "${DISTRIB:-}" "deb" "$STABILITY" "promote" "$PULP_CONTENT_URL" + exit 0 +fi + +# batched promote, mirroring the batched delivery. Testing and stable share +# their domain (and content) since the domain merge: the batch below associates +# the testing package hrefs directly, no re-download/re-upload. Only the FIRST +# package of each arch goes through the legacy upload path, to establish the +# release component and deduce its href (see deliver-deb.sh; listing release +# components requires admin rights). +declare -A ARCH_SEEN=() +LEGACY_PACKAGES=() +BATCH_PACKAGES=() +while read -r PACKAGE; do + ARCH=$(echo "$PACKAGE" | jq -r '.architecture') + if [[ -z "${ARCH_SEEN[$ARCH]+set}" ]]; then + ARCH_SEEN[$ARCH]=1 + LEGACY_PACKAGES+=("$PACKAGE") + else + BATCH_PACKAGES+=("$PACKAGE") + fi +done < <(echo "$PACKAGES" | jq -c '.[]') + +# capture the reference package's pre-upload stable associations (if it +# already exists there, e.g. a rerun) so the new one stands out as the diff +refresh_pulp_token +LEGACY_REF_SHA256=$(echo "${LEGACY_PACKAGES[0]}" | jq -r '.sha256') +LEGACY_REF_BEFORE_HREF=$(lookup_deb_content "packages" "--data-urlencode sha256=$LEGACY_REF_SHA256") +LEGACY_REF_BEFORE="" +[[ -n "$LEGACY_REF_BEFORE_HREF" ]] && LEGACY_REF_BEFORE=$(lookup_prcs "$LEGACY_REF_BEFORE_HREF" | sort) + +for PACKAGE in "${LEGACY_PACKAGES[@]}"; do + RELATIVE_PATH=$(echo "$PACKAGE" | jq -r '.relative_path') + SHA256=$(echo "$PACKAGE" | jq -r '.sha256') + ARCH=$(echo "$PACKAGE" | jq -r '.architecture') + FILE_NAME=$(basename "$RELATIVE_PATH") + FILE="promoted-packages/$FILE_NAME" + + echo "[INFO] Downloading $FILE_NAME from $TESTING_SUITE" + download_testing_package "$SHA256" "$ARCH" "$FILE" + + # same relative_path into stable: pulp reuses existing content and adds the + # stable suite association, creating the ReleaseComponent/ReleaseArchitecture + # retried: the legacy path creates a repository version and can lose the race + for legacy_attempt in 1 2 3; do + echo "[INFO] Promoting $FILE_NAME to $STABLE_SUITE/main [legacy path, attempt $legacy_attempt]" + TASK_HREF=$( + pulp_upload \ + -F "file=@\"$FILE\"" \ + -F "relative_path=$RELATIVE_PATH" \ + -F "distribution=$STABLE_SUITE" \ + -F "component=main" \ + -F "repository=$STABLE_REPOSITORY_HREF" \ + -F "pulp_labels=$PULP_LABELS" \ + "$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/packages/" + ) + wait_task_race "$TASK_HREF" && rc=0 || rc=$? + if [[ $rc -eq 0 ]]; then + break + elif [[ $rc -eq 2 && $legacy_attempt -lt 3 ]]; then + echo "[WARN] Legacy promotion of $FILE_NAME lost the repository-version race, retrying" + sleep $((legacy_attempt * 15)) + else + echo "::error::Legacy promotion of $FILE_NAME failed" + exit 1 + fi + done +done + +if ((${#BATCH_PACKAGES[@]} > 0)); then + # the release component is the association the legacy re-upload just added + # to the reference package (diffed against its pre-upload set captured above) + refresh_pulp_token + LEGACY_REF_STABLE_HREF=$(lookup_deb_content "packages" "--data-urlencode sha256=$LEGACY_REF_SHA256") + if [[ -z "$LEGACY_REF_STABLE_HREF" ]]; then + echo "::error::Cannot resolve the promoted reference package (sha256 $LEGACY_REF_SHA256) in the stable domain" + exit 1 + fi + LEGACY_REF_AFTER=$(lookup_prcs "$LEGACY_REF_STABLE_HREF" | sort) + STABLE_RC_SET=$(comm -13 <(echo "$LEGACY_REF_BEFORE") <(echo "$LEGACY_REF_AFTER") | grep . || true) + if [[ $(echo "$STABLE_RC_SET" | grep -c .) -ne 1 ]]; then + refresh_pulp_token + STABLE_LATEST=$(pulp deb repository show --name "$STABLE_REPOSITORY_NAME" | jq -r '.latest_version_href') + STABLE_RC_SET=$( + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Bearer $PULP_TOKEN" \ + "$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/release_components/?$( + printf 'repository_version=%s&distribution=%s&component=main&limit=1' \ + "$(jq -rn --arg v "$STABLE_LATEST" '$v | @uri')" "$(jq -rn --arg v "$STABLE_SUITE" '$v | @uri')" + )" | jq -r '.results[0].pulp_href // empty' + ) + fi + STABLE_RC=$(echo "$STABLE_RC_SET" | grep . | head -1) + if [[ -z "$STABLE_RC" ]]; then + # a promotion interrupted after its repository modify only misses the + # publication: republish and re-check every candidate before giving up + echo "[WARN] Cannot deduce the $STABLE_SUITE/main release component (got: ${STABLE_RC_SET:-none}); republishing to check for an interrupted promotion" + ensure_legacy_suite_associations + create_publication deb "$STABLE_REPOSITORY_NAME" --structured + RECHECK_SHAS_FILE=$(suite_sha_file "${LEGACY_STABLE_BASE_PATH:-$PULP_STABLE_DOMAIN/$STABLE_BASE_PATH}" "$STABLE_SUITE") + MISSING_COUNT=0 + while read -r sha; do + grep -qxF "$sha" "$RECHECK_SHAS_FILE" || MISSING_COUNT=$((MISSING_COUNT + 1)) + done < <(echo "$PACKAGES" | jq -r '.[].sha256') + rm -f "$RECHECK_SHAS_FILE" + if ((MISSING_COUNT == 0)); then + echo "[INFO] Every candidate package is published in $STABLE_SUITE; the interrupted promotion only missed the publication" + while read -r PACKAGE; do + manifest_add "$(echo "$PACKAGE" | jq -c \ + --arg repository "$STABLE_REPOSITORY_NAME" --arg base_path "${LEGACY_STABLE_BASE_PATH:-$PULP_STABLE_DOMAIN/$STABLE_BASE_PATH}" \ + --arg suite "$STABLE_SUITE" \ + '{filename: (.relative_path | sub(".*/"; "")), name: .package, version, arch: .architecture, sha256, repository: $repository, base_path: $base_path, suite: $suite, relative_path}')" + done < <(echo "$PACKAGES" | jq -c '.[]') + echo "::notice::Packages are available with: deb $PULP_CONTENT_URL/${LEGACY_STABLE_BASE_PATH:-$PULP_STABLE_DOMAIN/$STABLE_BASE_PATH}/ $STABLE_SUITE main" + manifest_write "$MODULE_NAME" "${DISTRIB:-}" "deb" "$STABILITY" "promote" "$PULP_CONTENT_URL" + exit 0 + fi + echo "::error::Cannot deduce the $STABLE_SUITE/main release component (got: ${STABLE_RC_SET:-none}) and $MISSING_COUNT candidate package(s) are missing from the published $STABLE_SUITE suite. A previous promotion likely left the suite associations without the repository content; deliver a rebuilt (fresh) build or add the packages to the stable repository manually." + exit 1 + fi + echo "[INFO] Release component of $STABLE_SUITE/main: $STABLE_RC" + + # remaining packages: the testing content hrefs are associated with the + # stable release component directly (same domain, shared content); the + # packages keep their delivery-time labels + PACKAGE_HREFS=() + for PACKAGE in "${BATCH_PACKAGES[@]}"; do + PACKAGE_HREFS+=("$(echo "$PACKAGE" | jq -r '.pulp_href')") + done + + # package_release_components create is synchronous (201 with the unit, no + # task); a failed create (already existing on a job re-run) falls back to a lookup + echo "[INFO] Associating ${#PACKAGE_HREFS[@]} package(s) with $STABLE_SUITE/main" + MAX_PARALLEL=8 + PRC_DIR=$(mktemp -d) + for i in "${!PACKAGE_HREFS[@]}"; do + refresh_pulp_token + ( + refresh_pulp_token + package_href="${PACKAGE_HREFS[$i]}" + out=$(post_json "$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/package_release_components/" \ + "{\"package\": \"$package_href\", \"release_component\": \"$STABLE_RC\"}") || out=$'\n000' + code="${out##*$'\n'}" + body="${out%$'\n'*}" + href="" + if [[ "$code" == 2* ]]; then + # tolerate a non-json body (gateway error page behind a 2xx) + href=$(echo "$body" | jq -r '.pulp_href // empty' 2>/dev/null) || href="" + fi + if [[ -z "$href" ]]; then + # api answers 500 on a duplicate synchronous create; expected on a rerun + echo "[INFO] $(echo "${BATCH_PACKAGES[$i]}" | jq -r '.package'): stable association create answered HTTP $code, falling back to the lookup (expected on a rerun)" + href=$(lookup_deb_content "package_release_components" \ + "--data-urlencode package=$package_href --data-urlencode release_component=$STABLE_RC") + fi + printf '%s\n%s' "$package_href" "$href" > "$PRC_DIR/$i.pair" + ) & + while (($(jobs -rp | wc -l) >= MAX_PARALLEL)); do + wait -n || true + done + done + wait || true + + ADD_UNITS_FILE=$(mktemp) + for i in "${!BATCH_PACKAGES[@]}"; do + if [[ ! -s "$PRC_DIR/$i.pair" ]] || [[ -z "$(sed -n '2p' "$PRC_DIR/$i.pair")" ]]; then + echo "::error::Stable suite association failed for package $(echo "${BATCH_PACKAGES[$i]}" | jq -r '.package') (see the worker error above)" + exit 1 + fi + cat "$PRC_DIR/$i.pair" >> "$ADD_UNITS_FILE" + echo >> "$ADD_UNITS_FILE" + done + rm -rf "$PRC_DIR" + + echo "[INFO] Adding ${#BATCH_PACKAGES[@]} package(s) and their suite associations to $STABLE_REPOSITORY_NAME in a single task" + refresh_pulp_token + # body through a file: thousands of hrefs exceed the argv limit + ADD_BODY_FILE=$(mktemp) + jq -R . "$ADD_UNITS_FILE" | jq -cs '{add_content_units: [.[] | select(. != "")]}' > "$ADD_BODY_FILE" + rm -f "$ADD_UNITS_FILE" + # retried like the legacy uploads: same repository-version race + for modify_attempt in 1 2 3; do + MODIFY_TASK=$(start_modify_task "$PULP_URL${STABLE_REPOSITORY_HREF}modify/" "$ADD_BODY_FILE") + wait_task_race "$MODIFY_TASK" && rc=0 || rc=$? + if [[ $rc -eq 0 ]]; then + break + elif [[ $rc -eq 2 && $modify_attempt -lt 3 ]]; then + echo "[WARN] Stable repository modify lost the repository-version race, retrying" + sleep $((modify_attempt * 15)) + else + echo "::error::Stable repository modify failed" + exit 1 + fi + done +fi + +# record the promoted packages (with their stable suite coordinates) so the +# verification step verifies exactly this set against the stable suite +while read -r PACKAGE; do + manifest_add "$(echo "$PACKAGE" | jq -c \ + --arg repository "$STABLE_REPOSITORY_NAME" --arg base_path "${LEGACY_STABLE_BASE_PATH:-$PULP_STABLE_DOMAIN/$STABLE_BASE_PATH}" \ + --arg suite "$STABLE_SUITE" \ + '{filename: (.relative_path | sub(".*/"; "")), name: .package, version, arch: .architecture, sha256, repository: $repository, base_path: $base_path, suite: $suite, relative_path}')" +done < <(echo "$PACKAGES" | jq -c '.[]') + +ensure_legacy_suite_associations +echo "[INFO] Publishing repository $STABLE_REPOSITORY_NAME" +create_publication deb "$STABLE_REPOSITORY_NAME" --structured + +echo "::notice::Packages are available with: deb $PULP_CONTENT_URL/${LEGACY_STABLE_BASE_PATH:-$PULP_STABLE_DOMAIN/$STABLE_BASE_PATH}/ $STABLE_SUITE main" + +manifest_write "$MODULE_NAME" "${DISTRIB:-}" "deb" "$STABILITY" "promote" "$PULP_CONTENT_URL" diff --git a/pulp/promote-to-stable/promote-rpm.sh b/pulp/promote-to-stable/promote-rpm.sh new file mode 100755 index 0000000..a88c6eb --- /dev/null +++ b/pulp/promote-to-stable/promote-rpm.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash +set -euo pipefail + +# shellcheck source=pulp/lib/manifest.sh +source "$(dirname "$0")/../lib/manifest.sh" +# shellcheck source=pulp/lib/api.sh +source "$(dirname "$0")/../lib/api.sh" + +# an unset org variable is forwarded as an empty string, overriding the default +PULP_URL="${PULP_URL:-https://pulp-api.int.centreon.com}" +PULP_CONTENT_URL="${PULP_CONTENT_URL:-https://packages.int.centreon.com}" +# stable shares its Domain with testing since the domain merge (PULP_STABLE_DOMAIN +# now equals PULP_DOMAIN); the read/write phase switch is kept as a no-op +PULP_DOMAIN="${PULP_DOMAIN:-default}" +PULP_STABLE_DOMAIN="${PULP_STABLE_DOMAIN:-default}" +declare -A ARCH_CONTENT +declare -A ARCH_RESULTS +TOTAL_PACKAGES_COUNT=0 + +for ARCH in noarch x86_64; do + TESTING_REPOSITORY_NAME="$TESTING_REPOSITORY_PREFIX-$ARCH" + + if ! pulp_resource_exists "repositories/rpm/rpm" "$TESTING_REPOSITORY_NAME"; then + echo "[INFO] Testing repository $TESTING_REPOSITORY_NAME does not exist" + continue + fi + + VERSION_HREF=$(pulp rpm repository show --name "$TESTING_REPOSITORY_NAME" | jq -r '.latest_version_href') + + # paginate: the testing repository keeps every delivered version, so the + # module listing can exceed a single page + RESULTS_FILE=$(mktemp) + url="$PULP_URL/$PULP_DOMAIN/api/v3/content/rpm/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&limit=1000' \ + "$(jq -rn --arg v "$VERSION_HREF" '$v | @uri')" \ + "$(jq -rn --arg v "module=$MODULE_NAME" '$v | @uri')" + )" + while [[ -n "$url" ]]; do + refresh_pulp_token + page=$(curl -fsSL -H "Authorization: Bearer $PULP_TOKEN" "$url") + echo "$page" | jq -c '.results[]' >> "$RESULTS_FILE" + url=$(echo "$page" | jq -r '.next // empty') + done + + # only the LATEST build of each package is promoted; compare version/release + # segment by segment (numeric as numbers) since a plain string max ranks "0.9" above "0.10" + RESULTS=$(jq -s 'def vkey: [scan("[0-9]+|[^0-9]+") | (tonumber? // .)]; + [.[] | {pulp_href, name, version, release, arch, location_href, sha256}] + | group_by(.name, .arch) | map(max_by([(.version | vkey), (.release | vkey)]))' "$RESULTS_FILE") + rm -f "$RESULTS_FILE" + CONTENT=$(echo "$RESULTS" | jq '[.[].pulp_href]') + ARCH_PACKAGES_COUNT=$(echo "$CONTENT" | jq 'length') + + echo "[INFO] $ARCH_PACKAGES_COUNT $ARCH packages of module $MODULE_NAME found in $TESTING_REPOSITORY_NAME" + ARCH_CONTENT[$ARCH]="$CONTENT" + ARCH_RESULTS[$ARCH]="$RESULTS" + TOTAL_PACKAGES_COUNT=$((TOTAL_PACKAGES_COUNT + ARCH_PACKAGES_COUNT)) +done + +if [[ "$TOTAL_PACKAGES_COUNT" -eq 0 ]]; then + echo "::error::Nothing to promote, no package of module $MODULE_NAME found in $TESTING_REPOSITORY_PREFIX repositories" + exit 1 +fi + +if [[ "$STABILITY" != "stable" ]]; then + echo "[INFO] Dry run, $TOTAL_PACKAGES_COUNT packages would be promoted to $STABLE_REPOSITORY_PREFIX repositories" + exit 0 +fi + +# everything from here on targets the stable-tier domain +refresh_pulp_token +switch_pulp_domain "$PULP_STABLE_DOMAIN" + +for ARCH in noarch x86_64; do + CONTENT="${ARCH_CONTENT[$ARCH]:-[]}" + RESULTS="${ARCH_RESULTS[$ARCH]:-[]}" + ARCH_PACKAGES_COUNT=$(echo "$CONTENT" | jq 'length') + + if [[ "$ARCH_PACKAGES_COUNT" -eq 0 ]]; then + echo "[INFO] No $ARCH package to promote" + continue + fi + + STABLE_REPOSITORY_NAME="$STABLE_REPOSITORY_PREFIX-$ARCH" + STABLE_BASE_PATH="$STABLE_BASE_PATH_PREFIX/$ARCH" + LEGACY_STABLE_BASE_PATH="" + [[ -n "$LEGACY_STABLE_BASE_PATH_PREFIX" ]] && LEGACY_STABLE_BASE_PATH="$LEGACY_STABLE_BASE_PATH_PREFIX/$ARCH" + + if ! pulp_resource_exists "repositories/rpm/rpm" "$STABLE_REPOSITORY_NAME"; then + echo "::error::stable rpm repository $STABLE_REPOSITORY_NAME does not exist. Pulp repositories and distributions are provisioned centrally by delivery-tooling create-repos; run create-repos for this version before promoting." + exit 1 + fi + + if ! pulp_resource_exists "distributions/rpm/rpm" "$STABLE_REPOSITORY_NAME"; then + echo "::error::stable rpm distribution $STABLE_REPOSITORY_NAME does not exist. Pulp distributions are provisioned centrally by delivery-tooling create-repos; run create-repos for this version before promoting. Refusing to create it here to avoid an unguarded distribution." + exit 1 + fi + + STABLE_REPOSITORY_HREF=$(pulp rpm repository show --name "$STABLE_REPOSITORY_NAME" | jq -r '.pulp_href') + + # Same-domain promotion: testing and stable share their domain, so the + # testing content hrefs (already collected above) are associated with the + # stable repository directly — no re-download/re-upload, the packages keep + # their delivery-time labels (check-rpm.sh only needs "module"). + echo "[INFO] Promoting $ARCH_PACKAGES_COUNT packages to $STABLE_REPOSITORY_NAME" + # pulp-cli repository content modify does not resolve content by pulp_href, use the api directly + ADD_BODY_FILE=$(mktemp) + echo "$CONTENT" | jq -c '{add_content_units: .}' > "$ADD_BODY_FILE" + # retried like the deb path: concurrent promotions can race on the repository version + for modify_attempt in 1 2 3; do + TASK_HREF=$(start_modify_task "$PULP_URL${STABLE_REPOSITORY_HREF}modify/" "$ADD_BODY_FILE") + wait_task_race "$TASK_HREF" && rc=0 || rc=$? + if [[ $rc -eq 0 ]]; then + break + elif [[ $rc -eq 2 && $modify_attempt -lt 3 ]]; then + echo "[WARN] Stable repository modify was interrupted server-side, retrying" + sleep $((modify_attempt * 15)) + else + echo "::error::Stable repository modify failed" + exit 1 + fi + done + + echo "[INFO] Publishing repository $STABLE_REPOSITORY_NAME" + create_publication rpm "$STABLE_REPOSITORY_NAME" + + # record the promoted packages (with their stable target coordinates) so the + # verification step verifies exactly this set against the stable repo + while read -r PKG; do + manifest_add "$(echo "$PKG" | jq -c \ + --arg repository "$STABLE_REPOSITORY_NAME" --arg base_path "${LEGACY_STABLE_BASE_PATH:-$PULP_STABLE_DOMAIN/$STABLE_BASE_PATH}" \ + '{filename: (.location_href | sub(".*/"; "")), name, version, release, arch, sha256, repository: $repository, base_path: $base_path}')" + done < <(echo "${ARCH_RESULTS[$ARCH]}" | jq -c '.[]') + + echo "::notice::Packages are available at $PULP_CONTENT_URL/${LEGACY_STABLE_BASE_PATH:-$PULP_STABLE_DOMAIN/$STABLE_BASE_PATH}/" +done + +manifest_write "$MODULE_NAME" "${DISTRIB:-}" "rpm" "$STABILITY" "promote" "$PULP_CONTENT_URL" diff --git a/pulp/repository-properties/action.yml b/pulp/repository-properties/action.yml new file mode 100644 index 0000000..345cca5 --- /dev/null +++ b/pulp/repository-properties/action.yml @@ -0,0 +1,146 @@ +name: "pulp-repository-properties" +description: "Compute pulp repository names, base paths and suites of a package delivery" +inputs: + module_name: + description: "The package module name" + required: true + distrib: + description: "The distribution used for packaging" + required: true + version: + description: "Centreon packaged major version (not used by the plugins repository family)" + required: false + default: "" + stability: + description: "The package stability (stable, testing, unstable)" + required: true + release_type: + description: "Type of release (hotfix, release)" + required: false + default: "" + is_cloud: + description: "Release context (cloud or not cloud)" + required: true + delivery_type: + description: "Delivery to feature or standard repository" + required: false + default: "standard" + repository_name: + description: "Target repository family: standard, standard-internal, business, business-internal, plugins or connectors" + required: false + default: "standard" +outputs: + package_extension: + description: "Package extension according to the distribution" + value: ${{ steps.properties.outputs.package_extension }} + distrib_family: + description: "Distrib family (el, debian, ubuntu)" + value: ${{ steps.properties.outputs.distrib_family }} + package_distrib_name: + description: "Distribution suffix in package name" + value: ${{ steps.properties.outputs.package_distrib_name }} + skip_delivery: + description: "Whether the delivery must be skipped" + value: ${{ steps.properties.outputs.skip_delivery }} + repository_prefix: + description: "Repository name prefix of rpm repositories (one repository per architecture)" + value: ${{ steps.properties.outputs.repository_prefix }} + base_path_prefix: + description: "Base path prefix of rpm distributions (one distribution per architecture)" + value: ${{ steps.properties.outputs.base_path_prefix }} + legacy_base_path_prefix: + description: "Legacy (front) rpm base path prefix, used by the content verifications" + value: ${{ steps.properties.outputs.legacy_base_path_prefix }} + legacy_testing_base_path_prefix: + description: "Legacy (front) testing rpm base path prefix" + value: ${{ steps.properties.outputs.legacy_testing_base_path_prefix }} + legacy_stable_base_path_prefix: + description: "Legacy (front) stable rpm base path prefix" + value: ${{ steps.properties.outputs.legacy_stable_base_path_prefix }} + repository_name: + description: "Repository name of the deb repository" + value: ${{ steps.properties.outputs.repository_name }} + base_path: + description: "Base path of the deb distribution" + value: ${{ steps.properties.outputs.base_path }} + legacy_base_path: + description: "Legacy (front) deb base path, used by the content verifications" + value: ${{ steps.properties.outputs.legacy_base_path }} + legacy_testing_base_path: + description: "Legacy (front) testing deb base path" + value: ${{ steps.properties.outputs.legacy_testing_base_path }} + legacy_stable_base_path: + description: "Legacy (front) stable deb base path" + value: ${{ steps.properties.outputs.legacy_stable_base_path }} + suite: + description: "Apt suite of the deb delivery" + value: ${{ steps.properties.outputs.suite }} + testing_repository_prefix: + description: "Repository name prefix of testing rpm repositories, used as promotion source" + value: ${{ steps.properties.outputs.testing_repository_prefix }} + testing_base_path_prefix: + description: "Base path prefix of testing rpm distributions, used to download promotion source content" + value: ${{ steps.properties.outputs.testing_base_path_prefix }} + stable_repository_prefix: + description: "Repository name prefix of stable rpm repositories" + value: ${{ steps.properties.outputs.stable_repository_prefix }} + stable_base_path_prefix: + description: "Base path prefix of stable rpm distributions" + value: ${{ steps.properties.outputs.stable_base_path_prefix }} + testing_repository_name: + description: "Repository name of the testing deb repository, used as promotion source" + value: ${{ steps.properties.outputs.testing_repository_name }} + testing_base_path: + description: "Base path of the testing deb distribution, used to download promotion source content" + value: ${{ steps.properties.outputs.testing_base_path }} + testing_suite: + description: "Apt suite of testing deb packages" + value: ${{ steps.properties.outputs.testing_suite }} + stable_suite: + description: "Apt suite of stable deb packages" + value: ${{ steps.properties.outputs.stable_suite }} + stable_legacy_suite: + description: "Plain-codename suite of the dedicated 24.x stable legacy repositories (empty otherwise)" + value: ${{ steps.properties.outputs.stable_legacy_suite }} + stable_legacy_repository_name: + description: "Dedicated 24.x stable legacy repository fed by the promote mirror (empty otherwise)" + value: ${{ steps.properties.outputs.stable_legacy_repository_name }} + stable_repository_name: + description: "Repository holding the stable deb packages (one deb repository per stability)" + value: ${{ steps.properties.outputs.stable_repository_name }} + stable_base_path: + description: "Base path serving the stable packages" + value: ${{ steps.properties.outputs.stable_base_path }} + pool_path: + description: "Module scoped pool path of the deb delivery" + value: ${{ steps.properties.outputs.pool_path }} + testing_pool_path: + description: "Module scoped pool path of testing deb packages, used as promotion source" + value: ${{ steps.properties.outputs.testing_pool_path }} + stable_pool_path: + description: "Module scoped pool path of stable deb packages" + value: ${{ steps.properties.outputs.stable_pool_path }} + domain: + description: "Pulp Domain of the delivery (testing/unstable tier) repository" + value: ${{ steps.properties.outputs.domain }} + stable_domain: + description: "Pulp Domain of the stable-tier repository" + value: ${{ steps.properties.outputs.stable_domain }} + +runs: + using: "composite" + steps: + - name: Compute pulp repository properties + id: properties + shell: bash + env: + MODULE_NAME: ${{ inputs.module_name }} + DISTRIB: ${{ inputs.distrib }} + VERSION: ${{ inputs.version }} + STABILITY: ${{ inputs.stability }} + RELEASE_TYPE: ${{ inputs.release_type }} + IS_CLOUD: ${{ inputs.is_cloud }} + DELIVERY_TYPE: ${{ inputs.delivery_type }} + REPOSITORY_TYPE: ${{ inputs.repository_name }} + GH_HEAD_REF: ${{ github.head_ref || github.ref_name }} + run: ${{ github.action_path }}/properties.sh diff --git a/pulp/repository-properties/parse-distrib.sh b/pulp/repository-properties/parse-distrib.sh new file mode 100644 index 0000000..1769bf8 --- /dev/null +++ b/pulp/repository-properties/parse-distrib.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Sourced by properties.sh: sets DISTRIB_FAMILY, PACKAGE_EXTENSION, +# PACKAGE_DISTRIB_NAME and PACKAGE_DISTRIB_SEPARATOR from DISTRIB. + +case "$DISTRIB" in + centos7 | el7) PACKAGE_DISTRIB_SEPARATOR="." PACKAGE_DISTRIB_NAME="el7" PACKAGE_EXTENSION="rpm" DISTRIB_FAMILY="el" ;; + alma8 | el8) PACKAGE_DISTRIB_SEPARATOR="." PACKAGE_DISTRIB_NAME="el8" PACKAGE_EXTENSION="rpm" DISTRIB_FAMILY="el" ;; + alma9 | el9) PACKAGE_DISTRIB_SEPARATOR="." PACKAGE_DISTRIB_NAME="el9" PACKAGE_EXTENSION="rpm" DISTRIB_FAMILY="el" ;; + alma10 | el10) PACKAGE_DISTRIB_SEPARATOR="." PACKAGE_DISTRIB_NAME="el10" PACKAGE_EXTENSION="rpm" DISTRIB_FAMILY="el" ;; + bullseye) PACKAGE_DISTRIB_SEPARATOR="+" PACKAGE_DISTRIB_NAME="deb11u1" PACKAGE_EXTENSION="deb" DISTRIB_FAMILY="debian" ;; + bookworm) PACKAGE_DISTRIB_SEPARATOR="+" PACKAGE_DISTRIB_NAME="deb12u1" PACKAGE_EXTENSION="deb" DISTRIB_FAMILY="debian" ;; + trixie) PACKAGE_DISTRIB_SEPARATOR="+" PACKAGE_DISTRIB_NAME="deb13u1" PACKAGE_EXTENSION="deb" DISTRIB_FAMILY="debian" ;; + jammy) PACKAGE_DISTRIB_SEPARATOR="-" PACKAGE_DISTRIB_NAME="0ubuntu.22.04" PACKAGE_EXTENSION="deb" DISTRIB_FAMILY="ubuntu" ;; + noble) PACKAGE_DISTRIB_SEPARATOR="-" PACKAGE_DISTRIB_NAME="0ubuntu.24.04" PACKAGE_EXTENSION="deb" DISTRIB_FAMILY="ubuntu" ;; + *) + echo "::error::Distrib $DISTRIB cannot be parsed" + exit 1 + ;; +esac + +{ + echo "package_distrib_separator=$PACKAGE_DISTRIB_SEPARATOR" + echo "package_distrib_name=$PACKAGE_DISTRIB_NAME" + echo "package_extension=$PACKAGE_EXTENSION" + echo "distrib_family=$DISTRIB_FAMILY" +} >> "$GITHUB_OUTPUT" diff --git a/pulp/repository-properties/properties.sh b/pulp/repository-properties/properties.sh new file mode 100755 index 0000000..37dce1b --- /dev/null +++ b/pulp/repository-properties/properties.sh @@ -0,0 +1,281 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ -z "$MODULE_NAME" || -z "$DISTRIB" || -z "$STABILITY" || -z "$IS_CLOUD" ]]; then + echo "::error::some mandatory inputs are empty, please check the logs." + exit 1 +fi + +# shellcheck source=pulp/repository-properties/parse-distrib.sh +source "$(dirname "$0")/parse-distrib.sh" + +# repository_name selects the standard (open-source), business (paid) or plugins +# repository family. standard/business have "-internal" variants: either a cloud +# context (is_cloud) or an explicit "-internal" suffix means the internal repo. +# plugins repositories are not versioned (packages carry their own date-based +# versioning), so the version input is only required for the other families. +REPOSITORY_TYPE="${REPOSITORY_TYPE:-standard}" +case "$REPOSITORY_TYPE" in + standard | standard-internal) REPO_BASE="standard" ;; + business | business-internal) REPO_BASE="business" ;; + plugins) REPO_BASE="plugins" ;; + connectors) REPO_BASE="connectors" ;; + *) + echo "::error::Unsupported repository_name: $REPOSITORY_TYPE" + exit 1 + ;; +esac + +if [[ "$REPO_BASE" != "plugins" && "$REPO_BASE" != "connectors" && -z "$VERSION" ]]; then + echo "::error::version input is mandatory for the $REPOSITORY_TYPE repository family." + exit 1 +fi + +# parse-distrib.sh emits "el"; per-version families ("el9", ...) are still +# accepted and normalized to "el" for the family checks below. +DEB_PREFIX="" +case "$DISTRIB_FAMILY" in + el | el[0-9]*) + DISTRIB_FAMILY="el" + ;; + debian) DEB_PREFIX="apt-" ;; + ubuntu) DEB_PREFIX="ubuntu-" ;; + *) + echo "::error::Unsupported distribution family: $DISTRIB_FAMILY" + exit 1 + ;; +esac + +# uniform stability segments across every edition (plugins included): +# unstable, testing-release, testing-hotfix, stable +TESTING_SEGMENT="testing" +TESTING_POOL_SEGMENT="testing" +if [[ "$RELEASE_TYPE" == "release" || "$RELEASE_TYPE" == "hotfix" ]]; then + TESTING_SEGMENT="testing-$RELEASE_TYPE" + TESTING_POOL_SEGMENT="testing/$RELEASE_TYPE" +fi + +STABILITY_SEGMENT="$STABILITY" +POOL_SEGMENT="$STABILITY" +if [[ "$STABILITY" == "testing" ]]; then + STABILITY_SEGMENT="$TESTING_SEGMENT" + POOL_SEGMENT="$TESTING_POOL_SEGMENT" +fi + +REPOSITORY_PREFIX="" +BASE_PATH_PREFIX="" +LEGACY_BASE_PATH_PREFIX="" +LEGACY_TESTING_BASE_PATH_PREFIX="" +LEGACY_STABLE_BASE_PATH_PREFIX="" +LEGACY_BASE_PATH="" +LEGACY_TESTING_BASE_PATH="" +LEGACY_STABLE_BASE_PATH="" +TESTING_REPOSITORY_PREFIX="" +TESTING_BASE_PATH_PREFIX="" +STABLE_REPOSITORY_PREFIX="" +STABLE_BASE_PATH_PREFIX="" +REPOSITORY_NAME="" +BASE_PATH="" +SUITE="" +TESTING_REPOSITORY_NAME="" +TESTING_BASE_PATH="" +TESTING_SUITE="" +STABLE_SUITE="" +STABLE_LEGACY_SUITE="" +STABLE_LEGACY_REPOSITORY_NAME="" +POOL_PATH="" +TESTING_POOL_PATH="" +STABLE_POOL_PATH="" + +if [[ "$REPO_BASE" == "plugins" || "$REPO_BASE" == "connectors" ]]; then + # plugins and connectors repositories are not versioned: rpm paths carry no + # version segment and deb suites are the plain distribution codename. + if [[ "$DELIVERY_TYPE" == "feature" ]]; then + echo "::notice::Feature delivery is not supported for $REPO_BASE packages, skipping delivery." + echo "skip_delivery=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + if [[ "$DISTRIB_FAMILY" == "el" ]]; then + REPOSITORY_PREFIX="rpm-$DISTRIB-$STABILITY_SEGMENT" + BASE_PATH_PREFIX="rpm/$DISTRIB/$STABILITY_SEGMENT" + TESTING_REPOSITORY_PREFIX="rpm-$DISTRIB-$TESTING_SEGMENT" + TESTING_BASE_PATH_PREFIX="rpm/$DISTRIB/$TESTING_SEGMENT" + STABLE_REPOSITORY_PREFIX="rpm-$DISTRIB-stable" + STABLE_BASE_PATH_PREFIX="rpm/$DISTRIB/stable" + LEGACY_SEGMENT_RPM="$STABILITY_SEGMENT" + [[ "$STABILITY_SEGMENT" == "testing-release" ]] && LEGACY_SEGMENT_RPM="testing" + LEGACY_TESTING_SEGMENT_RPM="$TESTING_SEGMENT" + [[ "$TESTING_SEGMENT" == "testing-release" ]] && LEGACY_TESTING_SEGMENT_RPM="testing" + LEGACY_BASE_PATH_PREFIX="rpm-$REPO_BASE/$DISTRIB/$LEGACY_SEGMENT_RPM" + LEGACY_TESTING_BASE_PATH_PREFIX="rpm-$REPO_BASE/$DISTRIB/$LEGACY_TESTING_SEGMENT_RPM" + LEGACY_STABLE_BASE_PATH_PREFIX="rpm-$REPO_BASE/$DISTRIB/stable" + else + # one deb repository per stability (base path = repository name), suites + # carry the plain codename only + REPOSITORY_NAME="${DEB_PREFIX}${STABILITY_SEGMENT}" + BASE_PATH="$REPOSITORY_NAME" + # historical layout: one repository per stability with plain-codename + # suites; the legacy stability (bare "testing") lives in the front prefix + LEGACY_SEGMENT="$STABILITY_SEGMENT" + [[ "$STABILITY_SEGMENT" == "testing-release" ]] && LEGACY_SEGMENT="testing" + LEGACY_TESTING_SEGMENT="$TESTING_SEGMENT" + [[ "$TESTING_SEGMENT" == "testing-release" ]] && LEGACY_TESTING_SEGMENT="testing" + SUITE="$DISTRIB" + TESTING_REPOSITORY_NAME="${DEB_PREFIX}${TESTING_SEGMENT}" + TESTING_BASE_PATH="$TESTING_REPOSITORY_NAME" + TESTING_SUITE="$DISTRIB" + STABLE_REPOSITORY_NAME="${DEB_PREFIX}stable" + STABLE_BASE_PATH="$STABLE_REPOSITORY_NAME" + STABLE_SUITE="$DISTRIB" + # legacy (front) paths used by the content verifications: the CI exercises + # the compatibility rewrites on every delivery + LEGACY_BASE_PATH="${DEB_PREFIX}${REPO_BASE}-${LEGACY_SEGMENT}" + LEGACY_TESTING_BASE_PATH="${DEB_PREFIX}${REPO_BASE}-${LEGACY_TESTING_SEGMENT}" + LEGACY_STABLE_BASE_PATH="${DEB_PREFIX}${REPO_BASE}-stable" + POOL_PATH="pool/$POOL_SEGMENT/$MODULE_NAME" + TESTING_POOL_PATH="pool/$TESTING_POOL_SEGMENT/$MODULE_NAME" + STABLE_POOL_PATH="pool/stable/$MODULE_NAME" + fi +elif [[ "$DELIVERY_TYPE" == "feature" ]]; then + if [[ "$DISTRIB_FAMILY" != "el" ]]; then + echo "::notice::Feature delivery is not supported for $DISTRIB_FAMILY packages, skipping delivery." + echo "skip_delivery=true" >> "$GITHUB_OUTPUT" + exit 0 + fi + + FEATURE_TICKET=$(echo "$GH_HEAD_REF" | grep -oE 'MON-[0-9]+' | head -1 || true) + if [[ -z "$FEATURE_TICKET" ]]; then + echo "::error::Cannot extract the feature ticket from branch name $GH_HEAD_REF" + exit 1 + fi + + REPOSITORY_PREFIX="rpm-feature-$FEATURE_TICKET-$VERSION-$DISTRIB-$STABILITY" + BASE_PATH_PREFIX="rpm-feature/$FEATURE_TICKET/$VERSION/$DISTRIB/$STABILITY" + LEGACY_BASE_PATH_PREFIX="rpm-$REPO_BASE-feature/$FEATURE_TICKET/$VERSION/$DISTRIB/$STABILITY" +else + RPM_ROOT="rpm" + DEB_INFIX="" + if [[ "$IS_CLOUD" == "true" || "$REPOSITORY_TYPE" == *-internal ]]; then + RPM_ROOT="rpm-internal" + DEB_INFIX="internal-" + fi + + if [[ "$DISTRIB_FAMILY" == "el" ]]; then + REPOSITORY_PREFIX="$RPM_ROOT-$VERSION-$DISTRIB-$STABILITY_SEGMENT" + BASE_PATH_PREFIX="$RPM_ROOT/$VERSION/$DISTRIB/$STABILITY_SEGMENT" + TESTING_REPOSITORY_PREFIX="$RPM_ROOT-$VERSION-$DISTRIB-$TESTING_SEGMENT" + TESTING_BASE_PATH_PREFIX="$RPM_ROOT/$VERSION/$DISTRIB/$TESTING_SEGMENT" + STABLE_REPOSITORY_PREFIX="$RPM_ROOT-$VERSION-$DISTRIB-stable" + STABLE_BASE_PATH_PREFIX="$RPM_ROOT/$VERSION/$DISTRIB/stable" + # legacy (front) paths used by the content verifications (the front also + # accepts the historical token'd business form) + LEGACY_RPM_ROOT="rpm-$REPO_BASE" + [[ "$RPM_ROOT" == "rpm-internal" ]] && LEGACY_RPM_ROOT="rpm-$REPO_BASE-internal" + LEGACY_BASE_PATH_PREFIX="$LEGACY_RPM_ROOT/$VERSION/$DISTRIB/$STABILITY_SEGMENT" + LEGACY_TESTING_BASE_PATH_PREFIX="$LEGACY_RPM_ROOT/$VERSION/$DISTRIB/$TESTING_SEGMENT" + LEGACY_STABLE_BASE_PATH_PREFIX="$LEGACY_RPM_ROOT/$VERSION/$DISTRIB/stable" + else + # one deb repository per stability (base path = repository name), shared by + # every major version: the version lives in the suite name only + # (e.g. "trixie-26.09") + REPOSITORY_NAME="${DEB_PREFIX}${DEB_INFIX}${STABILITY_SEGMENT}" + BASE_PATH="$REPOSITORY_NAME" + # suites keep their historical stability-suffixed names so the legacy + # client lines (deb .../apt-standard/ trixie-26.10-stable main) work + # through the content-front rewrites; each suite lives in its own + # per-stability repository regardless + SUITE="$DISTRIB-$VERSION-$STABILITY_SEGMENT" + TESTING_REPOSITORY_NAME="${DEB_PREFIX}${DEB_INFIX}${TESTING_SEGMENT}" + TESTING_BASE_PATH="$TESTING_REPOSITORY_NAME" + TESTING_SUITE="$DISTRIB-$VERSION-$TESTING_SEGMENT" + STABLE_REPOSITORY_NAME="${DEB_PREFIX}${DEB_INFIX}stable" + STABLE_BASE_PATH="$STABLE_REPOSITORY_NAME" + STABLE_SUITE="$DISTRIB-$VERSION-stable" + # 24.10 clients use dedicated legacy stable repositories with plain-codename + # suites (apt|ubuntu-standard--stable, apt--business-stable): + # the promote mirrors its package associations into them + if [[ -z "$DEB_INFIX" && "$VERSION" == "24.10" ]]; then + if [[ "$REPO_BASE" == "standard" ]]; then + STABLE_LEGACY_REPOSITORY_NAME="${DEB_PREFIX}standard-$VERSION-stable" + STABLE_LEGACY_SUITE="$DISTRIB" + elif [[ "$REPO_BASE" == "business" && "$DEB_PREFIX" == "apt-" ]]; then + STABLE_LEGACY_REPOSITORY_NAME="apt-$VERSION-business-stable" + STABLE_LEGACY_SUITE="$DISTRIB" + fi + fi + LEGACY_DEB_ROOT="${DEB_PREFIX}${REPO_BASE}" + [[ -n "$DEB_INFIX" ]] && LEGACY_DEB_ROOT="${DEB_PREFIX}${REPO_BASE}-internal" + LEGACY_BASE_PATH="$LEGACY_DEB_ROOT" + LEGACY_TESTING_BASE_PATH="$LEGACY_DEB_ROOT" + LEGACY_STABLE_BASE_PATH="$LEGACY_DEB_ROOT" + POOL_PATH="pool/$VERSION/$POOL_SEGMENT/$MODULE_NAME" + TESTING_POOL_PATH="pool/$VERSION/$TESTING_POOL_SEGMENT/$MODULE_NAME" + STABLE_POOL_PATH="pool/$VERSION/stable/$MODULE_NAME" + fi +fi + +# unless a dedicated stable repository was selected above, stable shares the +# delivery repository (rpm resolves stable through the *_PREFIX variables) +STABLE_REPOSITORY_NAME="${STABLE_REPOSITORY_NAME:-$REPOSITORY_NAME}" +STABLE_BASE_PATH="${STABLE_BASE_PATH:-$BASE_PATH}" + +# one Pulp Domain per edition; stable and non-stable repositories share it +# ("-internal"/cloud repos too). The stable/non-stable write boundary is the +# repository name, enforced server-side by name-scoped grants. stable_domain +# is kept as a distinct output for the scripts that address the stable tier, +# even though it now always equals domain. +DOMAIN="$REPO_BASE" +STABLE_DOMAIN="$DOMAIN" + +echo "[DEBUG] - repository_type: $REPOSITORY_TYPE" +echo "[DEBUG] - repository_prefix: $REPOSITORY_PREFIX" +echo "[DEBUG] - base_path_prefix: $BASE_PATH_PREFIX" +echo "[DEBUG] - testing_repository_prefix: $TESTING_REPOSITORY_PREFIX" +echo "[DEBUG] - testing_base_path_prefix: $TESTING_BASE_PATH_PREFIX" +echo "[DEBUG] - stable_repository_prefix: $STABLE_REPOSITORY_PREFIX" +echo "[DEBUG] - stable_base_path_prefix: $STABLE_BASE_PATH_PREFIX" +echo "[DEBUG] - repository_name: $REPOSITORY_NAME" +echo "[DEBUG] - base_path: $BASE_PATH" +echo "[DEBUG] - suite: $SUITE" +echo "[DEBUG] - testing_repository_name: $TESTING_REPOSITORY_NAME" +echo "[DEBUG] - testing_base_path: $TESTING_BASE_PATH" +echo "[DEBUG] - testing_suite: $TESTING_SUITE" +echo "[DEBUG] - stable_suite: $STABLE_SUITE" +echo "[DEBUG] - pool_path: $POOL_PATH" +echo "[DEBUG] - testing_pool_path: $TESTING_POOL_PATH" +echo "[DEBUG] - stable_pool_path: $STABLE_POOL_PATH" +echo "[DEBUG] - domain: $DOMAIN" +echo "[DEBUG] - stable_domain: $STABLE_DOMAIN" + +{ + echo "skip_delivery=false" + echo "repository_prefix=$REPOSITORY_PREFIX" + echo "base_path_prefix=$BASE_PATH_PREFIX" + echo "legacy_base_path_prefix=$LEGACY_BASE_PATH_PREFIX" + echo "legacy_testing_base_path_prefix=$LEGACY_TESTING_BASE_PATH_PREFIX" + echo "legacy_stable_base_path_prefix=$LEGACY_STABLE_BASE_PATH_PREFIX" + echo "testing_repository_prefix=$TESTING_REPOSITORY_PREFIX" + echo "testing_base_path_prefix=$TESTING_BASE_PATH_PREFIX" + echo "stable_repository_prefix=$STABLE_REPOSITORY_PREFIX" + echo "stable_base_path_prefix=$STABLE_BASE_PATH_PREFIX" + echo "repository_name=$REPOSITORY_NAME" + echo "base_path=$BASE_PATH" + echo "legacy_base_path=$LEGACY_BASE_PATH" + echo "legacy_testing_base_path=$LEGACY_TESTING_BASE_PATH" + echo "legacy_stable_base_path=$LEGACY_STABLE_BASE_PATH" + echo "suite=$SUITE" + echo "testing_repository_name=$TESTING_REPOSITORY_NAME" + echo "testing_base_path=$TESTING_BASE_PATH" + echo "testing_suite=$TESTING_SUITE" + echo "stable_suite=$STABLE_SUITE" + echo "stable_legacy_suite=$STABLE_LEGACY_SUITE" + echo "stable_legacy_repository_name=$STABLE_LEGACY_REPOSITORY_NAME" + echo "stable_repository_name=$STABLE_REPOSITORY_NAME" + echo "stable_base_path=$STABLE_BASE_PATH" + echo "pool_path=$POOL_PATH" + echo "testing_pool_path=$TESTING_POOL_PATH" + echo "stable_pool_path=$STABLE_POOL_PATH" + echo "domain=$DOMAIN" + echo "stable_domain=$STABLE_DOMAIN" +} >> "$GITHUB_OUTPUT" diff --git a/pulp/setup-cli/action.yml b/pulp/setup-cli/action.yml new file mode 100644 index 0000000..f02c17d --- /dev/null +++ b/pulp/setup-cli/action.yml @@ -0,0 +1,29 @@ +name: "setup-pulp-cli" +description: "Install pulp-cli and authenticate to pulp with a GitHub OIDC token" +inputs: + pulp_url: + description: "The pulp server api url" + required: true + audience: + description: "OIDC audience, must match the pulp server GITHUB_OIDC.audience" + required: true + domain: + description: "Pulp Domain to operate in (DOMAIN_ENABLED). Pre-Domains repositories live in \"default\"." + required: false + default: "default" + +runs: + using: "composite" + steps: + - name: Setup python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: "3.12" + + - name: Setup pulp-cli + shell: bash + env: + PULP_URL: ${{ inputs.pulp_url }} + AUDIENCE: ${{ inputs.audience }} + DOMAIN: ${{ inputs.domain }} + run: ${{ github.action_path }}/setup.sh diff --git a/pulp/setup-cli/setup.sh b/pulp/setup-cli/setup.sh new file mode 100755 index 0000000..db01d40 --- /dev/null +++ b/pulp/setup-cli/setup.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Install pulp-cli, request a GitHub OIDC token and configure pulp-cli. +# Inputs (env): PULP_URL, AUDIENCE, DOMAIN. +set -euo pipefail + +if ! command -v pulp > /dev/null 2>&1; then + echo "[INFO] Installing pulp-cli" + pip install --quiet "pulp-cli==0.37.1" "pulp-cli-deb==0.4.3" +fi + +# an unset org variable is forwarded as "": fail loudly rather than +# falling back to a default audience, a token minted for the wrong +# audience must never be silently requested against another server +if [[ -z "${AUDIENCE:-}" ]]; then + echo "::error::audience input is empty (is the PULP_OIDC_AUDIENCE org variable defined?)" + exit 1 +fi + +# the job must grant id-token write permission for these to be set +if [[ -z "${ACTIONS_ID_TOKEN_REQUEST_URL:-}" || -z "${ACTIONS_ID_TOKEN_REQUEST_TOKEN:-}" ]]; then + echo "::error::Missing OIDC token request context, the job needs 'permissions: id-token: write'" + exit 1 +fi + +# retry on transient GitHub OIDC token service errors (5xx/timeouts): +# the endpoint occasionally returns 503, and a single blip must not break +# the whole delivery. +PULP_TOKEN=$( + curl -fsSL --retry 5 --retry-delay 3 --retry-all-errors \ + -H "Authorization: bearer ${ACTIONS_ID_TOKEN_REQUEST_TOKEN}" \ + -G --data-urlencode "audience=${AUDIENCE}" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}" | jq -r '.value' +) +if [[ -z "$PULP_TOKEN" || "$PULP_TOKEN" == "null" ]]; then + echo "::error::Could not obtain a GitHub OIDC token" + exit 1 +fi + +# keep the token out of any log and expose it to the next steps. +# the issuance time and audience let the api helpers refresh the token +# mid-delivery: github oidc tokens expire ~5 minutes after issuance, +# which is shorter than a large delivery. +echo "::add-mask::${PULP_TOKEN}" +{ + echo "PULP_TOKEN=${PULP_TOKEN}" + echo "PULP_TOKEN_ISSUED_AT=$(date +%s)" + echo "PULP_OIDC_AUDIENCE=${AUDIENCE}" +} >> "$GITHUB_ENV" + +# fall back to the default when the caller passes an empty value (unset +# org variable forwarded as "" would otherwise override the default). +PULP_URL="${PULP_URL:-https://pulp-api.int.centreon.com}" +DOMAIN="${DOMAIN:-default}" + +# keep the pulp-cli config (which stores the OIDC token) under the per-job +# RUNNER_TEMP rather than the runner's persistent ~/.config, and share the +# location with the later steps, so a self-hosted runner does not retain +# the token in a home reused across jobs. +export XDG_CONFIG_HOME="${RUNNER_TEMP:-/tmp}/pulp-cli-config" +mkdir -p "$XDG_CONFIG_HOME" +echo "XDG_CONFIG_HOME=${XDG_CONFIG_HOME}" >> "$GITHUB_ENV" + +# WorkloadIdentityAuthentication reads a Bearer token, no username/password +# pulp-cli splits --header on the first colon and keeps the value +# verbatim, so no space after the colon (rejected as leading whitespace) +pulp config create --overwrite \ + --base-url "$PULP_URL" \ + --api-root "/" \ + --domain "$DOMAIN" \ + --header "Authorization:Bearer ${PULP_TOKEN}" \ + --timeout 0 + +# fail fast on connectivity issues, scripts cannot distinguish +# a missing resource from an unreachable server otherwise +if ! pulp status > /dev/null; then + echo "::error::Pulp server ${PULP_URL} is not reachable" + exit 1 +fi From 8941e4e162fbf871aaf2842ee126f65d31774db2 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 19 Nov 2026 11:07:27 +0100 Subject: [PATCH 02/15] ci: add release-please versioning and lint workflows Release the repository as vX.Y.Z tags from the conventional commit titles merged on main, through a release pull request and a dedicated app token. Lint the actions (self-contained, SHA-pinned), the scripts and the workflows on every pull request. Refs: MON-215510 --- .github/dependabot.yml | 19 ++++++++ .github/scripts/check-actions.sh | 29 ++++++++++++ .github/workflows/lint.yml | 44 +++++++++++++++++++ .github/workflows/release.yml | 57 ++++++++++++++++++++++++ .release-please-manifest.json | 1 + README.md | 75 +++++++++++++++++++++++++++++--- release-please-config.json | 49 +++++++++++++++++++++ 7 files changed, 269 insertions(+), 5 deletions(-) create mode 100644 .github/dependabot.yml create mode 100755 .github/scripts/check-actions.sh create mode 100644 .github/workflows/lint.yml create mode 100644 .github/workflows/release.yml create mode 100644 .release-please-manifest.json create mode 100644 release-please-config.json diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 0000000..08371fb --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,19 @@ +version: 2 +updates: + # pins of the workflows of this repository and of the shared actions + - package-ecosystem: github-actions + directories: + - "/" + - "/pulp/*" + schedule: + interval: monthly + open-pull-requests-limit: 10 + groups: + github-actions: + patterns: + - "*" + commit-message: + prefix: "fix(deps)" + labels: + - "dependencies" + - "gha" diff --git a/.github/scripts/check-actions.sh b/.github/scripts/check-actions.sh new file mode 100755 index 0000000..db48a09 --- /dev/null +++ b/.github/scripts/check-actions.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Repository rules for the shared actions (every action.yml outside .github): +# - no local "uses: ./...": it resolves in the caller's workspace, not here; +# - every remote action is pinned to a full commit SHA with a version comment. +set -euo pipefail + +errors=0 + +while IFS= read -r -d '' action_file; do + while IFS= read -r line; do + number=${line%%:*} + ref=${line#*uses:} + ref=${ref#"${ref%%[![:space:]]*}"} + ref=${ref#[\"\']} + if [[ "$ref" == ./* ]]; then + echo "::error file=${action_file},line=${number}::local action '${ref}' resolves in the caller's workspace; call a script through github.action_path instead" + errors=$((errors + 1)) + elif [[ ! "$ref" =~ ^[^@[:space:]]+@[0-9a-f]{40}[\"\']?[[:space:]]+#[[:space:]]*v?[0-9]+(\.[0-9]+)*$ ]]; then + echo "::error file=${action_file},line=${number}::'${ref}' must be pinned to a full commit SHA followed by a '# vX.Y.Z' comment" + errors=$((errors + 1)) + fi + done < <(grep -nE '^[[:space:]-]*uses:' "$action_file" || true) +done < <(find . -path ./.git -prune -o -path ./.github -prune -o -name action.yml -print0) + +if ((errors > 0)); then + echo "${errors} error(s) found" + exit 1 +fi +echo "All actions are self-contained and SHA-pinned" diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 0000000..b5799c4 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,44 @@ +name: lint + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +on: + pull_request: + workflow_dispatch: + +permissions: {} + +env: + ACTIONLINT_VERSION: "1.7.12" + ACTIONLINT_SHA256: "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8" + +jobs: + lint: + runs-on: ubuntu-24.04 + permissions: + contents: read + steps: + - name: Checkout sources + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Check actions are self-contained and pinned + run: .github/scripts/check-actions.sh + + - name: Shellcheck + run: | + mapfile -t scripts < <(git ls-files '*.sh') + shellcheck --version + shellcheck -x -P SCRIPTDIR -S warning "${scripts[@]}" + + - name: Actionlint + run: | + archive="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" + curl -fsSL --retry 3 -o "${RUNNER_TEMP}/${archive}" \ + "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/${archive}" + echo "${ACTIONLINT_SHA256} ${RUNNER_TEMP}/${archive}" | sha256sum --check --strict + tar -xzf "${RUNNER_TEMP}/${archive}" -C "${RUNNER_TEMP}" actionlint + "${RUNNER_TEMP}/actionlint" -color diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..09b3518 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,57 @@ +name: release + +# Maintains the release PR (version bump + CHANGELOG.md) from the conventional +# commit titles merged on main; merging it tags vX.Y.Z and publishes the release. + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +on: + push: + branches: + - main + +permissions: {} + +jobs: + release-please: + runs-on: ubuntu-24.04 + # the environment only allows main and holds the release app key + environment: release + permissions: + contents: read + steps: + # an app token, not GITHUB_TOKEN: the release PR must trigger the required + # checks, and the app is the only actor allowed to create release tags + - name: Create release app token + id: app-token + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + client-id: ${{ vars.RELEASE_APP_CLIENT_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + permission-issues: write + + - name: Run release-please + id: release + uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5.0.0 + with: + token: ${{ steps.app-token.outputs.token }} + config-file: release-please-config.json + manifest-file: .release-please-manifest.json + + - name: Summary + if: steps.release.outputs.release_created == 'true' + env: + TAG_NAME: ${{ steps.release.outputs.tag_name }} + SHA: ${{ steps.release.outputs.sha }} + run: | + { + echo "Released \`${TAG_NAME}\` at \`${SHA}\`:" + echo + echo '```yaml' + echo "uses: centreon/github-actions/@${SHA} # ${TAG_NAME}" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1 @@ +{} diff --git a/README.md b/README.md index 18f672b..7dc75d9 100644 --- a/README.md +++ b/README.md @@ -4,15 +4,66 @@ Shared GitHub Actions used by Centreon CI workflows. ## Usage -Reference an action by its path in this repository, pinned to a full commit SHA: +Reference an action by its path in this repository, pinned to the full commit SHA of a +release, with the release tag as a comment: ```yaml -- uses: centreon/github-actions/pulp/package-delivery@ # +- uses: centreon/github-actions/pulp/package-delivery@ # v1.2.0 ``` +- Always pin a SHA, never a tag or a branch: a SHA cannot be moved, and Dependabot + reads the `# vX.Y.Z` comment to propose the next release. +- Use the same release for every action of this repository in a workflow. +- Each maintained branch of a product repository keeps its own pin: an older branch + stays on the release it was validated with until it is bumped on purpose. + +Let Dependabot bump the pins in one pull request per product repository: + +```yaml +# .github/dependabot.yml of the product repository +- package-ecosystem: github-actions + directory: "/" + schedule: + interval: monthly + groups: + centreon-github-actions: + patterns: + - "centreon/github-actions*" +``` + +An urgent fix does not wait for Dependabot: bump the SHA and the comment by hand. + +## Versioning + +The repository is released as a whole with [Semantic Versioning](https://semver.org/) +tags `vX.Y.Z`, each with an immutable GitHub release and an entry in `CHANGELOG.md`. + +| Change to any action | Pull request title | Release | +| ---------------------------------------------------------------------------------------------- | ---------------------------- | ------- | +| Removed or renamed input/output, new required input, changed default or behavior a caller sees | `feat(pulp)!: …` (or `fix!`) | major | +| New action, new optional input or output | `feat(pulp): …` | minor | +| Bug fix, pinned dependency bump, internal refactoring | `fix(pulp): …` | patch | +| Documentation, CI of this repository | `docs: …`, `ci: …` | none | + +Pull requests are squash-merged, so their title is the commit that decides the release. + +How a release happens: + +1. Every merge on `main` updates the release pull request (`chore: release X.Y.Z`), + which carries the version bump and the `CHANGELOG.md` entry. +2. Merging the release pull request creates the `vX.Y.Z` tag and its GitHub release. +3. The release summary gives the SHA to pin. + +Tags are never moved or deleted, and there is no floating major tag (`v1`): callers pin +SHAs, so a moving tag would bring nothing but risk. + +## Writing an action + Each action is self-contained: it never calls a sibling action through `uses: ./...` -(that path would resolve in the caller's workspace). Shared logic lives in scripts -called through `${{ github.action_path }}`, so every file comes from the same commit. +(that path resolves in the caller's workspace). Shared logic lives in scripts called +through `${{ github.action_path }}`, so every file comes from the pinned commit. Remote +actions are pinned to a full SHA with a `# vX.Y.Z` comment. The `lint` workflow checks +both rules, plus shellcheck and actionlint. ## Pulp delivery @@ -44,7 +95,7 @@ deliver-pulp: contents: read id-token: write steps: - - uses: centreon/github-actions/pulp/package-delivery@ # main + - uses: centreon/github-actions/pulp/package-delivery@ # v1.0.0 with: module_name: centreon-web distrib: el9 @@ -57,3 +108,17 @@ deliver-pulp: pulp_content_url: ${{ vars.PULP_CONTENT_URL }} audience: ${{ vars.PULP_OIDC_AUDIENCE }} ``` + +## Repository settings + +Kept outside the code, set by a repository admin: + +- **Release app**: a GitHub App with `contents`, `pull-requests` and `issues` write + access to this repository only; its client id in the `RELEASE_APP_CLIENT_ID` variable + and its private key in the `RELEASE_APP_PRIVATE_KEY` secret of the `release` + environment. +- **`release` environment**: deployment branches limited to `main`. +- **`Release tags` ruleset**: the release app is the only bypass actor, so no one else + can create, move or delete a tag. +- **Immutable releases**: enabled, so a published release and its tag cannot change. +- **Required status check**: `lint` on `main`. diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000..fd7ff27 --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,49 @@ +{ + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "release-type": "simple", + "include-component-in-tag": false, + "include-v-in-tag": true, + "pull-request-title-pattern": "chore: release ${version}", + "changelog-sections": [ + { + "type": "feat", + "section": "Features" + }, + { + "type": "fix", + "section": "Bug Fixes" + }, + { + "type": "perf", + "section": "Performance Improvements" + }, + { + "type": "revert", + "section": "Reverts" + }, + { + "type": "refactor", + "section": "Code Refactoring" + }, + { + "type": "docs", + "section": "Documentation", + "hidden": true + }, + { + "type": "ci", + "section": "Continuous Integration", + "hidden": true + }, + { + "type": "chore", + "section": "Miscellaneous", + "hidden": true + } + ], + "packages": { + ".": { + "initial-version": "1.0.0" + } + } +} From f122f333e6f4da293e0987d89b9648aa74e8d8b9 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 19 Nov 2026 11:18:43 +0100 Subject: [PATCH 03/15] ci: release each action with its own floating major tag Replace release-please with a per-action release: every merge on main tags the changed actions -vX.Y.Z (immutable release) and moves their -vX tag, so callers get fixes by referencing the major tag. The lint check validates the PR title and previews the releases. Refs: MON-215510 --- .github/release-components.json | 16 ++ .github/scripts/release.py | 283 ++++++++++++++++++++++++++++++++ .github/scripts/ruff.toml | 1 + .github/scripts/test_release.py | 205 +++++++++++++++++++++++ .github/workflows/lint.yml | 20 +++ .github/workflows/release.yml | 38 ++--- .gitignore | 1 + .release-please-manifest.json | 1 - README.md | 76 +++++---- release-please-config.json | 49 ------ 10 files changed, 580 insertions(+), 110 deletions(-) create mode 100644 .github/release-components.json create mode 100644 .github/scripts/release.py create mode 100644 .github/scripts/ruff.toml create mode 100644 .github/scripts/test_release.py create mode 100644 .gitignore delete mode 100644 .release-please-manifest.json delete mode 100644 release-please-config.json diff --git a/.github/release-components.json b/.github/release-components.json new file mode 100644 index 0000000..e5befb2 --- /dev/null +++ b/.github/release-components.json @@ -0,0 +1,16 @@ +{ + "pulp-package-delivery": [ + "pulp/package-delivery", + "pulp/repository-properties", + "pulp/setup-cli", + "pulp/lib" + ], + "pulp-promote-to-stable": [ + "pulp/promote-to-stable", + "pulp/repository-properties", + "pulp/setup-cli", + "pulp/lib" + ], + "pulp-repository-properties": ["pulp/repository-properties"], + "pulp-setup-cli": ["pulp/setup-cli"] +} diff --git a/.github/scripts/release.py b/.github/scripts/release.py new file mode 100644 index 0000000..2f2c0a8 --- /dev/null +++ b/.github/scripts/release.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +"""Per-action releases of this repository. + +Each component of .github/release-components.json is an action released on +its own: immutable "-vX.Y.Z" tags carry a GitHub release, and the +floating "-vX" tag follows the latest X.y.z release. The bump comes +from the conventional commit titles touching the component's paths. + + validate check the component file against the actions + plan --title T --base B preview the releases a pull request would make + release release what changed on HEAD (needs GH_TOKEN) +""" + +import argparse +import json +import os +import re +import subprocess +import sys +import tempfile +from pathlib import Path + +COMPONENTS_FILE = Path(".github/release-components.json") +INITIAL_VERSION = (1, 0, 0) +TITLE_RE = re.compile( + r"^(?Pfeat|fix|perf|refactor|revert|docs|ci|chore|test|build|style)" + r"(?:\([a-z0-9._-]+\))?(?P!)?: \S" +) +PATCH_TYPES = {"fix", "perf", "refactor", "revert"} +LEVELS = {None: 0, "patch": 1, "minor": 2, "major": 3} + +Version = tuple[int, int, int] +Components = dict[str, list[str]] +Commit = tuple[str, str, str] + + +class CommandError(Exception): + """A git or gh command failed; carries its stderr.""" + + +def command(*args: str, check: bool = True) -> subprocess.CompletedProcess: + result = subprocess.run(args, check=False, capture_output=True, text=True) + if check and result.returncode != 0: + raise CommandError(f"{' '.join(args)}: {result.stderr.strip()}") + return result + + +def git(*args: str) -> str: + return command("git", *args).stdout + + +def gh(*args: str, check: bool = True) -> subprocess.CompletedProcess: + return command("gh", *args, check=check) + + +def action_dirs() -> list[str]: + return sorted( + str(p.parent) + for p in Path(".").glob("**/action.yml") + if not p.parts[0].startswith(".") + ) + + +def bump_level(title: str, body: str = "") -> str | None: + """Release level of a commit; None for a non-conventional or non-releasing type.""" + match = TITLE_RE.match(title) + if not match: + return None + if match["breaking"] or re.search(r"^BREAKING[ -]CHANGE: ", body, re.MULTILINE): + return "major" + if match["type"] == "feat": + return "minor" + if match["type"] in PATCH_TYPES: + return "patch" + return None + + +def released_versions(component: str) -> list[Version]: + pattern = re.compile(rf"^{re.escape(component)}-v(\d+)\.(\d+)\.(\d+)$") + tags = git("tag", "--list", f"{component}-v*").split() + return [(int(m[1]), int(m[2]), int(m[3])) for m in map(pattern.match, tags) if m] + + +def next_version(current: Version | None, level: str) -> Version: + if current is None: + return INITIAL_VERSION + major, minor, patch = current + if level == "major": + return (major + 1, 0, 0) + if level == "minor": + return (major, minor + 1, 0) + return (major, minor, patch + 1) + + +def tag_name(component: str, version: Version) -> str: + return f"{component}-v{'.'.join(map(str, version))}" + + +def commits_since( + component: str, paths: list[str], current: Version | None +) -> list[Commit]: + rev = "HEAD" if current is None else f"{tag_name(component, current)}..HEAD" + out = git("log", "--format=%H%x1f%s%x1f%b%x1e", rev, "--", *paths) + commits = [] + for record in filter(None, (r.strip("\n") for r in out.split("\x1e"))): + sha, subject, body = record.split("\x1f") + commits.append((sha, subject, body)) + return commits + + +def validate(components: Components) -> bool: + """Every action is the first path of the component named after it, and every path exists.""" + errors = [] + dirs = action_dirs() + for directory in dirs: + name = directory.replace("/", "-") + if components.get(name, [None])[0] != directory: + errors.append( + f"action {directory} needs a '{name}' component whose first path is {directory}" + ) + for name, paths in components.items(): + errors += [ + f"component {name}: path {p} does not exist" + for p in paths + if not Path(p).is_dir() + ] + if not paths or paths[0] not in dirs: + errors.append( + f"component {name}: its first path must be an action directory" + ) + for error in errors: + print(f"::error file={COMPONENTS_FILE}::{error}") + return not errors + + +def summary(lines: list[str]) -> None: + text = "\n".join(lines) + "\n" + print(text) + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: + f.write(text) + + +def plan(components: Components, title: str, base: str) -> bool: + """Show what merging a pull request with this title would release.""" + if not TITLE_RE.match(title): + print( + f"::error::'{title}' is not a conventional commit title: type(scope)!: description" + ) + return False + level = bump_level(title) + changed = git("diff", "--name-only", f"{base}...HEAD").split() + rows = [] + for name, paths in components.items(): + if not any(f.startswith(p + "/") for f in changed for p in paths): + continue + current = max(released_versions(name), default=None) + before = f"`{tag_name(name, current)}`" if current else "-" + if level is None: + rows.append(f"| `{name}` | {before} | no release | |") + else: + new = next_version(current, level) + rows.append( + f"| `{name}` | {before} | `{tag_name(name, new)}` | `{name}-v{new[0]}` |" + ) + if not rows: + summary(["No action is changed by this pull request: no release."]) + else: + header = [ + "| Action | Current | After merge | Floating tag |", + "| --- | --- | --- | --- |", + ] + summary([f"Releases on merge of `{title}`:", "", *header, *rows]) + return True + + +def move_tag(repo: str, tag: str, sha: str) -> None: + ref = f"repos/{repo}/git/refs/tags/{tag}" + if gh("api", ref, check=False).returncode == 0: + gh("api", "-X", "PATCH", ref, "-f", f"sha={sha}", "-F", "force=true") + else: + gh( + "api", + "-X", + "POST", + f"repos/{repo}/git/refs", + "-f", + f"ref=refs/tags/{tag}", + "-f", + f"sha={sha}", + ) + + +def sync_floating_tags(components: Components, repo: str) -> None: + """Point every -vX at the latest X.y.z release; idempotent, so a rerun repairs a failed move.""" + for name in components: + versions = released_versions(name) + for major in sorted({v[0] for v in versions}): + target = tag_name(name, max(v for v in versions if v[0] == major)) + sha = git("rev-list", "-n", "1", target).strip() + floating = f"{name}-v{major}" + current = subprocess.run( + ["git", "rev-list", "-n", "1", floating], + check=False, + capture_output=True, + text=True, + ).stdout.strip() + if current != sha: + move_tag(repo, floating, sha) + print(f"{floating} -> {target} ({sha[:12]})") + + +def release(components: Components, repo: str) -> None: + """Release every component with releasing commits since its last tag.""" + head = git("rev-parse", "HEAD").strip() + released = [] + for name, paths in components.items(): + current = max(released_versions(name), default=None) + commits = [ + c for c in commits_since(name, paths, current) if bump_level(c[1], c[2]) + ] + if not commits: + continue + level = max( + (bump_level(subject, body) or "patch" for _, subject, body in commits), + key=lambda lv: LEVELS[lv], + ) + version = next_version(current, level) + tag = tag_name(name, version) + major = version[0] + notes = [f"- {subject} ({sha[:12]})" for sha, subject, _ in commits] + notes += [ + "", + "```yaml", + f"uses: centreon/github-actions/{paths[0]}@{name}-v{major}", + "```", + ] + with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as f: + f.write("\n".join(notes) + "\n") + gh( + "release", + "create", + tag, + "--repo", + repo, + "--target", + head, + "--title", + tag, + "--notes-file", + f.name, + "--latest=false", + ) + git("fetch", "--quiet", "origin", f"refs/tags/{tag}:refs/tags/{tag}") + released.append(f"- `{tag}` ({level})") + sync_floating_tags(components, repo) + summary(["Released:", "", *released] if released else ["Nothing to release."]) + + +def main() -> int: + parser = argparse.ArgumentParser( + description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter + ) + sub = parser.add_subparsers(dest="command", required=True) + sub.add_parser("validate") + plan_parser = sub.add_parser("plan") + plan_parser.add_argument("--title", required=True) + plan_parser.add_argument("--base", required=True) + sub.add_parser("release") + args = parser.parse_args() + + components: Components = json.loads(COMPONENTS_FILE.read_text()) + if not validate(components): + return 1 + if args.command == "plan": + return 0 if plan(components, args.title, args.base) else 1 + if args.command == "release": + release(components, os.environ["GITHUB_REPOSITORY"]) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/scripts/ruff.toml b/.github/scripts/ruff.toml new file mode 100644 index 0000000..d45ece4 --- /dev/null +++ b/.github/scripts/ruff.toml @@ -0,0 +1 @@ +src = ["."] diff --git a/.github/scripts/test_release.py b/.github/scripts/test_release.py new file mode 100644 index 0000000..1d23267 --- /dev/null +++ b/.github/scripts/test_release.py @@ -0,0 +1,205 @@ +"""Tests of the per-action release logic, against a throwaway git repository.""" + +import json +import subprocess +from pathlib import Path + +import pytest + +import release + + +def run(*args: str) -> None: + subprocess.run(args, check=True, capture_output=True) + + +@pytest.fixture +def repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: + monkeypatch.chdir(tmp_path) + run("git", "init", "-q", "-b", "main") + run("git", "config", "user.email", "ci@example.com") + run("git", "config", "user.name", "ci") + run("git", "config", "commit.gpgsign", "false") + run("git", "config", "tag.gpgsign", "false") + for path in ("act/one", "act/two", "act/lib"): + Path(path).mkdir(parents=True) + Path("act/one/action.yml").write_text("name: one\n") + Path("act/two/action.yml").write_text("name: two\n") + Path(".github").mkdir() + Path(".github/release-components.json").write_text( + json.dumps({"act-one": ["act/one", "act/lib"], "act-two": ["act/two"]}) + ) + run("git", "add", "-A") + run("git", "commit", "-q", "-m", "chore: init") + return tmp_path + + +def commit(path: str, title: str, body: str = "") -> None: + with open(path, "a") as f: + f.write(title + "\n") + run("git", "add", "-A") + run("git", "commit", "-q", "-m", title, *(["-m", body] if body else [])) + + +@pytest.mark.parametrize( + ("title", "body", "level"), + [ + ("feat(pulp): add an input", "", "minor"), + ("fix(pulp): handle 503", "", "patch"), + ("fix(deps): bump actions/cache", "", "patch"), + ("perf: batch uploads", "", "patch"), + ("feat(pulp)!: rename an input", "", "major"), + ("fix: drop input", "BREAKING CHANGE: input x is removed", "major"), + ("docs: readme", "", None), + ("ci: lint", "", None), + ("Update action.yml", "", None), + ], +) +def test_bump_level(title: str, body: str, level: str) -> None: + assert release.bump_level(title, body) == level + + +def test_next_version() -> None: + assert release.next_version(None, "patch") == (1, 0, 0) + assert release.next_version((1, 2, 3), "patch") == (1, 2, 4) + assert release.next_version((1, 2, 3), "minor") == (1, 3, 0) + assert release.next_version((1, 2, 3), "major") == (2, 0, 0) + + +def test_validate(repo: Path) -> None: + components = json.loads(Path(".github/release-components.json").read_text()) + assert release.validate(components) + Path("act/three").mkdir() + Path("act/three/action.yml").write_text("name: three\n") + assert not release.validate(components) + + +def test_versions_and_commits_follow_component_paths(repo: Path) -> None: + commit("act/one/action.yml", "feat: one") + run("git", "tag", "act-one-v1.0.0") + run("git", "tag", "act-one-v1") + commit("act/lib/x.sh", "fix: shared helper") + commit("act/two/action.yml", "feat: two") + commit("act/one/action.yml", "docs: one") + + assert release.released_versions("act-one") == [(1, 0, 0)] + titles = [ + c[1] + for c in release.commits_since("act-one", ["act/one", "act/lib"], (1, 0, 0)) + ] + assert titles == ["docs: one", "fix: shared helper"] + assert [c[1] for c in release.commits_since("act-two", ["act/two"], None)] == [ + "feat: two", + "chore: init", + ] + + +def test_plan(repo: Path, capsys: pytest.CaptureFixture) -> None: + commit("act/one/action.yml", "feat: one") + run("git", "tag", "act-one-v1.4.2") + run("git", "checkout", "-q", "-b", "pr") + commit("act/lib/x.sh", "wip") + components = json.loads(Path(".github/release-components.json").read_text()) + + assert release.plan(components, "fix(lib): retry", "main") + out = capsys.readouterr().out + assert "`act-one-v1.4.3`" in out and "act-two" not in out + + assert release.plan(components, "feat(lib)!: new layout", "main") + assert "`act-one-v2.0.0` | `act-one-v2`" in capsys.readouterr().out + + assert not release.plan(components, "Update x.sh", "main") + + +FAKE_GH = """#!/usr/bin/env bash +# Stand-in for gh: releases and refs are written to the "origin" remote. +set -euo pipefail +remote=$(git remote get-url origin) +if [[ "$1" == release && "$2" == create ]]; then + tag=$3 + shift 3 + while (($# > 0)); do + [[ "$1" == --target ]] && target=$2 + shift + done + git -c tag.gpgSign=false -c tag.forceSignAnnotated=false --git-dir "$remote" tag "$tag" "$target" + exit 0 +fi +args=("$@") +last=${args[-1]} +case "${args[*]}" in + "api -X PATCH "*) tag=${args[3]##*/tags/}; git -c tag.gpgSign=false -c tag.forceSignAnnotated=false --git-dir "$remote" tag -f "$tag" "${args[5]#sha=}" ;; + "api -X POST "*) tag=${args[5]#ref=refs/tags/}; git -c tag.gpgSign=false -c tag.forceSignAnnotated=false --git-dir "$remote" tag "$tag" "${last#sha=}" ;; + "api "*) git --git-dir "$remote" rev-parse -q --verify "refs/tags/${args[1]##*/tags/}" > /dev/null ;; +esac +""" + + +def remote_tags(remote: Path) -> dict[str, str]: + out = subprocess.run( + [ + "git", + "--git-dir", + str(remote), + "tag", + "--format=%(refname:short) %(objectname:short=7)", + ], + check=True, + capture_output=True, + text=True, + ).stdout + return dict(line.split() for line in out.splitlines()) + + +def short_head() -> str: + return subprocess.run( + ["git", "rev-parse", "--short=7", "HEAD"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + + +def test_release_end_to_end( + repo: Path, + tmp_path_factory: pytest.TempPathFactory, + monkeypatch: pytest.MonkeyPatch, +) -> None: + remote = tmp_path_factory.mktemp("remote") / "origin.git" + run("git", "init", "-q", "--bare", str(remote)) + run("git", "remote", "add", "origin", str(remote)) + bin_dir = tmp_path_factory.mktemp("bin") + (bin_dir / "gh").write_text(FAKE_GH) + (bin_dir / "gh").chmod(0o755) + monkeypatch.setenv("PATH", f"{bin_dir}:{Path('/usr/bin')}:{Path('/bin')}") + components = json.loads(Path(".github/release-components.json").read_text()) + + def publish() -> dict[str, str]: + run("git", "push", "-q", "origin", "main") + release.release(components, "centreon/github-actions") + run("git", "fetch", "-q", "--force", "origin", "+refs/tags/*:refs/tags/*") + return remote_tags(remote) + + commit("act/one/action.yml", "feat(one): add") + commit("act/two/action.yml", "feat(two): add") + first = short_head() + tags = publish() + assert tags == { + k: first + for k in ("act-one-v1.0.0", "act-one-v1", "act-two-v1.0.0", "act-two-v1") + } + + commit("act/lib/x.sh", "fix(lib): retry") + fix = short_head() + tags = publish() + assert tags["act-one-v1.0.1"] == fix and tags["act-one-v1"] == fix + assert tags["act-two-v1"] == first and "act-two-v1.0.1" not in tags + + commit("act/two/action.yml", "feat(two)!: rename input") + major = short_head() + tags = publish() + assert tags["act-two-v2.0.0"] == major and tags["act-two-v2"] == major + assert tags["act-two-v1"] == first + + commit("act/one/action.yml", "docs(one): typo") + assert publish() == tags diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b5799c4..19f4276 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -6,6 +6,8 @@ concurrency: on: pull_request: + # edited: the title decides the release, re-check it when it changes + types: [opened, synchronize, reopened, edited] workflow_dispatch: permissions: {} @@ -13,6 +15,8 @@ permissions: {} env: ACTIONLINT_VERSION: "1.7.12" ACTIONLINT_SHA256: "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8" + PYTEST_VERSION: "9.2.0" + RUFF_VERSION: "0.17.4" jobs: lint: @@ -23,11 +27,27 @@ jobs: - name: Checkout sources uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: + fetch-depth: 0 + fetch-tags: true persist-credentials: false - name: Check actions are self-contained and pinned run: .github/scripts/check-actions.sh + - name: Preview the releases of this pull request + if: github.event_name == 'pull_request' + env: + PR_TITLE: ${{ github.event.pull_request.title }} + BASE_REF: ${{ github.base_ref }} + run: python3 .github/scripts/release.py plan --title "$PR_TITLE" --base "origin/${BASE_REF}" + + - name: Test the release script + run: | + pipx run "ruff==${RUFF_VERSION}" check .github/scripts + pipx run "ruff==${RUFF_VERSION}" format --check .github/scripts + cd .github/scripts + pipx run --spec "pytest==${PYTEST_VERSION}" pytest -q -p no:cacheprovider + - name: Shellcheck run: | mapfile -t scripts < <(git ls-files '*.sh') diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 09b3518..816204b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,7 @@ name: release -# Maintains the release PR (version bump + CHANGELOG.md) from the conventional -# commit titles merged on main; merging it tags vX.Y.Z and publishes the release. +# Releases every action changed on main as -vX.Y.Z (immutable, with a +# GitHub release) and moves its floating -vX tag; see the README. concurrency: group: ${{ github.workflow }} @@ -11,19 +11,19 @@ on: push: branches: - main + workflow_dispatch: permissions: {} jobs: - release-please: + release: runs-on: ubuntu-24.04 # the environment only allows main and holds the release app key environment: release permissions: contents: read steps: - # an app token, not GITHUB_TOKEN: the release PR must trigger the required - # checks, and the app is the only actor allowed to create release tags + # the release app is the only actor allowed to create and move tags - name: Create release app token id: app-token uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 @@ -31,27 +31,15 @@ jobs: client-id: ${{ vars.RELEASE_APP_CLIENT_ID }} private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} permission-contents: write - permission-pull-requests: write - permission-issues: write - - name: Run release-please - id: release - uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5.0.0 + - name: Checkout sources + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - token: ${{ steps.app-token.outputs.token }} - config-file: release-please-config.json - manifest-file: .release-please-manifest.json + fetch-depth: 0 + fetch-tags: true + persist-credentials: false - - name: Summary - if: steps.release.outputs.release_created == 'true' + - name: Release changed actions env: - TAG_NAME: ${{ steps.release.outputs.tag_name }} - SHA: ${{ steps.release.outputs.sha }} - run: | - { - echo "Released \`${TAG_NAME}\` at \`${SHA}\`:" - echo - echo '```yaml' - echo "uses: centreon/github-actions/@${SHA} # ${TAG_NAME}" - echo '```' - } >> "$GITHUB_STEP_SUMMARY" + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: python3 .github/scripts/release.py release diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/.release-please-manifest.json b/.release-please-manifest.json deleted file mode 100644 index 0967ef4..0000000 --- a/.release-please-manifest.json +++ /dev/null @@ -1 +0,0 @@ -{} diff --git a/README.md b/README.md index 7dc75d9..e6d699c 100644 --- a/README.md +++ b/README.md @@ -4,58 +4,57 @@ Shared GitHub Actions used by Centreon CI workflows. ## Usage -Reference an action by its path in this repository, pinned to the full commit SHA of a -release, with the release tag as a comment: +Reference an action by its path and the floating major tag of that action: ```yaml -- uses: centreon/github-actions/pulp/package-delivery@ # v1.2.0 +- uses: centreon/github-actions/pulp/package-delivery@pulp-package-delivery-v1 ``` -- Always pin a SHA, never a tag or a branch: a SHA cannot be moved, and Dependabot - reads the `# vX.Y.Z` comment to propose the next release. -- Use the same release for every action of this repository in a workflow. -- Each maintained branch of a product repository keeps its own pin: an older branch - stays on the release it was validated with until it is bumped on purpose. +- `-v1` always points at the latest `1.y.z` release of that action: fixes and + new optional features reach every branch of every product repository without any + change there, including maintained branches. +- A new major version (`-v2`) is opt-in: move to it by changing the tag, once + per branch, after reading its release notes. +- Pin an exact release only when a branch must not move: + `@pulp-package-delivery-v1.4.2` (immutable) or the SHA of that tag. -Let Dependabot bump the pins in one pull request per product repository: +Keep Dependabot away from these references in the product repositories, their +updates come from the floating tag: ```yaml # .github/dependabot.yml of the product repository - package-ecosystem: github-actions directory: "/" - schedule: - interval: monthly - groups: - centreon-github-actions: - patterns: - - "centreon/github-actions*" + ignore: + - dependency-name: "centreon/github-actions*" ``` -An urgent fix does not wait for Dependabot: bump the SHA and the comment by hand. - ## Versioning -The repository is released as a whole with [Semantic Versioning](https://semver.org/) -tags `vX.Y.Z`, each with an immutable GitHub release and an entry in `CHANGELOG.md`. +Every action has its own [Semantic Versioning](https://semver.org/) version, declared +in `.github/release-components.json` with the paths it is built from: an action that +uses `pulp/lib` is released again when `pulp/lib` changes, the others are not. -| Change to any action | Pull request title | Release | +| Change to an action | Pull request title | Release | | ---------------------------------------------------------------------------------------------- | ---------------------------- | ------- | | Removed or renamed input/output, new required input, changed default or behavior a caller sees | `feat(pulp)!: …` (or `fix!`) | major | -| New action, new optional input or output | `feat(pulp): …` | minor | +| New optional input or output | `feat(pulp): …` | minor | | Bug fix, pinned dependency bump, internal refactoring | `fix(pulp): …` | patch | | Documentation, CI of this repository | `docs: …`, `ci: …` | none | -Pull requests are squash-merged, so their title is the commit that decides the release. +Pull requests are squash-merged with their title as commit title, so the title decides +the release; the `lint` check rejects a non-conventional title and its summary shows +the versions the merge will release. -How a release happens: +Every merge on `main` runs the `release` workflow, which for each action changed since +its last release: -1. Every merge on `main` updates the release pull request (`chore: release X.Y.Z`), - which carries the version bump and the `CHANGELOG.md` entry. -2. Merging the release pull request creates the `vX.Y.Z` tag and its GitHub release. -3. The release summary gives the SHA to pin. +1. creates the immutable `-vX.Y.Z` tag and its GitHub release (commit list and + usage line); +2. moves the floating `-vX` tag to it. -Tags are never moved or deleted, and there is no floating major tag (`v1`): callers pin -SHAs, so a moving tag would bring nothing but risk. +A breaking change starts `-v(X+1)` and leaves `-vX` on its last +release: older majors receive no further fixes. ## Writing an action @@ -65,6 +64,11 @@ through `${{ github.action_path }}`, so every file comes from the pinned commit. actions are pinned to a full SHA with a `# vX.Y.Z` comment. The `lint` workflow checks both rules, plus shellcheck and actionlint. +A new action gets a component named after its path (`pulp/setup-cli` -> +`pulp-setup-cli`) in `.github/release-components.json`, listing its own directory first +and then every directory it uses through `github.action_path`; the `lint` check fails +otherwise. + ## Pulp delivery | Action | Purpose | @@ -95,7 +99,7 @@ deliver-pulp: contents: read id-token: write steps: - - uses: centreon/github-actions/pulp/package-delivery@ # v1.0.0 + - uses: centreon/github-actions/pulp/package-delivery@pulp-package-delivery-v1 with: module_name: centreon-web distrib: el9 @@ -113,12 +117,14 @@ deliver-pulp: Kept outside the code, set by a repository admin: -- **Release app**: a GitHub App with `contents`, `pull-requests` and `issues` write - access to this repository only; its client id in the `RELEASE_APP_CLIENT_ID` variable - and its private key in the `RELEASE_APP_PRIVATE_KEY` secret of the `release` - environment. +- **Release app**: a GitHub App with `contents: write` on this repository only; its + client id in the `RELEASE_APP_CLIENT_ID` variable and its private key in the + `RELEASE_APP_PRIVATE_KEY` secret of the `release` environment. - **`release` environment**: deployment branches limited to `main`. - **`Release tags` ruleset**: the release app is the only bypass actor, so no one else can create, move or delete a tag. -- **Immutable releases**: enabled, so a published release and its tag cannot change. +- **Immutable releases**: enabled, so a published `-vX.Y.Z` release and its tag + cannot change (floating tags carry no release and stay movable by the app). +- **Squash merge commit title**: "Pull request title", so the PR title checked by `lint` + is the commit the release reads. - **Required status check**: `lint` on `main`. diff --git a/release-please-config.json b/release-please-config.json deleted file mode 100644 index fd7ff27..0000000 --- a/release-please-config.json +++ /dev/null @@ -1,49 +0,0 @@ -{ - "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", - "release-type": "simple", - "include-component-in-tag": false, - "include-v-in-tag": true, - "pull-request-title-pattern": "chore: release ${version}", - "changelog-sections": [ - { - "type": "feat", - "section": "Features" - }, - { - "type": "fix", - "section": "Bug Fixes" - }, - { - "type": "perf", - "section": "Performance Improvements" - }, - { - "type": "revert", - "section": "Reverts" - }, - { - "type": "refactor", - "section": "Code Refactoring" - }, - { - "type": "docs", - "section": "Documentation", - "hidden": true - }, - { - "type": "ci", - "section": "Continuous Integration", - "hidden": true - }, - { - "type": "chore", - "section": "Miscellaneous", - "hidden": true - } - ], - "packages": { - ".": { - "initial-version": "1.0.0" - } - } -} From 4a7a7945240781a6815c19f45f82e826b0eb8c30 Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 19 Nov 2026 11:19:19 +0100 Subject: [PATCH 04/15] ci: make the release script executable Refs: MON-215510 --- .github/scripts/release.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .github/scripts/release.py diff --git a/.github/scripts/release.py b/.github/scripts/release.py old mode 100644 new mode 100755 From acbe158068d23040ef79d3c3481627306d00fa7d Mon Sep 17 00:00:00 2001 From: Kevin Duret Date: Thu, 19 Nov 2026 11:42:00 +0100 Subject: [PATCH 05/15] ci: release packages with release-please and floating major tags Version each top-level directory as a package: release-please keeps a release pull request per package, and merging it tags /vX.Y.Z and moves /vX. The Pulp repository layout becomes an internal script, with a default for the delivery type the promotion does not pass. Bats tests cover the layout, the setup guards and the repository scripts. Refs: MON-215510 --- .github/release-components.json | 16 - .github/scripts/check-actions.sh | 51 +++- .github/scripts/check-pr-title.sh | 14 + .github/scripts/release.py | 283 ------------------ .github/scripts/ruff.toml | 1 - .github/scripts/test_release.py | 205 ------------- .github/scripts/tests/check-actions.bats | 65 ++++ .github/scripts/tests/check-pr-title.bats | 20 ++ .github/scripts/tests/update-major-tags.bats | 57 ++++ .github/scripts/update-major-tags.sh | 27 ++ .github/workflows/lint.yml | 26 +- .github/workflows/release.yml | 28 +- .gitignore | 1 - .release-please-manifest.json | 1 + README.md | 158 +++++----- .../parse-distrib.sh | 0 .../repository-properties.sh} | 15 +- pulp/package-delivery/action.yml | 2 +- pulp/promote-to-stable/action.yml | 2 +- pulp/repository-properties/action.yml | 146 --------- pulp/tests/repository-properties.bats | 101 +++++++ pulp/tests/setup-cli.bats | 28 ++ release-please-config.json | 22 ++ 23 files changed, 514 insertions(+), 755 deletions(-) delete mode 100644 .github/release-components.json create mode 100755 .github/scripts/check-pr-title.sh delete mode 100755 .github/scripts/release.py delete mode 100644 .github/scripts/ruff.toml delete mode 100644 .github/scripts/test_release.py create mode 100644 .github/scripts/tests/check-actions.bats create mode 100644 .github/scripts/tests/check-pr-title.bats create mode 100644 .github/scripts/tests/update-major-tags.bats create mode 100755 .github/scripts/update-major-tags.sh delete mode 100644 .gitignore create mode 100644 .release-please-manifest.json rename pulp/{repository-properties => lib}/parse-distrib.sh (100%) rename pulp/{repository-properties/properties.sh => lib/repository-properties.sh} (95%) delete mode 100644 pulp/repository-properties/action.yml create mode 100644 pulp/tests/repository-properties.bats create mode 100644 pulp/tests/setup-cli.bats create mode 100644 release-please-config.json diff --git a/.github/release-components.json b/.github/release-components.json deleted file mode 100644 index e5befb2..0000000 --- a/.github/release-components.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "pulp-package-delivery": [ - "pulp/package-delivery", - "pulp/repository-properties", - "pulp/setup-cli", - "pulp/lib" - ], - "pulp-promote-to-stable": [ - "pulp/promote-to-stable", - "pulp/repository-properties", - "pulp/setup-cli", - "pulp/lib" - ], - "pulp-repository-properties": ["pulp/repository-properties"], - "pulp-setup-cli": ["pulp/setup-cli"] -} diff --git a/.github/scripts/check-actions.sh b/.github/scripts/check-actions.sh index db48a09..45dd9e1 100755 --- a/.github/scripts/check-actions.sh +++ b/.github/scripts/check-actions.sh @@ -1,29 +1,64 @@ #!/usr/bin/env bash -# Repository rules for the shared actions (every action.yml outside .github): +# Repository rules for the shared actions. A package is a top-level directory +# released on its own by release-please; it holds one or several actions. +# - every action belongs to a package declared in release-please-config.json, +# whose component is the directory name (it prefixes the release tags); # - no local "uses: ./...": it resolves in the caller's workspace, not here; +# - no path leaving the package: a change there would not release it; # - every remote action is pinned to a full commit SHA with a version comment. +# Usage: check-actions.sh [repository root] set -euo pipefail +cd "${1:-.}" +CONFIG=release-please-config.json errors=0 -while IFS= read -r -d '' action_file; do +error() { + echo "::error file=$1::$2" + errors=$((errors + 1)) +} + +mapfile -t action_files < <(find . -mindepth 2 -name action.yml -not -path './.*' | sed 's|^\./||' | sort) + +for action_file in "${action_files[@]}"; do + package=${action_file%%/*} + component=$(jq -r --arg p "$package" '.packages[$p].component // empty' "$CONFIG") + if [[ "$component" != "$package" ]]; then + error "$action_file" "package '${package}' must be declared in ${CONFIG} with \"component\": \"${package}\"" + fi + while IFS= read -r line; do number=${line%%:*} ref=${line#*uses:} ref=${ref#"${ref%%[![:space:]]*}"} ref=${ref#[\"\']} if [[ "$ref" == ./* ]]; then - echo "::error file=${action_file},line=${number}::local action '${ref}' resolves in the caller's workspace; call a script through github.action_path instead" - errors=$((errors + 1)) + error "$action_file" "line ${number}: local action '${ref}' resolves in the caller's workspace; call a script through github.action_path instead" elif [[ ! "$ref" =~ ^[^@[:space:]]+@[0-9a-f]{40}[\"\']?[[:space:]]+#[[:space:]]*v?[0-9]+(\.[0-9]+)*$ ]]; then - echo "::error file=${action_file},line=${number}::'${ref}' must be pinned to a full commit SHA followed by a '# vX.Y.Z' comment" - errors=$((errors + 1)) + error "$action_file" "line ${number}: '${ref}' must be pinned to a full commit SHA followed by a '# vX.Y.Z' comment" fi done < <(grep -nE '^[[:space:]-]*uses:' "$action_file" || true) -done < <(find . -path ./.git -prune -o -path ./.github -prune -o -name action.yml -print0) +done + +while IFS= read -r package; do + if [[ -z "$(find "$package" -name action.yml 2> /dev/null)" ]]; then + error "$CONFIG" "package '${package}' holds no action.yml" + continue + fi + while IFS= read -r file; do + directory=${file%/*} + depth=$(awk -F/ '{print NF - 1}' <<< "$directory") + while IFS= read -r match; do + ups=$(grep -o '\.\./' <<< "$match" | wc -l) + if ((ups > depth)); then + error "$file" "'${match}' leaves the package '${package}'" + fi + done < <(grep -oE '(\.\./)+' "$file" | sort -u || true) + done < <(find "$package" -type f \( -name '*.yml' -o -name '*.sh' \) -not -path '*/tests/*') +done < <(jq -r '.packages | keys[]' "$CONFIG") if ((errors > 0)); then echo "${errors} error(s) found" exit 1 fi -echo "All actions are self-contained and SHA-pinned" +echo "${#action_files[@]} action(s) checked: packaged, self-contained and SHA-pinned" diff --git a/.github/scripts/check-pr-title.sh b/.github/scripts/check-pr-title.sh new file mode 100755 index 0000000..8f0a761 --- /dev/null +++ b/.github/scripts/check-pr-title.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# release-please only releases conventional commits, and the pull request +# title becomes the squash commit: reject a title it would silently ignore. +# Usage: check-pr-title.sh "" +set -euo pipefail + +TITLE="${1:-}" +TYPES="feat|fix|perf|refactor|revert|docs|test|ci|chore" + +if [[ ! "$TITLE" =~ ^(${TYPES})(\([a-z0-9._/-]+\))?!?:\ [^[:space:]] ]]; then + echo "::error::'${TITLE}' is not a conventional commit title: <type>(<scope>)!: <description>, type one of ${TYPES//|/, }" + exit 1 +fi +echo "Pull request title is a conventional commit title" diff --git a/.github/scripts/release.py b/.github/scripts/release.py deleted file mode 100755 index 2f2c0a8..0000000 --- a/.github/scripts/release.py +++ /dev/null @@ -1,283 +0,0 @@ -#!/usr/bin/env python3 -"""Per-action releases of this repository. - -Each component of .github/release-components.json is an action released on -its own: immutable "<component>-vX.Y.Z" tags carry a GitHub release, and the -floating "<component>-vX" tag follows the latest X.y.z release. The bump comes -from the conventional commit titles touching the component's paths. - - validate check the component file against the actions - plan --title T --base B preview the releases a pull request would make - release release what changed on HEAD (needs GH_TOKEN) -""" - -import argparse -import json -import os -import re -import subprocess -import sys -import tempfile -from pathlib import Path - -COMPONENTS_FILE = Path(".github/release-components.json") -INITIAL_VERSION = (1, 0, 0) -TITLE_RE = re.compile( - r"^(?P<type>feat|fix|perf|refactor|revert|docs|ci|chore|test|build|style)" - r"(?:\([a-z0-9._-]+\))?(?P<breaking>!)?: \S" -) -PATCH_TYPES = {"fix", "perf", "refactor", "revert"} -LEVELS = {None: 0, "patch": 1, "minor": 2, "major": 3} - -Version = tuple[int, int, int] -Components = dict[str, list[str]] -Commit = tuple[str, str, str] - - -class CommandError(Exception): - """A git or gh command failed; carries its stderr.""" - - -def command(*args: str, check: bool = True) -> subprocess.CompletedProcess: - result = subprocess.run(args, check=False, capture_output=True, text=True) - if check and result.returncode != 0: - raise CommandError(f"{' '.join(args)}: {result.stderr.strip()}") - return result - - -def git(*args: str) -> str: - return command("git", *args).stdout - - -def gh(*args: str, check: bool = True) -> subprocess.CompletedProcess: - return command("gh", *args, check=check) - - -def action_dirs() -> list[str]: - return sorted( - str(p.parent) - for p in Path(".").glob("**/action.yml") - if not p.parts[0].startswith(".") - ) - - -def bump_level(title: str, body: str = "") -> str | None: - """Release level of a commit; None for a non-conventional or non-releasing type.""" - match = TITLE_RE.match(title) - if not match: - return None - if match["breaking"] or re.search(r"^BREAKING[ -]CHANGE: ", body, re.MULTILINE): - return "major" - if match["type"] == "feat": - return "minor" - if match["type"] in PATCH_TYPES: - return "patch" - return None - - -def released_versions(component: str) -> list[Version]: - pattern = re.compile(rf"^{re.escape(component)}-v(\d+)\.(\d+)\.(\d+)$") - tags = git("tag", "--list", f"{component}-v*").split() - return [(int(m[1]), int(m[2]), int(m[3])) for m in map(pattern.match, tags) if m] - - -def next_version(current: Version | None, level: str) -> Version: - if current is None: - return INITIAL_VERSION - major, minor, patch = current - if level == "major": - return (major + 1, 0, 0) - if level == "minor": - return (major, minor + 1, 0) - return (major, minor, patch + 1) - - -def tag_name(component: str, version: Version) -> str: - return f"{component}-v{'.'.join(map(str, version))}" - - -def commits_since( - component: str, paths: list[str], current: Version | None -) -> list[Commit]: - rev = "HEAD" if current is None else f"{tag_name(component, current)}..HEAD" - out = git("log", "--format=%H%x1f%s%x1f%b%x1e", rev, "--", *paths) - commits = [] - for record in filter(None, (r.strip("\n") for r in out.split("\x1e"))): - sha, subject, body = record.split("\x1f") - commits.append((sha, subject, body)) - return commits - - -def validate(components: Components) -> bool: - """Every action is the first path of the component named after it, and every path exists.""" - errors = [] - dirs = action_dirs() - for directory in dirs: - name = directory.replace("/", "-") - if components.get(name, [None])[0] != directory: - errors.append( - f"action {directory} needs a '{name}' component whose first path is {directory}" - ) - for name, paths in components.items(): - errors += [ - f"component {name}: path {p} does not exist" - for p in paths - if not Path(p).is_dir() - ] - if not paths or paths[0] not in dirs: - errors.append( - f"component {name}: its first path must be an action directory" - ) - for error in errors: - print(f"::error file={COMPONENTS_FILE}::{error}") - return not errors - - -def summary(lines: list[str]) -> None: - text = "\n".join(lines) + "\n" - print(text) - if os.environ.get("GITHUB_STEP_SUMMARY"): - with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as f: - f.write(text) - - -def plan(components: Components, title: str, base: str) -> bool: - """Show what merging a pull request with this title would release.""" - if not TITLE_RE.match(title): - print( - f"::error::'{title}' is not a conventional commit title: type(scope)!: description" - ) - return False - level = bump_level(title) - changed = git("diff", "--name-only", f"{base}...HEAD").split() - rows = [] - for name, paths in components.items(): - if not any(f.startswith(p + "/") for f in changed for p in paths): - continue - current = max(released_versions(name), default=None) - before = f"`{tag_name(name, current)}`" if current else "-" - if level is None: - rows.append(f"| `{name}` | {before} | no release | |") - else: - new = next_version(current, level) - rows.append( - f"| `{name}` | {before} | `{tag_name(name, new)}` | `{name}-v{new[0]}` |" - ) - if not rows: - summary(["No action is changed by this pull request: no release."]) - else: - header = [ - "| Action | Current | After merge | Floating tag |", - "| --- | --- | --- | --- |", - ] - summary([f"Releases on merge of `{title}`:", "", *header, *rows]) - return True - - -def move_tag(repo: str, tag: str, sha: str) -> None: - ref = f"repos/{repo}/git/refs/tags/{tag}" - if gh("api", ref, check=False).returncode == 0: - gh("api", "-X", "PATCH", ref, "-f", f"sha={sha}", "-F", "force=true") - else: - gh( - "api", - "-X", - "POST", - f"repos/{repo}/git/refs", - "-f", - f"ref=refs/tags/{tag}", - "-f", - f"sha={sha}", - ) - - -def sync_floating_tags(components: Components, repo: str) -> None: - """Point every <component>-vX at the latest X.y.z release; idempotent, so a rerun repairs a failed move.""" - for name in components: - versions = released_versions(name) - for major in sorted({v[0] for v in versions}): - target = tag_name(name, max(v for v in versions if v[0] == major)) - sha = git("rev-list", "-n", "1", target).strip() - floating = f"{name}-v{major}" - current = subprocess.run( - ["git", "rev-list", "-n", "1", floating], - check=False, - capture_output=True, - text=True, - ).stdout.strip() - if current != sha: - move_tag(repo, floating, sha) - print(f"{floating} -> {target} ({sha[:12]})") - - -def release(components: Components, repo: str) -> None: - """Release every component with releasing commits since its last tag.""" - head = git("rev-parse", "HEAD").strip() - released = [] - for name, paths in components.items(): - current = max(released_versions(name), default=None) - commits = [ - c for c in commits_since(name, paths, current) if bump_level(c[1], c[2]) - ] - if not commits: - continue - level = max( - (bump_level(subject, body) or "patch" for _, subject, body in commits), - key=lambda lv: LEVELS[lv], - ) - version = next_version(current, level) - tag = tag_name(name, version) - major = version[0] - notes = [f"- {subject} ({sha[:12]})" for sha, subject, _ in commits] - notes += [ - "", - "```yaml", - f"uses: centreon/github-actions/{paths[0]}@{name}-v{major}", - "```", - ] - with tempfile.NamedTemporaryFile("w", suffix=".md", delete=False) as f: - f.write("\n".join(notes) + "\n") - gh( - "release", - "create", - tag, - "--repo", - repo, - "--target", - head, - "--title", - tag, - "--notes-file", - f.name, - "--latest=false", - ) - git("fetch", "--quiet", "origin", f"refs/tags/{tag}:refs/tags/{tag}") - released.append(f"- `{tag}` ({level})") - sync_floating_tags(components, repo) - summary(["Released:", "", *released] if released else ["Nothing to release."]) - - -def main() -> int: - parser = argparse.ArgumentParser( - description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter - ) - sub = parser.add_subparsers(dest="command", required=True) - sub.add_parser("validate") - plan_parser = sub.add_parser("plan") - plan_parser.add_argument("--title", required=True) - plan_parser.add_argument("--base", required=True) - sub.add_parser("release") - args = parser.parse_args() - - components: Components = json.loads(COMPONENTS_FILE.read_text()) - if not validate(components): - return 1 - if args.command == "plan": - return 0 if plan(components, args.title, args.base) else 1 - if args.command == "release": - release(components, os.environ["GITHUB_REPOSITORY"]) - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/.github/scripts/ruff.toml b/.github/scripts/ruff.toml deleted file mode 100644 index d45ece4..0000000 --- a/.github/scripts/ruff.toml +++ /dev/null @@ -1 +0,0 @@ -src = ["."] diff --git a/.github/scripts/test_release.py b/.github/scripts/test_release.py deleted file mode 100644 index 1d23267..0000000 --- a/.github/scripts/test_release.py +++ /dev/null @@ -1,205 +0,0 @@ -"""Tests of the per-action release logic, against a throwaway git repository.""" - -import json -import subprocess -from pathlib import Path - -import pytest - -import release - - -def run(*args: str) -> None: - subprocess.run(args, check=True, capture_output=True) - - -@pytest.fixture -def repo(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path: - monkeypatch.chdir(tmp_path) - run("git", "init", "-q", "-b", "main") - run("git", "config", "user.email", "ci@example.com") - run("git", "config", "user.name", "ci") - run("git", "config", "commit.gpgsign", "false") - run("git", "config", "tag.gpgsign", "false") - for path in ("act/one", "act/two", "act/lib"): - Path(path).mkdir(parents=True) - Path("act/one/action.yml").write_text("name: one\n") - Path("act/two/action.yml").write_text("name: two\n") - Path(".github").mkdir() - Path(".github/release-components.json").write_text( - json.dumps({"act-one": ["act/one", "act/lib"], "act-two": ["act/two"]}) - ) - run("git", "add", "-A") - run("git", "commit", "-q", "-m", "chore: init") - return tmp_path - - -def commit(path: str, title: str, body: str = "") -> None: - with open(path, "a") as f: - f.write(title + "\n") - run("git", "add", "-A") - run("git", "commit", "-q", "-m", title, *(["-m", body] if body else [])) - - -@pytest.mark.parametrize( - ("title", "body", "level"), - [ - ("feat(pulp): add an input", "", "minor"), - ("fix(pulp): handle 503", "", "patch"), - ("fix(deps): bump actions/cache", "", "patch"), - ("perf: batch uploads", "", "patch"), - ("feat(pulp)!: rename an input", "", "major"), - ("fix: drop input", "BREAKING CHANGE: input x is removed", "major"), - ("docs: readme", "", None), - ("ci: lint", "", None), - ("Update action.yml", "", None), - ], -) -def test_bump_level(title: str, body: str, level: str) -> None: - assert release.bump_level(title, body) == level - - -def test_next_version() -> None: - assert release.next_version(None, "patch") == (1, 0, 0) - assert release.next_version((1, 2, 3), "patch") == (1, 2, 4) - assert release.next_version((1, 2, 3), "minor") == (1, 3, 0) - assert release.next_version((1, 2, 3), "major") == (2, 0, 0) - - -def test_validate(repo: Path) -> None: - components = json.loads(Path(".github/release-components.json").read_text()) - assert release.validate(components) - Path("act/three").mkdir() - Path("act/three/action.yml").write_text("name: three\n") - assert not release.validate(components) - - -def test_versions_and_commits_follow_component_paths(repo: Path) -> None: - commit("act/one/action.yml", "feat: one") - run("git", "tag", "act-one-v1.0.0") - run("git", "tag", "act-one-v1") - commit("act/lib/x.sh", "fix: shared helper") - commit("act/two/action.yml", "feat: two") - commit("act/one/action.yml", "docs: one") - - assert release.released_versions("act-one") == [(1, 0, 0)] - titles = [ - c[1] - for c in release.commits_since("act-one", ["act/one", "act/lib"], (1, 0, 0)) - ] - assert titles == ["docs: one", "fix: shared helper"] - assert [c[1] for c in release.commits_since("act-two", ["act/two"], None)] == [ - "feat: two", - "chore: init", - ] - - -def test_plan(repo: Path, capsys: pytest.CaptureFixture) -> None: - commit("act/one/action.yml", "feat: one") - run("git", "tag", "act-one-v1.4.2") - run("git", "checkout", "-q", "-b", "pr") - commit("act/lib/x.sh", "wip") - components = json.loads(Path(".github/release-components.json").read_text()) - - assert release.plan(components, "fix(lib): retry", "main") - out = capsys.readouterr().out - assert "`act-one-v1.4.3`" in out and "act-two" not in out - - assert release.plan(components, "feat(lib)!: new layout", "main") - assert "`act-one-v2.0.0` | `act-one-v2`" in capsys.readouterr().out - - assert not release.plan(components, "Update x.sh", "main") - - -FAKE_GH = """#!/usr/bin/env bash -# Stand-in for gh: releases and refs are written to the "origin" remote. -set -euo pipefail -remote=$(git remote get-url origin) -if [[ "$1" == release && "$2" == create ]]; then - tag=$3 - shift 3 - while (($# > 0)); do - [[ "$1" == --target ]] && target=$2 - shift - done - git -c tag.gpgSign=false -c tag.forceSignAnnotated=false --git-dir "$remote" tag "$tag" "$target" - exit 0 -fi -args=("$@") -last=${args[-1]} -case "${args[*]}" in - "api -X PATCH "*) tag=${args[3]##*/tags/}; git -c tag.gpgSign=false -c tag.forceSignAnnotated=false --git-dir "$remote" tag -f "$tag" "${args[5]#sha=}" ;; - "api -X POST "*) tag=${args[5]#ref=refs/tags/}; git -c tag.gpgSign=false -c tag.forceSignAnnotated=false --git-dir "$remote" tag "$tag" "${last#sha=}" ;; - "api "*) git --git-dir "$remote" rev-parse -q --verify "refs/tags/${args[1]##*/tags/}" > /dev/null ;; -esac -""" - - -def remote_tags(remote: Path) -> dict[str, str]: - out = subprocess.run( - [ - "git", - "--git-dir", - str(remote), - "tag", - "--format=%(refname:short) %(objectname:short=7)", - ], - check=True, - capture_output=True, - text=True, - ).stdout - return dict(line.split() for line in out.splitlines()) - - -def short_head() -> str: - return subprocess.run( - ["git", "rev-parse", "--short=7", "HEAD"], - check=True, - capture_output=True, - text=True, - ).stdout.strip() - - -def test_release_end_to_end( - repo: Path, - tmp_path_factory: pytest.TempPathFactory, - monkeypatch: pytest.MonkeyPatch, -) -> None: - remote = tmp_path_factory.mktemp("remote") / "origin.git" - run("git", "init", "-q", "--bare", str(remote)) - run("git", "remote", "add", "origin", str(remote)) - bin_dir = tmp_path_factory.mktemp("bin") - (bin_dir / "gh").write_text(FAKE_GH) - (bin_dir / "gh").chmod(0o755) - monkeypatch.setenv("PATH", f"{bin_dir}:{Path('/usr/bin')}:{Path('/bin')}") - components = json.loads(Path(".github/release-components.json").read_text()) - - def publish() -> dict[str, str]: - run("git", "push", "-q", "origin", "main") - release.release(components, "centreon/github-actions") - run("git", "fetch", "-q", "--force", "origin", "+refs/tags/*:refs/tags/*") - return remote_tags(remote) - - commit("act/one/action.yml", "feat(one): add") - commit("act/two/action.yml", "feat(two): add") - first = short_head() - tags = publish() - assert tags == { - k: first - for k in ("act-one-v1.0.0", "act-one-v1", "act-two-v1.0.0", "act-two-v1") - } - - commit("act/lib/x.sh", "fix(lib): retry") - fix = short_head() - tags = publish() - assert tags["act-one-v1.0.1"] == fix and tags["act-one-v1"] == fix - assert tags["act-two-v1"] == first and "act-two-v1.0.1" not in tags - - commit("act/two/action.yml", "feat(two)!: rename input") - major = short_head() - tags = publish() - assert tags["act-two-v2.0.0"] == major and tags["act-two-v2"] == major - assert tags["act-two-v1"] == first - - commit("act/one/action.yml", "docs(one): typo") - assert publish() == tags diff --git a/.github/scripts/tests/check-actions.bats b/.github/scripts/tests/check-actions.bats new file mode 100644 index 0000000..5ab41e2 --- /dev/null +++ b/.github/scripts/tests/check-actions.bats @@ -0,0 +1,65 @@ +#!/usr/bin/env bats + +setup() { + SCRIPT="${BATS_TEST_DIRNAME}/../check-actions.sh" + REPO="${BATS_TEST_TMPDIR}/repo" + mkdir -p "${REPO}/tool/run" "${REPO}/tool/lib" + echo '{"packages": {"tool": {"component": "tool"}}}' > "${REPO}/release-please-config.json" + action 'uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5' \ + 'run: ${{ github.action_path }}/../lib/run.sh' +} + +# write tool/run/action.yml with the given step lines +action() { + { + printf 'runs:\n using: composite\n steps:\n' + printf ' - %s\n' "$@" + } > "${REPO}/tool/run/action.yml" +} + +@test "a packaged, self-contained, pinned action passes" { + run "$SCRIPT" "$REPO" + [ "$status" -eq 0 ] +} + +@test "an action outside any declared package fails" { + mkdir -p "${REPO}/other" + cp "${REPO}/tool/run/action.yml" "${REPO}/other/action.yml" + run "$SCRIPT" "$REPO" + [ "$status" -ne 0 ] + [[ "$output" == *"package 'other' must be declared"* ]] +} + +@test "a component that differs from the directory fails" { + echo '{"packages": {"tool": {"component": "tools"}}}' > "${REPO}/release-please-config.json" + run "$SCRIPT" "$REPO" + [ "$status" -ne 0 ] +} + +@test "a local uses fails" { + action 'uses: ./.github/actions/foo' + run "$SCRIPT" "$REPO" + [ "$status" -ne 0 ] + [[ "$output" == *"resolves in the caller's workspace"* ]] +} + +@test "a tag-pinned action fails" { + action 'uses: actions/checkout@v4' + run "$SCRIPT" "$REPO" + [ "$status" -ne 0 ] + [[ "$output" == *"must be pinned to a full commit SHA"* ]] +} + +@test "a path leaving the package fails" { + action 'run: ${{ github.action_path }}/../../shared/run.sh' + run "$SCRIPT" "$REPO" + [ "$status" -ne 0 ] + [[ "$output" == *"leaves the package 'tool'"* ]] +} + +@test "a declared package without action fails" { + echo '{"packages": {"tool": {"component": "tool"}, "ghost": {"component": "ghost"}}}' > "${REPO}/release-please-config.json" + run "$SCRIPT" "$REPO" + [ "$status" -ne 0 ] + [[ "$output" == *"package 'ghost' holds no action.yml"* ]] +} diff --git a/.github/scripts/tests/check-pr-title.bats b/.github/scripts/tests/check-pr-title.bats new file mode 100644 index 0000000..d06dcf0 --- /dev/null +++ b/.github/scripts/tests/check-pr-title.bats @@ -0,0 +1,20 @@ +#!/usr/bin/env bats + +setup() { + SCRIPT="${BATS_TEST_DIRNAME}/../check-pr-title.sh" +} + +@test "conventional titles are accepted" { + for title in "feat(pulp): add an input" "fix: handle 503" "feat(pulp)!: rename an input" \ + "chore(main): release pulp 1.2.0" "fix(deps): bump actions/cache"; do + run "$SCRIPT" "$title" + [ "$status" -eq 0 ] || { echo "rejected: ${title}"; return 1; } + done +} + +@test "other titles are rejected" { + for title in "Update action.yml" "feat add an input" "feature(pulp): x" "fix(pulp):missing space" ""; do + run "$SCRIPT" "$title" + [ "$status" -ne 0 ] || { echo "accepted: ${title}"; return 1; } + done +} diff --git a/.github/scripts/tests/update-major-tags.bats b/.github/scripts/tests/update-major-tags.bats new file mode 100644 index 0000000..106d41e --- /dev/null +++ b/.github/scripts/tests/update-major-tags.bats @@ -0,0 +1,57 @@ +#!/usr/bin/env bats + +SHA=0123456789abcdef0123456789abcdef01234567 + +setup() { + SCRIPT="${BATS_TEST_DIRNAME}/../update-major-tags.sh" + export GITHUB_REPOSITORY=centreon/github-actions + export GH_CALLS="${BATS_TEST_TMPDIR}/calls" + : > "$GH_CALLS" + # gh stand-in: logs its arguments; the tag lookup succeeds when TAG_EXISTS=1 + mkdir -p "${BATS_TEST_TMPDIR}/bin" + cat > "${BATS_TEST_TMPDIR}/bin/gh" <<'STUB' +#!/usr/bin/env bash +echo "$*" >> "$GH_CALLS" +if [[ "$2" == repos/*/git/ref/tags/* ]]; then + [[ "${TAG_EXISTS:-0}" == 1 ]] +fi +STUB + chmod +x "${BATS_TEST_TMPDIR}/bin/gh" + PATH="${BATS_TEST_TMPDIR}/bin:${PATH}" +} + +releases() { + jq -cn --arg tag "$1" --arg sha "$SHA" \ + '{paths_released: "[\"pulp\"]", "pulp--tag_name": $tag, "pulp--sha": $sha}' +} + +@test "first release of a major creates its floating tag" { + RELEASES=$(releases pulp/v1.0.0) run "$SCRIPT" + [ "$status" -eq 0 ] + grep -qF -- "-X POST repos/centreon/github-actions/git/refs -f ref=refs/tags/pulp/v1 -f sha=${SHA}" "$GH_CALLS" +} + +@test "later release moves the floating tag" { + RELEASES=$(releases pulp/v1.4.2) TAG_EXISTS=1 run "$SCRIPT" + [ "$status" -eq 0 ] + grep -qF -- "-X PATCH repos/centreon/github-actions/git/refs/tags/pulp/v1 -f sha=${SHA} -F force=true" "$GH_CALLS" +} + +@test "new major gets its own floating tag" { + RELEASES=$(releases pulp/v2.0.0) run "$SCRIPT" + [ "$status" -eq 0 ] + grep -qF "ref=refs/tags/pulp/v2 " "$GH_CALLS" + ! grep -qF "tags/pulp/v1" "$GH_CALLS" +} + +@test "nothing released, nothing moved" { + RELEASES='{"releases_created": "false"}' run "$SCRIPT" + [ "$status" -eq 0 ] + [ ! -s "$GH_CALLS" ] +} + +@test "unexpected tag format fails without touching tags" { + RELEASES=$(releases pulp-v1.0.0) run "$SCRIPT" + [ "$status" -ne 0 ] + [ ! -s "$GH_CALLS" ] +} diff --git a/.github/scripts/update-major-tags.sh b/.github/scripts/update-major-tags.sh new file mode 100755 index 0000000..a89b2f5 --- /dev/null +++ b/.github/scripts/update-major-tags.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Point the floating major tag of every package release-please just released +# (<package>/vX) at its new release (<package>/vX.Y.Z). +# Inputs (env): RELEASES (JSON outputs of release-please-action), +# GITHUB_REPOSITORY, GH_TOKEN. +set -euo pipefail + +paths=$(jq -r '(.paths_released // "[]") | fromjson | .[]' <<< "$RELEASES") + +for path in $paths; do + tag=$(jq -r --arg key "${path}--tag_name" '.[$key] // empty' <<< "$RELEASES") + sha=$(jq -r --arg key "${path}--sha" '.[$key] // empty' <<< "$RELEASES") + if [[ ! "$sha" =~ ^[0-9a-f]{40}$ || ! "$tag" =~ ^(.+)/v([0-9]+)\.[0-9]+\.[0-9]+$ ]]; then + echo "::error::unexpected release of ${path}: tag '${tag}', sha '${sha}'" + exit 1 + fi + # BASH_REMATCH holds the tag match: it is the last regex evaluated above + major_tag="${BASH_REMATCH[1]}/v${BASH_REMATCH[2]}" + + if gh api "repos/${GITHUB_REPOSITORY}/git/ref/tags/${major_tag}" > /dev/null 2>&1; then + gh api -X PATCH "repos/${GITHUB_REPOSITORY}/git/refs/tags/${major_tag}" -f "sha=${sha}" -F force=true > /dev/null + else + gh api -X POST "repos/${GITHUB_REPOSITORY}/git/refs" -f "ref=refs/tags/${major_tag}" -f "sha=${sha}" > /dev/null + fi + echo "${major_tag} -> ${tag} (${sha})" + echo "- \`${major_tag}\` now points at \`${tag}\`" >> "${GITHUB_STEP_SUMMARY:-/dev/null}" +done diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 19f4276..05cac6f 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -15,8 +15,7 @@ permissions: {} env: ACTIONLINT_VERSION: "1.7.12" ACTIONLINT_SHA256: "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8" - PYTEST_VERSION: "9.2.0" - RUFF_VERSION: "0.17.4" + BATS_VERSION: "1.13.0" jobs: lint: @@ -27,26 +26,16 @@ jobs: - name: Checkout sources uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - fetch-depth: 0 - fetch-tags: true persist-credentials: false - - name: Check actions are self-contained and pinned - run: .github/scripts/check-actions.sh - - - name: Preview the releases of this pull request + - name: Check the pull request title if: github.event_name == 'pull_request' env: PR_TITLE: ${{ github.event.pull_request.title }} - BASE_REF: ${{ github.base_ref }} - run: python3 .github/scripts/release.py plan --title "$PR_TITLE" --base "origin/${BASE_REF}" + run: .github/scripts/check-pr-title.sh "$PR_TITLE" - - name: Test the release script - run: | - pipx run "ruff==${RUFF_VERSION}" check .github/scripts - pipx run "ruff==${RUFF_VERSION}" format --check .github/scripts - cd .github/scripts - pipx run --spec "pytest==${PYTEST_VERSION}" pytest -q -p no:cacheprovider + - name: Check actions are packaged, self-contained and pinned + run: .github/scripts/check-actions.sh - name: Shellcheck run: | @@ -54,6 +43,11 @@ jobs: shellcheck --version shellcheck -x -P SCRIPTDIR -S warning "${scripts[@]}" + - name: Bats tests + run: | + mapfile -t test_dirs < <(git ls-files '*.bats' | xargs -n 1 dirname | sort -u) + npx --yes "bats@${BATS_VERSION}" "${test_dirs[@]}" + - name: Actionlint run: | archive="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 816204b..f0647f6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,7 +1,8 @@ name: release -# Releases every action changed on main as <action>-vX.Y.Z (immutable, with a -# GitHub release) and moves its floating <action>-vX tag; see the README. +# release-please keeps one release pull request per package (version bump and +# CHANGELOG.md); merging it creates the <package>/vX.Y.Z tag and its GitHub +# release, then the floating <package>/vX tag is moved to it. See the README. concurrency: group: ${{ github.workflow }} @@ -11,7 +12,6 @@ on: push: branches: - main - workflow_dispatch: permissions: {} @@ -23,7 +23,8 @@ jobs: permissions: contents: read steps: - # the release app is the only actor allowed to create and move tags + # an app token, not GITHUB_TOKEN: the release pull requests must trigger + # the required checks, and the app is the only actor allowed to push tags - name: Create release app token id: app-token uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 @@ -31,15 +32,26 @@ jobs: client-id: ${{ vars.RELEASE_APP_CLIENT_ID }} private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} permission-contents: write + permission-pull-requests: write + permission-issues: write + + - name: Run release-please + id: release + uses: googleapis/release-please-action@45996ed1f6d02564a971a2fa1b5860e934307cf7 # v5.0.0 + with: + token: ${{ steps.app-token.outputs.token }} + config-file: release-please-config.json + manifest-file: .release-please-manifest.json - name: Checkout sources + if: steps.release.outputs.releases_created == 'true' uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: - fetch-depth: 0 - fetch-tags: true persist-credentials: false - - name: Release changed actions + - name: Move floating major tags + if: steps.release.outputs.releases_created == 'true' env: + RELEASES: ${{ toJSON(steps.release.outputs) }} GH_TOKEN: ${{ steps.app-token.outputs.token }} - run: python3 .github/scripts/release.py release + run: .github/scripts/update-major-tags.sh diff --git a/.gitignore b/.gitignore deleted file mode 100644 index c18dd8d..0000000 --- a/.gitignore +++ /dev/null @@ -1 +0,0 @@ -__pycache__/ diff --git a/.release-please-manifest.json b/.release-please-manifest.json new file mode 100644 index 0000000..0967ef4 --- /dev/null +++ b/.release-please-manifest.json @@ -0,0 +1 @@ +{} diff --git a/README.md b/README.md index e6d699c..7df195b 100644 --- a/README.md +++ b/README.md @@ -2,82 +2,105 @@ Shared GitHub Actions used by Centreon CI workflows. -## Usage - -Reference an action by its path and the floating major tag of that action: +The repository is a set of **packages**: a package is a top-level directory, versioned +and released on its own, that holds one action or several actions built on the same +code. -```yaml -- uses: centreon/github-actions/pulp/package-delivery@pulp-package-delivery-v1 -``` +| Package | Actions | +| --------------- | ------------------------------------------------------------------- | +| [`pulp`](#pulp) | `pulp/package-delivery`, `pulp/promote-to-stable`, `pulp/setup-cli` | -- `<action>-v1` always points at the latest `1.y.z` release of that action: fixes and - new optional features reach every branch of every product repository without any - change there, including maintained branches. -- A new major version (`<action>-v2`) is opt-in: move to it by changing the tag, once - per branch, after reading its release notes. -- Pin an exact release only when a branch must not move: - `@pulp-package-delivery-v1.4.2` (immutable) or the SHA of that tag. +## Usage -Keep Dependabot away from these references in the product repositories, their -updates come from the floating tag: +Reference an action by its path and the major tag of its package: ```yaml -# .github/dependabot.yml of the product repository -- package-ecosystem: github-actions - directory: "/" - ignore: - - dependency-name: "centreon/github-actions*" +- uses: centreon/github-actions/pulp/package-delivery@pulp/v1 ``` -## Versioning - -Every action has its own [Semantic Versioning](https://semver.org/) version, declared -in `.github/release-components.json` with the paths it is built from: an action that -uses `pulp/lib` is released again when `pulp/lib` changes, the others are not. +- `<package>/v1` always points at the latest `1.y.z` release of the package: fixes and + new optional features reach every branch of every repository without any change + there. +- A new major (`<package>/v2`) is opt-in: change the tag, branch by branch, after + reading its release notes. The previous major stays where it was and gets no + further release. +- To freeze a branch, reference an exact release: `@pulp/v1.4.2` (never moved) or its + commit SHA. -| Change to an action | Pull request title | Release | -| ---------------------------------------------------------------------------------------------- | ---------------------------- | ------- | -| Removed or renamed input/output, new required input, changed default or behavior a caller sees | `feat(pulp)!: …` (or `fix!`) | major | -| New optional input or output | `feat(pulp): …` | minor | -| Bug fix, pinned dependency bump, internal refactoring | `fix(pulp): …` | patch | -| Documentation, CI of this repository | `docs: …`, `ci: …` | none | +In the calling repository: -Pull requests are squash-merged with their title as commit title, so the title decides -the release; the `lint` check rejects a non-conventional title and its summary shows -the versions the merge will release. +- allow `centreon/github-actions` in the SHA pinning check, as for + `centreon/security-tools`; +- make Dependabot ignore it, the major tag already brings the updates: -Every merge on `main` runs the `release` workflow, which for each action changed since -its last release: + ```yaml + # .github/dependabot.yml + - package-ecosystem: github-actions + directory: "/" + ignore: + - dependency-name: "centreon/github-actions*" + ``` -1. creates the immutable `<action>-vX.Y.Z` tag and its GitHub release (commit list and - usage line); -2. moves the floating `<action>-vX` tag to it. +## Versioning and releases -A breaking change starts `<action>-v(X+1)` and leaves `<action>-vX` on its last -release: older majors receive no further fixes. +Each package follows [Semantic Versioning](https://semver.org/). Releases are made by +[release-please](https://github.com/googleapis/release-please) from the titles of the +pull requests merged on `main` (squash merge, so the title is the commit): -## Writing an action +| Change to a package | Pull request title | Release | +| ---------------------------------------------------------------------------------------------- | ---------------------------- | ------- | +| Removed or renamed input/output, new required input, changed default or behavior a caller sees | `feat(pulp)!: …` (or `fix!`) | major | +| New action, new optional input or output | `feat(pulp): …` | minor | +| Bug fix, pinned dependency bump, performance | `fix(pulp): …`, `perf: …` | patch | +| Refactoring, documentation, tests, CI | `refactor:`, `docs:`, `ci:`… | none | + +A commit counts for the packages whose directory it changes. + +1. A pull request with a releasing title is merged on `main`. +2. release-please opens, or updates, the release pull request of each changed package + (`chore(main): release pulp 1.2.0`): version bump and `<package>/CHANGELOG.md`. + Several changes can accumulate in it. +3. Merging the release pull request creates the `<package>/v1.2.0` tag and its GitHub + release, then moves the floating `<package>/v1` tag to it: callers get the release + on their next run. + +Nothing reaches the callers before step 3: the release pull request is the moment to +decide that `main` is good to ship. + +## Adding or changing an action + +- Put a new action in an existing package when it shares code with it, otherwise create + a package: a top-level directory, declared in `release-please-config.json` with its + directory name as `component`. +- An action is self-contained in its package: no `uses: ./...` (that path resolves in + the caller's workspace) and no path leaving the package. Shared logic lives in + scripts called through `${{ github.action_path }}`, so all files come from the same + release. +- Remote actions are pinned to a full commit SHA with a `# vX.Y.Z` comment. +- Shell scripts follow shellcheck; test them with + [bats](https://github.com/bats-core/bats-core) in `<package>/tests/`. + +The `lint` workflow enforces all of this on every pull request, with the title format. +Run it locally with: + +```shell +.github/scripts/check-actions.sh +shellcheck -x -P SCRIPTDIR $(git ls-files '*.sh') +npx bats pulp/tests .github/scripts/tests +``` -Each action is self-contained: it never calls a sibling action through `uses: ./...` -(that path resolves in the caller's workspace). Shared logic lives in scripts called -through `${{ github.action_path }}`, so every file comes from the pinned commit. Remote -actions are pinned to a full SHA with a `# vX.Y.Z` comment. The `lint` workflow checks -both rules, plus shellcheck and actionlint. +## pulp -A new action gets a component named after its path (`pulp/setup-cli` -> -`pulp-setup-cli`) in `.github/release-components.json`, listing its own directory first -and then every directory it uses through `github.action_path`; the `lint` check fails -otherwise. +Delivery of rpm/deb packages to the Pulp repositories. -## Pulp delivery +| Action | Purpose | +| ------------------------ | --------------------------------------------------------------------------------------------- | +| `pulp/package-delivery` | Deliver rpm/deb packages from the actions cache to an unstable, testing or feature repository | +| `pulp/promote-to-stable` | Promote testing packages to the stable repository | +| `pulp/setup-cli` | Install pulp-cli and authenticate with a GitHub OIDC token | -| Action | Purpose | -| ---------------------------- | --------------------------------------------------------------------------------------------- | -| `pulp/package-delivery` | Deliver rpm/deb packages from the actions cache to an unstable, testing or feature repository | -| `pulp/promote-to-stable` | Promote testing packages to the stable repository | -| `pulp/repository-properties` | Compute repository names, base paths, suites and domains of a delivery | -| `pulp/setup-cli` | Install pulp-cli and authenticate with a GitHub OIDC token | -| `pulp/lib` | Shell helpers (REST api, manifest, post-delivery checks), not an action | +`pulp/lib` holds their shared scripts (repository layout, REST api, manifest, +post-delivery checks); it is not an action. Requirements of the calling job: @@ -90,8 +113,6 @@ Requirements of the calling job: `repository_name` selects the repository family: `standard`, `standard-internal`, `business`, `business-internal`, `plugins` or `connectors`. -Example: - ```yaml deliver-pulp: runs-on: centreon-ubuntu-26.04-small @@ -99,7 +120,7 @@ deliver-pulp: contents: read id-token: write steps: - - uses: centreon/github-actions/pulp/package-delivery@pulp-package-delivery-v1 + - uses: centreon/github-actions/pulp/package-delivery@pulp/v1 with: module_name: centreon-web distrib: el9 @@ -117,14 +138,15 @@ deliver-pulp: Kept outside the code, set by a repository admin: -- **Release app**: a GitHub App with `contents: write` on this repository only; its - client id in the `RELEASE_APP_CLIENT_ID` variable and its private key in the - `RELEASE_APP_PRIVATE_KEY` secret of the `release` environment. +- **Release app**: a GitHub App installed on this repository only, with `contents`, + `pull-requests` and `issues` write access; its client id in the + `RELEASE_APP_CLIENT_ID` variable and its private key in the `RELEASE_APP_PRIVATE_KEY` + secret of the `release` environment. - **`release` environment**: deployment branches limited to `main`. - **`Release tags` ruleset**: the release app is the only bypass actor, so no one else can create, move or delete a tag. -- **Immutable releases**: enabled, so a published `<action>-vX.Y.Z` release and its tag +- **Immutable releases**: enabled, so a published `<package>/vX.Y.Z` release and its tag cannot change (floating tags carry no release and stay movable by the app). -- **Squash merge commit title**: "Pull request title", so the PR title checked by `lint` - is the commit the release reads. +- **Squash merge commit title**: "Pull request title", so the title checked by `lint` + is the commit release-please reads. - **Required status check**: `lint` on `main`. diff --git a/pulp/repository-properties/parse-distrib.sh b/pulp/lib/parse-distrib.sh similarity index 100% rename from pulp/repository-properties/parse-distrib.sh rename to pulp/lib/parse-distrib.sh diff --git a/pulp/repository-properties/properties.sh b/pulp/lib/repository-properties.sh similarity index 95% rename from pulp/repository-properties/properties.sh rename to pulp/lib/repository-properties.sh index 37dce1b..3cb1f0a 100755 --- a/pulp/repository-properties/properties.sh +++ b/pulp/lib/repository-properties.sh @@ -1,12 +1,25 @@ #!/usr/bin/env bash +# Compute the pulp repository names, base paths, suites and domains of a +# delivery, as step outputs. Inputs (env): MODULE_NAME, DISTRIB, STABILITY, +# IS_CLOUD, and optionally VERSION, RELEASE_TYPE, DELIVERY_TYPE, +# REPOSITORY_TYPE, GH_HEAD_REF. set -euo pipefail +MODULE_NAME="${MODULE_NAME:-}" +DISTRIB="${DISTRIB:-}" +STABILITY="${STABILITY:-}" +IS_CLOUD="${IS_CLOUD:-}" +VERSION="${VERSION:-}" +RELEASE_TYPE="${RELEASE_TYPE:-}" +DELIVERY_TYPE="${DELIVERY_TYPE:-standard}" +GH_HEAD_REF="${GH_HEAD_REF:-}" + if [[ -z "$MODULE_NAME" || -z "$DISTRIB" || -z "$STABILITY" || -z "$IS_CLOUD" ]]; then echo "::error::some mandatory inputs are empty, please check the logs." exit 1 fi -# shellcheck source=pulp/repository-properties/parse-distrib.sh +# shellcheck source=pulp/lib/parse-distrib.sh source "$(dirname "$0")/parse-distrib.sh" # repository_name selects the standard (open-source), business (paid) or plugins diff --git a/pulp/package-delivery/action.yml b/pulp/package-delivery/action.yml index 5877c42..5563adc 100644 --- a/pulp/package-delivery/action.yml +++ b/pulp/package-delivery/action.yml @@ -87,7 +87,7 @@ runs: DELIVERY_TYPE: ${{ inputs.delivery_type }} REPOSITORY_TYPE: ${{ inputs.repository_name }} GH_HEAD_REF: ${{ github.head_ref || github.ref_name }} - run: ${{ github.action_path }}/../repository-properties/properties.sh + run: ${{ github.action_path }}/../lib/repository-properties.sh - name: Restore packages from cache if: steps.pulp-properties.outputs.skip_delivery == 'false' diff --git a/pulp/promote-to-stable/action.yml b/pulp/promote-to-stable/action.yml index 22f1597..d17b271 100644 --- a/pulp/promote-to-stable/action.yml +++ b/pulp/promote-to-stable/action.yml @@ -65,7 +65,7 @@ runs: IS_CLOUD: ${{ inputs.is_cloud }} REPOSITORY_TYPE: ${{ inputs.repository_name }} GH_HEAD_REF: ${{ github.head_ref || github.ref_name }} - run: ${{ github.action_path }}/../repository-properties/properties.sh + run: ${{ github.action_path }}/../lib/repository-properties.sh - name: Setup python uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 diff --git a/pulp/repository-properties/action.yml b/pulp/repository-properties/action.yml deleted file mode 100644 index 345cca5..0000000 --- a/pulp/repository-properties/action.yml +++ /dev/null @@ -1,146 +0,0 @@ -name: "pulp-repository-properties" -description: "Compute pulp repository names, base paths and suites of a package delivery" -inputs: - module_name: - description: "The package module name" - required: true - distrib: - description: "The distribution used for packaging" - required: true - version: - description: "Centreon packaged major version (not used by the plugins repository family)" - required: false - default: "" - stability: - description: "The package stability (stable, testing, unstable)" - required: true - release_type: - description: "Type of release (hotfix, release)" - required: false - default: "" - is_cloud: - description: "Release context (cloud or not cloud)" - required: true - delivery_type: - description: "Delivery to feature or standard repository" - required: false - default: "standard" - repository_name: - description: "Target repository family: standard, standard-internal, business, business-internal, plugins or connectors" - required: false - default: "standard" -outputs: - package_extension: - description: "Package extension according to the distribution" - value: ${{ steps.properties.outputs.package_extension }} - distrib_family: - description: "Distrib family (el, debian, ubuntu)" - value: ${{ steps.properties.outputs.distrib_family }} - package_distrib_name: - description: "Distribution suffix in package name" - value: ${{ steps.properties.outputs.package_distrib_name }} - skip_delivery: - description: "Whether the delivery must be skipped" - value: ${{ steps.properties.outputs.skip_delivery }} - repository_prefix: - description: "Repository name prefix of rpm repositories (one repository per architecture)" - value: ${{ steps.properties.outputs.repository_prefix }} - base_path_prefix: - description: "Base path prefix of rpm distributions (one distribution per architecture)" - value: ${{ steps.properties.outputs.base_path_prefix }} - legacy_base_path_prefix: - description: "Legacy (front) rpm base path prefix, used by the content verifications" - value: ${{ steps.properties.outputs.legacy_base_path_prefix }} - legacy_testing_base_path_prefix: - description: "Legacy (front) testing rpm base path prefix" - value: ${{ steps.properties.outputs.legacy_testing_base_path_prefix }} - legacy_stable_base_path_prefix: - description: "Legacy (front) stable rpm base path prefix" - value: ${{ steps.properties.outputs.legacy_stable_base_path_prefix }} - repository_name: - description: "Repository name of the deb repository" - value: ${{ steps.properties.outputs.repository_name }} - base_path: - description: "Base path of the deb distribution" - value: ${{ steps.properties.outputs.base_path }} - legacy_base_path: - description: "Legacy (front) deb base path, used by the content verifications" - value: ${{ steps.properties.outputs.legacy_base_path }} - legacy_testing_base_path: - description: "Legacy (front) testing deb base path" - value: ${{ steps.properties.outputs.legacy_testing_base_path }} - legacy_stable_base_path: - description: "Legacy (front) stable deb base path" - value: ${{ steps.properties.outputs.legacy_stable_base_path }} - suite: - description: "Apt suite of the deb delivery" - value: ${{ steps.properties.outputs.suite }} - testing_repository_prefix: - description: "Repository name prefix of testing rpm repositories, used as promotion source" - value: ${{ steps.properties.outputs.testing_repository_prefix }} - testing_base_path_prefix: - description: "Base path prefix of testing rpm distributions, used to download promotion source content" - value: ${{ steps.properties.outputs.testing_base_path_prefix }} - stable_repository_prefix: - description: "Repository name prefix of stable rpm repositories" - value: ${{ steps.properties.outputs.stable_repository_prefix }} - stable_base_path_prefix: - description: "Base path prefix of stable rpm distributions" - value: ${{ steps.properties.outputs.stable_base_path_prefix }} - testing_repository_name: - description: "Repository name of the testing deb repository, used as promotion source" - value: ${{ steps.properties.outputs.testing_repository_name }} - testing_base_path: - description: "Base path of the testing deb distribution, used to download promotion source content" - value: ${{ steps.properties.outputs.testing_base_path }} - testing_suite: - description: "Apt suite of testing deb packages" - value: ${{ steps.properties.outputs.testing_suite }} - stable_suite: - description: "Apt suite of stable deb packages" - value: ${{ steps.properties.outputs.stable_suite }} - stable_legacy_suite: - description: "Plain-codename suite of the dedicated 24.x stable legacy repositories (empty otherwise)" - value: ${{ steps.properties.outputs.stable_legacy_suite }} - stable_legacy_repository_name: - description: "Dedicated 24.x stable legacy repository fed by the promote mirror (empty otherwise)" - value: ${{ steps.properties.outputs.stable_legacy_repository_name }} - stable_repository_name: - description: "Repository holding the stable deb packages (one deb repository per stability)" - value: ${{ steps.properties.outputs.stable_repository_name }} - stable_base_path: - description: "Base path serving the stable packages" - value: ${{ steps.properties.outputs.stable_base_path }} - pool_path: - description: "Module scoped pool path of the deb delivery" - value: ${{ steps.properties.outputs.pool_path }} - testing_pool_path: - description: "Module scoped pool path of testing deb packages, used as promotion source" - value: ${{ steps.properties.outputs.testing_pool_path }} - stable_pool_path: - description: "Module scoped pool path of stable deb packages" - value: ${{ steps.properties.outputs.stable_pool_path }} - domain: - description: "Pulp Domain of the delivery (testing/unstable tier) repository" - value: ${{ steps.properties.outputs.domain }} - stable_domain: - description: "Pulp Domain of the stable-tier repository" - value: ${{ steps.properties.outputs.stable_domain }} - -runs: - using: "composite" - steps: - - name: Compute pulp repository properties - id: properties - shell: bash - env: - MODULE_NAME: ${{ inputs.module_name }} - DISTRIB: ${{ inputs.distrib }} - VERSION: ${{ inputs.version }} - STABILITY: ${{ inputs.stability }} - RELEASE_TYPE: ${{ inputs.release_type }} - IS_CLOUD: ${{ inputs.is_cloud }} - DELIVERY_TYPE: ${{ inputs.delivery_type }} - REPOSITORY_TYPE: ${{ inputs.repository_name }} - GH_HEAD_REF: ${{ github.head_ref || github.ref_name }} - run: ${{ github.action_path }}/properties.sh diff --git a/pulp/tests/repository-properties.bats b/pulp/tests/repository-properties.bats new file mode 100644 index 0000000..2f39313 --- /dev/null +++ b/pulp/tests/repository-properties.bats @@ -0,0 +1,101 @@ +#!/usr/bin/env bats +# Repository layout computed for a delivery or a promotion. + +setup() { + SCRIPT="${BATS_TEST_DIRNAME}/../lib/repository-properties.sh" + export GITHUB_OUTPUT="${BATS_TEST_TMPDIR}/output" + : > "$GITHUB_OUTPUT" +} + +# run the script with only the given variables, like a composite action step +properties() { + run env -i PATH="$PATH" GITHUB_OUTPUT="$GITHUB_OUTPUT" "$@" "$SCRIPT" +} + +output_of() { + grep -E "^$1=" "$GITHUB_OUTPUT" | tail -n 1 | cut -d= -f2- +} + +@test "rpm stable release: one repository per architecture" { + properties MODULE_NAME=web DISTRIB=el9 STABILITY=stable IS_CLOUD=false VERSION=25.10 RELEASE_TYPE=release + [ "$status" -eq 0 ] + [ "$(output_of package_extension)" = "rpm" ] + [ "$(output_of repository_prefix)" = "rpm-25.10-el9-stable" ] + [ "$(output_of testing_repository_prefix)" = "rpm-25.10-el9-testing-release" ] + [ "$(output_of stable_base_path_prefix)" = "rpm/25.10/el9/stable" ] + [ "$(output_of domain)" = "standard" ] + [ "$(output_of skip_delivery)" = "false" ] +} + +@test "promotion env: delivery type, branch and repository family are optional" { + properties MODULE_NAME=web DISTRIB=el9 STABILITY=stable IS_CLOUD=false VERSION=25.10 RELEASE_TYPE=hotfix + [ "$status" -eq 0 ] + [ "$(output_of testing_repository_prefix)" = "rpm-25.10-el9-testing-hotfix" ] +} + +@test "deb testing hotfix: suite and pool carry the version" { + properties MODULE_NAME=web DISTRIB=bookworm STABILITY=testing IS_CLOUD=false VERSION=25.10 RELEASE_TYPE=hotfix + [ "$status" -eq 0 ] + [ "$(output_of package_extension)" = "deb" ] + [ "$(output_of repository_name)" = "apt-testing-hotfix" ] + [ "$(output_of suite)" = "bookworm-25.10-testing-hotfix" ] + [ "$(output_of stable_suite)" = "bookworm-25.10-stable" ] + [ "$(output_of stable_repository_name)" = "apt-stable" ] + [ "$(output_of pool_path)" = "pool/25.10/testing/hotfix/web" ] +} + +@test "rpm feature delivery: repository named after the branch ticket" { + properties MODULE_NAME=web DISTRIB=el9 STABILITY=unstable IS_CLOUD=true VERSION=26.10 \ + DELIVERY_TYPE=feature GH_HEAD_REF=MON-1234-my-feature + [ "$status" -eq 0 ] + [ "$(output_of repository_prefix)" = "rpm-feature-MON-1234-26.10-el9-unstable" ] + [ "$(output_of base_path_prefix)" = "rpm-feature/MON-1234/26.10/el9/unstable" ] +} + +@test "deb feature delivery is skipped" { + properties MODULE_NAME=web DISTRIB=bookworm STABILITY=unstable IS_CLOUD=false VERSION=25.10 DELIVERY_TYPE=feature + [ "$status" -eq 0 ] + [ "$(output_of skip_delivery)" = "true" ] +} + +@test "connectors: unversioned, plain codename suite, own domain" { + properties MODULE_NAME=packs DISTRIB=noble STABILITY=unstable IS_CLOUD=false REPOSITORY_TYPE=connectors + [ "$status" -eq 0 ] + [ "$(output_of repository_name)" = "ubuntu-unstable" ] + [ "$(output_of suite)" = "noble" ] + [ "$(output_of pool_path)" = "pool/unstable/packs" ] + [ "$(output_of domain)" = "connectors" ] +} + +@test "version is mandatory for versioned families" { + properties MODULE_NAME=web DISTRIB=el9 STABILITY=stable IS_CLOUD=false + [ "$status" -ne 0 ] + [[ "$output" == *"version input is mandatory"* ]] +} + +@test "unknown distribution is rejected" { + properties MODULE_NAME=web DISTRIB=fedora STABILITY=stable IS_CLOUD=false VERSION=25.10 + [ "$status" -ne 0 ] + [[ "$output" == *"Distrib fedora cannot be parsed"* ]] +} + +@test "unknown repository family is rejected" { + properties MODULE_NAME=web DISTRIB=el9 STABILITY=stable IS_CLOUD=false VERSION=25.10 REPOSITORY_TYPE=other + [ "$status" -ne 0 ] + [[ "$output" == *"Unsupported repository_name"* ]] +} + +@test "missing mandatory input is rejected" { + properties DISTRIB=el9 STABILITY=stable IS_CLOUD=false VERSION=25.10 + [ "$status" -ne 0 ] + [[ "$output" == *"mandatory inputs are empty"* ]] +} + +@test "every output the actions read is produced" { + properties MODULE_NAME=web DISTRIB=el9 STABILITY=stable IS_CLOUD=false VERSION=25.10 RELEASE_TYPE=release + [ "$status" -eq 0 ] + used=$(grep -ohE 'steps\.pulp-properties\.outputs\.[a-z_]+' "${BATS_TEST_DIRNAME}"/../*/action.yml | sed 's/.*\.//' | sort -u) + for name in $used; do + grep -qE "^${name}=" "$GITHUB_OUTPUT" || { echo "missing output: ${name}"; return 1; } + done +} diff --git a/pulp/tests/setup-cli.bats b/pulp/tests/setup-cli.bats new file mode 100644 index 0000000..6c20554 --- /dev/null +++ b/pulp/tests/setup-cli.bats @@ -0,0 +1,28 @@ +#!/usr/bin/env bats +# Guards of the pulp-cli setup that must fail before any token is requested. + +setup() { + SCRIPT="${BATS_TEST_DIRNAME}/../setup-cli/setup.sh" + # a pulp binary on the PATH skips the installation + mkdir -p "${BATS_TEST_TMPDIR}/bin" + printf '#!/bin/sh\nexit 0\n' > "${BATS_TEST_TMPDIR}/bin/pulp" + chmod +x "${BATS_TEST_TMPDIR}/bin/pulp" + export GITHUB_ENV="${BATS_TEST_TMPDIR}/env" +} + +setup_cli() { + run env -i PATH="${BATS_TEST_TMPDIR}/bin:${PATH}" GITHUB_ENV="$GITHUB_ENV" "$@" "$SCRIPT" +} + +@test "empty audience is rejected" { + setup_cli AUDIENCE= ACTIONS_ID_TOKEN_REQUEST_URL=https://example.invalid ACTIONS_ID_TOKEN_REQUEST_TOKEN=placeholder + [ "$status" -ne 0 ] + [[ "$output" == *"audience input is empty"* ]] +} + +@test "missing id-token permission is reported" { + setup_cli AUDIENCE=pulp + [ "$status" -ne 0 ] + [[ "$output" == *"permissions: id-token: write"* ]] + [ ! -s "$GITHUB_ENV" ] +} diff --git a/release-please-config.json b/release-please-config.json new file mode 100644 index 0000000..0a59495 --- /dev/null +++ b/release-please-config.json @@ -0,0 +1,22 @@ +{ + "$schema": "https://raw.githubusercontent.com/googleapis/release-please/main/schemas/config.json", + "release-type": "simple", + "separate-pull-requests": true, + "tag-separator": "/", + "include-v-in-tag": true, + "initial-version": "1.0.0", + "changelog-sections": [ + { "type": "feat", "section": "Features" }, + { "type": "fix", "section": "Bug Fixes" }, + { "type": "perf", "section": "Performance Improvements" }, + { "type": "revert", "section": "Reverts" }, + { "type": "refactor", "section": "Code Refactoring" }, + { "type": "docs", "section": "Documentation", "hidden": true }, + { "type": "test", "section": "Tests", "hidden": true }, + { "type": "ci", "section": "Continuous Integration", "hidden": true }, + { "type": "chore", "section": "Miscellaneous", "hidden": true } + ], + "packages": { + "pulp": { "component": "pulp" } + } +} From 00c13b51ffa2d13262f8de3f8680c9c3f865e354 Mon Sep 17 00:00:00 2001 From: Kevin Duret <kduret@centreon.com> Date: Thu, 19 Nov 2026 11:57:38 +0100 Subject: [PATCH 06/15] ci: add major tag rollback, wiring smoke test and zizmor audit Let a maintainer point a major tag back at a previous release, run the delivery action up to the repository layout on a real runner, audit workflows and actions with zizmor, and delay Dependabot updates by 7 days. Refs: MON-215510 --- .github/dependabot.yml | 7 +++- .github/scripts/tests/update-major-tags.bats | 27 +++++++++++-- .github/scripts/update-major-tags.sh | 37 ++++++++++++----- .github/workflows/lint.yml | 22 ++++++++++ .github/workflows/move-major-tag.yml | 42 ++++++++++++++++++++ .github/workflows/release.yml | 1 + .github/zizmor.yml | 7 ++++ README.md | 27 +++++++++++-- 8 files changed, 152 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/move-major-tag.yml create mode 100644 .github/zizmor.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 08371fb..dfa6531 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,12 +1,15 @@ version: 2 updates: - # pins of the workflows of this repository and of the shared actions + # pins of the workflows of this repository and of every shared action - package-ecosystem: github-actions directories: - "/" - - "/pulp/*" + - "/*" + - "/*/*" schedule: interval: monthly + cooldown: + default-days: 7 open-pull-requests-limit: 10 groups: github-actions: diff --git a/.github/scripts/tests/update-major-tags.bats b/.github/scripts/tests/update-major-tags.bats index 106d41e..a305270 100644 --- a/.github/scripts/tests/update-major-tags.bats +++ b/.github/scripts/tests/update-major-tags.bats @@ -6,15 +6,18 @@ setup() { SCRIPT="${BATS_TEST_DIRNAME}/../update-major-tags.sh" export GITHUB_REPOSITORY=centreon/github-actions export GH_CALLS="${BATS_TEST_TMPDIR}/calls" + export RELEASE_SHA="$SHA" : > "$GH_CALLS" # gh stand-in: logs its arguments; the tag lookup succeeds when TAG_EXISTS=1 mkdir -p "${BATS_TEST_TMPDIR}/bin" cat > "${BATS_TEST_TMPDIR}/bin/gh" <<'STUB' #!/usr/bin/env bash echo "$*" >> "$GH_CALLS" -if [[ "$2" == repos/*/git/ref/tags/* ]]; then - [[ "${TAG_EXISTS:-0}" == 1 ]] -fi +case "$2" in + repos/*/git/ref/tags/*) [[ "${TAG_EXISTS:-0}" == 1 ]] ;; + repos/*/commits/tags/missing/*) exit 1 ;; + repos/*/commits/tags/*) echo "$RELEASE_SHA" ;; +esac STUB chmod +x "${BATS_TEST_TMPDIR}/bin/gh" PATH="${BATS_TEST_TMPDIR}/bin:${PATH}" @@ -55,3 +58,21 @@ releases() { [ "$status" -ne 0 ] [ ! -s "$GH_CALLS" ] } + +@test "rollback: an explicit release tag moves its major tag back" { + TAG_EXISTS=1 run "$SCRIPT" pulp/v1.4.1 + [ "$status" -eq 0 ] + grep -qF -- "-X PATCH repos/centreon/github-actions/git/refs/tags/pulp/v1 -f sha=${SHA} -F force=true" "$GH_CALLS" +} + +@test "rollback: an unknown release tag fails without touching tags" { + run "$SCRIPT" missing/v1.0.0 + [ "$status" -ne 0 ] + ! grep -qE -- "-X (PATCH|POST)" "$GH_CALLS" +} + +@test "rollback: a major tag is not a release tag" { + run "$SCRIPT" pulp/v1 + [ "$status" -ne 0 ] + ! grep -qE -- "-X (PATCH|POST)" "$GH_CALLS" +} diff --git a/.github/scripts/update-major-tags.sh b/.github/scripts/update-major-tags.sh index a89b2f5..a74f2fd 100755 --- a/.github/scripts/update-major-tags.sh +++ b/.github/scripts/update-major-tags.sh @@ -1,17 +1,16 @@ #!/usr/bin/env bash -# Point the floating major tag of every package release-please just released -# (<package>/vX) at its new release (<package>/vX.Y.Z). -# Inputs (env): RELEASES (JSON outputs of release-please-action), -# GITHUB_REPOSITORY, GH_TOKEN. +# Point the floating major tag of a package (<package>/vX) at one of its +# releases (<package>/vX.Y.Z). +# - without argument: every package release-please just released, read from +# RELEASES (JSON outputs of release-please-action); +# - with release tags as arguments: those releases, to roll a major tag back. +# Environment: GITHUB_REPOSITORY, GH_TOKEN. set -euo pipefail -paths=$(jq -r '(.paths_released // "[]") | fromjson | .[]' <<< "$RELEASES") - -for path in $paths; do - tag=$(jq -r --arg key "${path}--tag_name" '.[$key] // empty' <<< "$RELEASES") - sha=$(jq -r --arg key "${path}--sha" '.[$key] // empty' <<< "$RELEASES") +move_major_tag() { + local tag=$1 sha=$2 major_tag if [[ ! "$sha" =~ ^[0-9a-f]{40}$ || ! "$tag" =~ ^(.+)/v([0-9]+)\.[0-9]+\.[0-9]+$ ]]; then - echo "::error::unexpected release of ${path}: tag '${tag}', sha '${sha}'" + echo "::error::unexpected release: tag '${tag}', sha '${sha}'" exit 1 fi # BASH_REMATCH holds the tag match: it is the last regex evaluated above @@ -24,4 +23,22 @@ for path in $paths; do fi echo "${major_tag} -> ${tag} (${sha})" echo "- \`${major_tag}\` now points at \`${tag}\`" >> "${GITHUB_STEP_SUMMARY:-/dev/null}" +} + +if (($# > 0)); then + for tag in "$@"; do + if ! sha=$(gh api "repos/${GITHUB_REPOSITORY}/commits/tags/${tag}" --jq .sha); then + echo "::error::release tag '${tag}' does not exist" + exit 1 + fi + move_major_tag "$tag" "$sha" + done + exit 0 +fi + +paths=$(jq -r '(.paths_released // "[]") | fromjson | .[]' <<< "$RELEASES") +for path in $paths; do + tag=$(jq -r --arg key "${path}--tag_name" '.[$key] // empty' <<< "$RELEASES") + sha=$(jq -r --arg key "${path}--sha" '.[$key] // empty' <<< "$RELEASES") + move_major_tag "$tag" "$sha" done diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 05cac6f..9299e03 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -16,9 +16,11 @@ env: ACTIONLINT_VERSION: "1.7.12" ACTIONLINT_SHA256: "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8" BATS_VERSION: "1.13.0" + ZIZMOR_VERSION: "1.32.0" jobs: lint: + name: lint runs-on: ubuntu-24.04 permissions: contents: read @@ -48,6 +50,26 @@ jobs: mapfile -t test_dirs < <(git ls-files '*.bats' | xargs -n 1 dirname | sort -u) npx --yes "bats@${BATS_VERSION}" "${test_dirs[@]}" + # deb feature deliveries are skipped: this runs the action up to the + # repository layout, on a real runner, without any Pulp server + - name: Smoke test the wiring of pulp/package-delivery + # not the $/ form yet: actionlint 1.7.12 rejects it + uses: ./pulp/package-delivery # zizmor: ignore[self-repository] + with: + module_name: smoke-test + distrib: bookworm + version: "26.10" + cache_key: unused + stability: unstable + is_cloud: "false" + delivery_type: feature + pulp_url: https://pulp.invalid + pulp_content_url: https://pulp.invalid + audience: unused + + - name: Zizmor + run: pipx run "zizmor==${ZIZMOR_VERSION}" --offline . + - name: Actionlint run: | archive="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" diff --git a/.github/workflows/move-major-tag.yml b/.github/workflows/move-major-tag.yml new file mode 100644 index 0000000..e3f0386 --- /dev/null +++ b/.github/workflows/move-major-tag.yml @@ -0,0 +1,42 @@ +name: move-major-tag + +# Roll a floating major tag back (or forward) to an existing release of its +# package, without waiting for a new release. See the README. + +on: + workflow_dispatch: + inputs: + release_tag: + description: "Release the major tag must point at, e.g. pulp/v1.4.1" + required: true + type: string + +permissions: {} + +jobs: + move-major-tag: + name: move-major-tag + runs-on: ubuntu-24.04 + # the environment only allows main and holds the release app key + environment: release + permissions: + contents: read + steps: + - name: Create release app token + id: app-token + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3.3.0 + with: + client-id: ${{ vars.RELEASE_APP_CLIENT_ID }} + private-key: ${{ secrets.RELEASE_APP_PRIVATE_KEY }} + permission-contents: write + + - name: Checkout sources + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Move the major tag + env: + RELEASE_TAG: ${{ inputs.release_tag }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} + run: .github/scripts/update-major-tags.sh "$RELEASE_TAG" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f0647f6..251fb6d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,6 +17,7 @@ permissions: {} jobs: release: + name: release runs-on: ubuntu-24.04 # the environment only allows main and holds the release app key environment: release diff --git a/.github/zizmor.yml b/.github/zizmor.yml new file mode 100644 index 0000000..aefae07 --- /dev/null +++ b/.github/zizmor.yml @@ -0,0 +1,7 @@ +rules: + unpinned-uses: + config: + policies: + # security workflows are synchronized from security-tools, pinned to its main branch + "centreon/security-tools/*": ref-pin + "*": hash-pin diff --git a/README.md b/README.md index 7df195b..f6b2af5 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,10 @@ Reference an action by its path and the major tag of its package: - To freeze a branch, reference an exact release: `@pulp/v1.4.2` (never moved) or its commit SHA. +The tag carries the package name because a ref applies to the whole repository: a +plain `@v1` would give every action the same version, and a breaking change in one +package would force all the others to a new major. + In the calling repository: - allow `centreon/github-actions` in the SHA pinning check, as for @@ -67,6 +71,22 @@ A commit counts for the packages whose directory it changes. Nothing reaches the callers before step 3: the release pull request is the moment to decide that `main` is good to ship. +### Testing a change before its release + +Reference the commit of the pull request from a branch of a calling repository, and +run the workflow there: + +```yaml +- uses: centreon/github-actions/pulp/package-delivery@<commit-sha> +``` + +### Rolling back a release + +A release reaches every caller at once. If it breaks them, run the `move-major-tag` +workflow with the previous release (for example `pulp/v1.4.1`): the major tag points +back at it within seconds. Then fix or revert on `main`; the next release moves the +tag forward again. + ## Adding or changing an action - Put a new action in an existing package when it shares code with it, otherwise create @@ -75,13 +95,14 @@ decide that `main` is good to ship. - An action is self-contained in its package: no `uses: ./...` (that path resolves in the caller's workspace) and no path leaving the package. Shared logic lives in scripts called through `${{ github.action_path }}`, so all files come from the same - release. + release, on any runner version. - Remote actions are pinned to a full commit SHA with a `# vX.Y.Z` comment. - Shell scripts follow shellcheck; test them with [bats](https://github.com/bats-core/bats-core) in `<package>/tests/`. -The `lint` workflow enforces all of this on every pull request, with the title format. -Run it locally with: +The `lint` workflow enforces all of this on every pull request, with the title format, +a [zizmor](https://docs.zizmor.sh/) audit and a run of `pulp/package-delivery` up to the +repository layout. Run the checks locally with: ```shell .github/scripts/check-actions.sh From 22aaa6d81de98314b18d9122c088ae3c1bacad7b Mon Sep 17 00:00:00 2001 From: Kevin Duret <kduret@centreon.com> Date: Thu, 19 Nov 2026 14:01:37 +0100 Subject: [PATCH 07/15] ci: replace actionlint with jactionlint actionlint has had no release since March and rejects the self-repository uses syntax; jactionlint, its maintained fork, also checks composite actions. Set the job timeouts it asks for and use the self-repository syntax in the wiring smoke test. Refs: MON-215510 --- .github/jactionlint.yaml | 12 ++++++++++++ .github/workflows/lint.yml | 17 +++++------------ .github/workflows/move-major-tag.yml | 1 + .github/workflows/release.yml | 1 + README.md | 7 +++++-- 5 files changed, 24 insertions(+), 14 deletions(-) create mode 100644 .github/jactionlint.yaml diff --git a/.github/jactionlint.yaml b/.github/jactionlint.yaml new file mode 100644 index 0000000..9506cfa --- /dev/null +++ b/.github/jactionlint.yaml @@ -0,0 +1,12 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/jdx/jactionlint/main/schema/jactionlint-config.json +rules: + unpinned-uses: + policies: + # security workflows are synchronized from security-tools, pinned to its main branch + "centreon/security-tools/*": ref-pin + +# names the workflows may read: a typo in vars.* or secrets.* is an error +config-variables: + - RELEASE_APP_CLIENT_ID +config-secrets: + - RELEASE_APP_PRIVATE_KEY diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9299e03..84f985a 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -13,8 +13,7 @@ on: permissions: {} env: - ACTIONLINT_VERSION: "1.7.12" - ACTIONLINT_SHA256: "8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8" + JACTIONLINT_VERSION: "2.6.0" BATS_VERSION: "1.13.0" ZIZMOR_VERSION: "1.32.0" @@ -22,6 +21,7 @@ jobs: lint: name: lint runs-on: ubuntu-24.04 + timeout-minutes: 10 permissions: contents: read steps: @@ -53,8 +53,7 @@ jobs: # deb feature deliveries are skipped: this runs the action up to the # repository layout, on a real runner, without any Pulp server - name: Smoke test the wiring of pulp/package-delivery - # not the $/ form yet: actionlint 1.7.12 rejects it - uses: ./pulp/package-delivery # zizmor: ignore[self-repository] + uses: $/pulp/package-delivery with: module_name: smoke-test distrib: bookworm @@ -70,11 +69,5 @@ jobs: - name: Zizmor run: pipx run "zizmor==${ZIZMOR_VERSION}" --offline . - - name: Actionlint - run: | - archive="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz" - curl -fsSL --retry 3 -o "${RUNNER_TEMP}/${archive}" \ - "https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/${archive}" - echo "${ACTIONLINT_SHA256} ${RUNNER_TEMP}/${archive}" | sha256sum --check --strict - tar -xzf "${RUNNER_TEMP}/${archive}" -C "${RUNNER_TEMP}" actionlint - "${RUNNER_TEMP}/actionlint" -color + - name: Jactionlint + run: pipx run "jactionlint==${JACTIONLINT_VERSION}" diff --git a/.github/workflows/move-major-tag.yml b/.github/workflows/move-major-tag.yml index e3f0386..0106657 100644 --- a/.github/workflows/move-major-tag.yml +++ b/.github/workflows/move-major-tag.yml @@ -17,6 +17,7 @@ jobs: move-major-tag: name: move-major-tag runs-on: ubuntu-24.04 + timeout-minutes: 5 # the environment only allows main and holds the release app key environment: release permissions: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 251fb6d..c2b08a3 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -19,6 +19,7 @@ jobs: release: name: release runs-on: ubuntu-24.04 + timeout-minutes: 10 # the environment only allows main and holds the release app key environment: release permissions: diff --git a/README.md b/README.md index f6b2af5..3c5eba9 100644 --- a/README.md +++ b/README.md @@ -101,13 +101,16 @@ tag forward again. [bats](https://github.com/bats-core/bats-core) in `<package>/tests/`. The `lint` workflow enforces all of this on every pull request, with the title format, -a [zizmor](https://docs.zizmor.sh/) audit and a run of `pulp/package-delivery` up to the -repository layout. Run the checks locally with: +a run of `pulp/package-delivery` up to the repository layout, and two linters of the +workflows and actions: [jactionlint](https://jactionlint.jdx.dev) (validity) and +[zizmor](https://docs.zizmor.sh/) (security). Run the checks locally with: ```shell .github/scripts/check-actions.sh shellcheck -x -P SCRIPTDIR $(git ls-files '*.sh') npx bats pulp/tests .github/scripts/tests +uvx jactionlint +uvx zizmor --offline . ``` ## pulp From f586e53cbe9cbc9fd879d332fad254b9cef1f540 Mon Sep 17 00:00:00 2001 From: Kevin Duret <kduret@centreon.com> Date: Thu, 19 Nov 2026 14:06:42 +0100 Subject: [PATCH 08/15] refactor(pulp): call setup-cli through the self-repository syntax package-delivery and promote-to-stable duplicated the two steps of setup-cli; they now call it with uses: $/pulp/setup-cli, which resolves to the commit of the release in use. The actions check accepts that syntax inside a package only. Refs: MON-215510 --- .github/scripts/check-actions.sh | 14 ++++++++++++-- .github/scripts/tests/check-actions.bats | 22 ++++++++++++++++++++++ README.md | 7 ++++--- pulp/package-delivery/action.yml | 17 +++++------------ pulp/promote-to-stable/action.yml | 16 +++++----------- pulp/setup-cli/action.yml | 2 +- 6 files changed, 49 insertions(+), 29 deletions(-) diff --git a/.github/scripts/check-actions.sh b/.github/scripts/check-actions.sh index 45dd9e1..ed42f4a 100755 --- a/.github/scripts/check-actions.sh +++ b/.github/scripts/check-actions.sh @@ -4,7 +4,9 @@ # - every action belongs to a package declared in release-please-config.json, # whose component is the directory name (it prefixes the release tags); # - no local "uses: ./...": it resolves in the caller's workspace, not here; -# - no path leaving the package: a change there would not release it; +# an action of the same package is called with "uses: $/<package>/..."; +# - no path or "$/" reference leaving the package: a change there would not +# release it; # - every remote action is pinned to a full commit SHA with a version comment. # Usage: check-actions.sh [repository root] set -euo pipefail @@ -33,7 +35,15 @@ for action_file in "${action_files[@]}"; do ref=${ref#"${ref%%[![:space:]]*}"} ref=${ref#[\"\']} if [[ "$ref" == ./* ]]; then - error "$action_file" "line ${number}: local action '${ref}' resolves in the caller's workspace; call a script through github.action_path instead" + error "$action_file" "line ${number}: local action '${ref}' resolves in the caller's workspace; use '\$/${package}/...' instead" + elif [[ "$ref" == \$/* ]]; then + target=${ref#\$/} + target=${target%%[[:space:]\"\']*} + if [[ "$target" != "${package}/"* ]]; then + error "$action_file" "line ${number}: '${ref}' leaves the package '${package}'" + elif [[ ! -f "${target}/action.yml" ]]; then + error "$action_file" "line ${number}: '${ref}' is not an action of this repository" + fi elif [[ ! "$ref" =~ ^[^@[:space:]]+@[0-9a-f]{40}[\"\']?[[:space:]]+#[[:space:]]*v?[0-9]+(\.[0-9]+)*$ ]]; then error "$action_file" "line ${number}: '${ref}' must be pinned to a full commit SHA followed by a '# vX.Y.Z' comment" fi diff --git a/.github/scripts/tests/check-actions.bats b/.github/scripts/tests/check-actions.bats index 5ab41e2..512bb7f 100644 --- a/.github/scripts/tests/check-actions.bats +++ b/.github/scripts/tests/check-actions.bats @@ -43,6 +43,28 @@ action() { [[ "$output" == *"resolves in the caller's workspace"* ]] } +@test "an action of the same package through the self-repository syntax passes" { + mkdir -p "${REPO}/tool/setup" + cp "${REPO}/tool/run/action.yml" "${REPO}/tool/setup/action.yml" + action 'uses: $/tool/setup' + run "$SCRIPT" "$REPO" + [ "$status" -eq 0 ] +} + +@test "a self-repository reference to another package fails" { + action 'uses: $/other/setup' + run "$SCRIPT" "$REPO" + [ "$status" -ne 0 ] + [[ "$output" == *"leaves the package 'tool'"* ]] +} + +@test "a self-repository reference to a missing action fails" { + action 'uses: $/tool/missing' + run "$SCRIPT" "$REPO" + [ "$status" -ne 0 ] + [[ "$output" == *"is not an action of this repository"* ]] +} + @test "a tag-pinned action fails" { action 'uses: actions/checkout@v4' run "$SCRIPT" "$REPO" diff --git a/README.md b/README.md index 3c5eba9..327feb3 100644 --- a/README.md +++ b/README.md @@ -93,9 +93,10 @@ tag forward again. a package: a top-level directory, declared in `release-please-config.json` with its directory name as `component`. - An action is self-contained in its package: no `uses: ./...` (that path resolves in - the caller's workspace) and no path leaving the package. Shared logic lives in - scripts called through `${{ github.action_path }}`, so all files come from the same - release, on any runner version. + the caller's workspace) and nothing referenced outside the package. It calls another + action of its package with `uses: $/<package>/<action>` and shared scripts through + `${{ github.action_path }}`: both resolve to the commit of the release in use. + `$/` needs a runner 2.336.0 or newer. - Remote actions are pinned to a full commit SHA with a `# vX.Y.Z` comment. - Shell scripts follow shellcheck; test them with [bats](https://github.com/bats-core/bats-core) in `<package>/tests/`. diff --git a/pulp/package-delivery/action.yml b/pulp/package-delivery/action.yml index 5563adc..9eea6e4 100644 --- a/pulp/package-delivery/action.yml +++ b/pulp/package-delivery/action.yml @@ -97,20 +97,13 @@ runs: key: ${{ inputs.cache_key }} fail-on-cache-miss: true - - name: Setup python - if: steps.pulp-properties.outputs.skip_delivery == 'false' - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: "3.12" - - name: Setup pulp-cli if: steps.pulp-properties.outputs.skip_delivery == 'false' - shell: bash - env: - PULP_URL: ${{ inputs.pulp_url }} - AUDIENCE: ${{ inputs.audience }} - DOMAIN: ${{ steps.pulp-properties.outputs.domain }} - run: ${{ github.action_path }}/../setup-cli/setup.sh + uses: $/pulp/setup-cli + with: + pulp_url: ${{ inputs.pulp_url }} + audience: ${{ inputs.audience }} + domain: ${{ steps.pulp-properties.outputs.domain }} - name: Ensure the rpm feature repository exists if: inputs.delivery_type == 'feature' && steps.pulp-properties.outputs.package_extension == 'rpm' && steps.pulp-properties.outputs.skip_delivery == 'false' diff --git a/pulp/promote-to-stable/action.yml b/pulp/promote-to-stable/action.yml index d17b271..af64194 100644 --- a/pulp/promote-to-stable/action.yml +++ b/pulp/promote-to-stable/action.yml @@ -67,18 +67,12 @@ runs: GH_HEAD_REF: ${{ github.head_ref || github.ref_name }} run: ${{ github.action_path }}/../lib/repository-properties.sh - - name: Setup python - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 - with: - python-version: "3.12" - - name: Setup pulp-cli - shell: bash - env: - PULP_URL: ${{ inputs.pulp_url }} - AUDIENCE: ${{ inputs.audience }} - DOMAIN: ${{ steps.pulp-properties.outputs.domain }} - run: ${{ github.action_path }}/../setup-cli/setup.sh + uses: $/pulp/setup-cli + with: + pulp_url: ${{ inputs.pulp_url }} + audience: ${{ inputs.audience }} + domain: ${{ steps.pulp-properties.outputs.domain }} - name: Promote RPM packages to stable id: promote-rpm diff --git a/pulp/setup-cli/action.yml b/pulp/setup-cli/action.yml index f02c17d..26ccde7 100644 --- a/pulp/setup-cli/action.yml +++ b/pulp/setup-cli/action.yml @@ -8,7 +8,7 @@ inputs: description: "OIDC audience, must match the pulp server GITHUB_OIDC.audience" required: true domain: - description: "Pulp Domain to operate in (DOMAIN_ENABLED). Pre-Domains repositories live in \"default\"." + description: 'Pulp Domain to operate in (DOMAIN_ENABLED). Pre-Domains repositories live in "default".' required: false default: "default" From 7d4419e07fdd80a8edc9b0c6c280798a9f0f07de Mon Sep 17 00:00:00 2001 From: Kevin Duret <kduret@centreon.com> Date: Thu, 19 Nov 2026 14:52:41 +0100 Subject: [PATCH 09/15] perf(pulp): check the presence of delivered deb packages in one listing The deb verification asked the api once per package, 687 sequential lookups and about 15 minutes for a full plugins delivery. A delivery labels its uploads with its run id, so one filtered listing per repository now answers for all of them; the per-package lookup remains for what the listing misses and for promotions. Refs: MON-215510 --- pulp/lib/check-deb.sh | 48 +++++++++++++++-- pulp/tests/check-deb.bats | 111 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 155 insertions(+), 4 deletions(-) create mode 100644 pulp/tests/check-deb.bats diff --git a/pulp/lib/check-deb.sh b/pulp/lib/check-deb.sh index 43672b2..4bcf016 100755 --- a/pulp/lib/check-deb.sh +++ b/pulp/lib/check-deb.sh @@ -27,21 +27,60 @@ mapfile -t E_BASEPATH < <(echo "$PACKAGES_JSON" | jq -r '.[].base_path') mapfile -t E_SUITE < <(echo "$PACKAGES_JSON" | jq -r '.[].suite') # --- physical presence: content units in the repository's latest version ---- -# one lookup per expected package, by sha256 within the repository version: the -# shared repositories hold 10k+ packages and listing them, even newest first, -# is slow enough to hit the gateway timeout, which then read as "absent" while -# the package was published and fetchable. --retry covers those transient 5xx. declare -A VERSION_BY_REPO # repo -> latest_version_href, empty when missing for repo in $(printf '%s\n' "${E_REPOSITORY[@]}" | sort -u); do VERSION_BY_REPO[$repo]=$(latest_version_href deb "$repo") [[ -n "${VERSION_BY_REPO[$repo]}" ]] || echo "[WARN] Repository $repo does not exist or has no version" done +# batch lookup: a delivery labels the packages it uploads with its run id, so +# one filtered listing per repository answers for all of them. Listing the +# whole repository version instead is not an option: the shared repositories +# hold 10k+ packages and that listing hits the gateway timeout. +declare -A BATCH_PRESENT # "repo|sha256" -> 1 +if [[ "${CHECK_MODE:-delivery}" == "delivery" && -n "${GITHUB_RUN_ID:-}" ]]; then + for repo in "${!VERSION_BY_REPO[@]}"; do + version_href=${VERSION_BY_REPO[$repo]} + [[ -n "$version_href" ]] || continue + url="$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/packages/?$( + printf 'repository_version=%s&pulp_label_select=%s&fields=sha256&limit=1000' \ + "$(jq -rn --arg v "$version_href" '$v | @uri')" \ + "$(jq -rn --arg v "github_run_id=$GITHUB_RUN_ID" '$v | @uri')" + )" + pages=0 + found=0 + while [[ -n "$url" ]] && ((pages < 20)); do + refresh_pulp_token + page=$(curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Bearer $PULP_TOKEN" "$url") || { + echo "[WARN] presence listing failed for $repo ($url), falling back to one lookup per package" >&2 + break + } + while IFS= read -r sha; do + BATCH_PRESENT["$repo|$sha"]=1 + found=$((found + 1)) + done < <(jq -r '.results[].sha256' <<< "$page") + pages=$((pages + 1)) + url=$(jq -r '.next // empty' <<< "$page") + done + echo "[INFO] Repository $repo holds $found package(s) delivered by this run in its latest version" + done +fi + +# one lookup per package the listing did not return (content that already +# existed keeps the labels of the run that uploaded it, and a promotion +# uploads nothing), by sha256 within the repository version. --retry covers +# transient 5xx, which would otherwise read as "absent". declare -A PRESENT_IDX +lookups=0 for i in "${!E_FILENAME[@]}"; do PRESENT_IDX[$i]=false version_href=${VERSION_BY_REPO[${E_REPOSITORY[$i]}]} [[ -n "$version_href" ]] || continue + if [[ -n "${BATCH_PRESENT["${E_REPOSITORY[$i]}|${E_SHA256[$i]}"]:-}" ]]; then + PRESENT_IDX[$i]=true + continue + fi + lookups=$((lookups + 1)) refresh_pulp_token url="$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/packages/?$( printf 'repository_version=%s&sha256=%s&fields=pulp_href&limit=1' \ @@ -54,6 +93,7 @@ for i in "${!E_FILENAME[@]}"; do } [[ "${count:-0}" -gt 0 ]] && PRESENT_IDX[$i]=true done +echo "[INFO] Presence checked for ${#E_FILENAME[@]} package(s), $lookups through a per-package lookup" # --- metadata resolvability + fetchability, with a bounded retry window ----- # resolve a package's published Filename from a suite Packages index by sha256 diff --git a/pulp/tests/check-deb.bats b/pulp/tests/check-deb.bats new file mode 100644 index 0000000..bec6efc --- /dev/null +++ b/pulp/tests/check-deb.bats @@ -0,0 +1,111 @@ +#!/usr/bin/env bats +# Presence check of the deb verification, against stand-ins for curl and pulp. + +SHAS=( + 1111111111111111111111111111111111111111111111111111111111111111 + 2222222222222222222222222222222222222222222222222222222222222222 + 3333333333333333333333333333333333333333333333333333333333333333 +) + +setup() { + SCRIPT="${BATS_TEST_DIRNAME}/../lib/check-deb.sh" + export STUB_DIR="${BATS_TEST_TMPDIR}/stub" + mkdir -p "${STUB_DIR}/bin" + : > "${STUB_DIR}/calls" + : > "${STUB_DIR}/batch_shas" + : > "${STUB_DIR}/lookup_shas" + + # pulp stand-in: every repository exists and has a latest version + cat > "${STUB_DIR}/bin/pulp" <<'STUB' +#!/usr/bin/env bash +if [[ "$*" == *"repository show"* ]]; then + echo '{"latest_version_href": "/pulp/plugins/api/v3/repositories/deb/apt/1/versions/5/"}' +fi +STUB + + # curl stand-in: answers by url, and logs the api lookups it serves + cat > "${STUB_DIR}/bin/curl" <<'STUB' +#!/usr/bin/env bash +url="${*: -1}" +case "$url" in + *pulp_label_select=*) + echo "batch" >> "${STUB_DIR}/calls" + [[ "${BATCH_FAILS:-0}" == 1 ]] && exit 22 + jq -Rn '{count: 0, next: null, results: [inputs | {sha256: .}]}' < "${STUB_DIR}/batch_shas" + ;; + *content/deb/packages/*sha256=*) + echo "lookup" >> "${STUB_DIR}/calls" + sha=${url##*sha256=} + sha=${sha%%&*} + if grep -qx "$sha" "${STUB_DIR}/lookup_shas"; then echo '{"count": 1}'; else echo '{"count": 0}'; fi + ;; + */Release) echo "Architectures: amd64" ;; + */Packages) cat "${STUB_DIR}/Packages" ;; + *) [[ " $* " == *" -I "* ]] && printf '200 %s' "$url" ;; +esac +STUB + chmod +x "${STUB_DIR}/bin/pulp" "${STUB_DIR}/bin/curl" + + # manifest and published index of three delivered packages + local entries=() i + : > "${STUB_DIR}/Packages" + for i in 0 1 2; do + entries+=("$(jq -cn --arg f "pkg${i}_1.0_all.deb" --arg s "${SHAS[$i]}" \ + '{filename: $f, arch: "all", sha256: $s, repository: "apt-unstable", base_path: "apt-plugins-unstable", suite: "bookworm"}')") + printf 'Package: pkg%s\nFilename: pool/main/p/pkg%s/pkg%s_1.0_all.deb\nSHA256: %s\n\n' "$i" "$i" "$i" "${SHAS[$i]}" >> "${STUB_DIR}/Packages" + done + printf '%s\n' "${entries[@]}" | jq -s '{packages: .}' > "${STUB_DIR}/manifest.json" +} + +check_deb() { + run env PATH="${STUB_DIR}/bin:${PATH}" STUB_DIR="$STUB_DIR" \ + MANIFEST="${STUB_DIR}/manifest.json" MODULE_NAME=plugins DISTRIB=bookworm STABILITY=unstable \ + PULP_URL=https://pulp.invalid PULP_CONTENT_URL=https://content.invalid PULP_DOMAIN=plugins \ + PULP_TOKEN=placeholder PULP_TOKEN_ISSUED_AT="$(date +%s)" \ + METADATA_TIMEOUT=1 METADATA_INTERVAL=1 GITHUB_STEP_SUMMARY="${STUB_DIR}/summary" \ + "$@" "$SCRIPT" +} + +calls() { + grep -cx "$1" "${STUB_DIR}/calls" || true +} + +@test "delivery: one listing answers for every package of the run" { + printf '%s\n' "${SHAS[@]}" > "${STUB_DIR}/batch_shas" + check_deb CHECK_MODE=delivery GITHUB_RUN_ID=42 + [ "$status" -eq 0 ] + [ "$(calls batch)" -eq 1 ] + [ "$(calls lookup)" -eq 0 ] + [[ "$output" == *"All 3 package(s) verified"* ]] +} + +@test "delivery: a package the listing misses gets its own lookup" { + printf '%s\n' "${SHAS[0]}" "${SHAS[1]}" > "${STUB_DIR}/batch_shas" + echo "${SHAS[2]}" > "${STUB_DIR}/lookup_shas" + check_deb CHECK_MODE=delivery GITHUB_RUN_ID=42 + [ "$status" -eq 0 ] + [ "$(calls lookup)" -eq 1 ] +} + +@test "delivery: a failed listing falls back to one lookup per package" { + printf '%s\n' "${SHAS[@]}" > "${STUB_DIR}/lookup_shas" + check_deb CHECK_MODE=delivery GITHUB_RUN_ID=42 BATCH_FAILS=1 + [ "$status" -eq 0 ] + [ "$(calls lookup)" -eq 3 ] + [[ "$output" == *"falling back to one lookup per package"* ]] +} + +@test "promotion: no listing, one lookup per package" { + printf '%s\n' "${SHAS[@]}" > "${STUB_DIR}/lookup_shas" + check_deb CHECK_MODE=promote GITHUB_RUN_ID=42 + [ "$status" -eq 0 ] + [ "$(calls batch)" -eq 0 ] + [ "$(calls lookup)" -eq 3 ] +} + +@test "a package absent from the repository fails the verification" { + printf '%s\n' "${SHAS[0]}" "${SHAS[1]}" > "${STUB_DIR}/batch_shas" + check_deb CHECK_MODE=delivery GITHUB_RUN_ID=42 + [ "$status" -ne 0 ] + [[ "$output" == *"2/3 present"* ]] +} From 02d3ab3556992f8ed963c18da0df113476f0265e Mon Sep 17 00:00:00 2001 From: Kevin Duret <kduret@centreon.com> Date: Thu, 19 Nov 2026 15:33:55 +0100 Subject: [PATCH 10/15] fix(pulp): wait for queued tasks with separate queue and run budgets Port of centreon-plugins #6516: a task waiting behind another one is not failing, it gets its own budget and the tasks ahead of it are logged; a promotion only republishes when the latest version has no publication. Refs: MON-215510 --- pulp/lib/api.sh | 116 ++++++++++++++++++++++++-- pulp/promote-to-stable/promote-deb.sh | 6 +- 2 files changed, 111 insertions(+), 11 deletions(-) diff --git a/pulp/lib/api.sh b/pulp/lib/api.sh index 5c4dddb..a320662 100644 --- a/pulp/lib/api.sh +++ b/pulp/lib/api.sh @@ -118,13 +118,25 @@ pulp_upload() { # - "Worker has gone missing": the task worker pod was terminated mid-task # (scale-down, node consolidation) # Re-running the operation converges, so callers retry on rc 2. -# 0=completed, 2=retryable failure, 1=failed or timed out. +# 0=completed, 2=retryable failure, 1=failed, unexpected state or timed out. +# Separate queue/run budgets (seconds): a queued task is not failing, it can +# sit behind a task whose worker was stopped mid-task until pulp reaps it. +[[ "${PULP_TASK_WAITING_TIMEOUT:-}" =~ ^[1-9][0-9]*$ ]] || PULP_TASK_WAITING_TIMEOUT=3600 +[[ "${PULP_TASK_RUNNING_TIMEOUT:-}" =~ ^[1-9][0-9]*$ ]] || PULP_TASK_RUNNING_TIMEOUT=2700 +[[ "${PULP_TASK_BLOCKERS_LOG_INTERVAL:-}" =~ ^[1-9][0-9]*$ ]] || PULP_TASK_BLOCKERS_LOG_INTERVAL=300 +PULP_TASK_MAX_POLL_FAILURES=100 wait_task_race() { - local task_href=$1 body state error attempt - for ((attempt = 0; attempt < 600; attempt++)); do + local task_href=$1 body state error now poll_failures=0 + local started_at running_since="" next_blockers_log="$PULP_TASK_BLOCKERS_LOG_INTERVAL" + started_at=$(date +%s) + while true; do refresh_pulp_token - body=$(curl -fsSL -H "Authorization: Bearer $PULP_TOKEN" "$PULP_URL$task_href" 2>/dev/null) || body="" - state=$(echo "$body" | jq -r '.state' 2>/dev/null) || state="" + body=$(curl -fsSL --max-time 30 -H "Authorization: Bearer $PULP_TOKEN" "${PULP_URL}${task_href}" 2>/dev/null) || body="" + state=$(echo "$body" | jq -r '.state // empty' 2>/dev/null) || state="" + now=$(date +%s) + if [[ -n "$state" ]]; then + poll_failures=0 + fi case "$state" in completed) return 0 @@ -141,13 +153,72 @@ wait_task_race() { echo "::error::Task $task_href $state: $error" return 1 ;; + running | canceling) + [[ -n "$running_since" ]] || running_since="$now" + ;; + waiting) ;; + "") + poll_failures=$((poll_failures + 1)) + if ((poll_failures >= PULP_TASK_MAX_POLL_FAILURES)); then + echo "::error::Cannot poll task $task_href ($poll_failures failed polls in a row)" + return 1 + fi + ;; *) - sleep 3 + echo "::error::Task $task_href ended in unexpected state $state" + return 1 ;; esac + # a failed poll counts against the current phase + if [[ -n "$running_since" ]]; then + if ((now - running_since >= PULP_TASK_RUNNING_TIMEOUT)); then + echo "::error::Task $task_href still running after $((PULP_TASK_RUNNING_TIMEOUT / 60)) min" + return 1 + fi + elif ((now - started_at >= PULP_TASK_WAITING_TIMEOUT)); then + echo "::error::Task $task_href still waiting for its resources after $((PULP_TASK_WAITING_TIMEOUT / 60)) min" + log_task_blockers "$task_href" "$body" + return 1 + elif ((now - started_at >= next_blockers_log)); then + if ((poll_failures > 0)); then + echo "[WARN] Task $task_href: its last $poll_failures poll(s) failed" + else + echo "[INFO] Task $task_href has been waiting for $(((now - started_at) / 60)) min" + log_task_blockers "$task_href" "$body" + fi + next_blockers_log=$((next_blockers_log + PULP_TASK_BLOCKERS_LOG_INTERVAL)) + fi + sleep 3 done - echo "::error::Task $task_href did not complete in time (~30 min)" - return 1 +} + +# best effort: log the tasks ahead of a task on its reserved resources (a +# shared hold only waits for exclusive holders); never fails the caller +log_task_blockers() { + local task_href=$1 body=$2 created filter resource blockers + created=$(echo "$body" | jq -r '.pulp_created // empty' 2>/dev/null) || created="" + [[ -n "$created" ]] || return 0 + if [[ -n "$(echo "$body" | jq -r '.unblocked_at // empty' 2>/dev/null)" ]]; then + echo "[INFO] Task $task_href is unblocked and waits for a free worker" + return 0 + fi + refresh_pulp_token + while read -r filter resource; do + blockers=$( + curl -fsSL --max-time 30 -H "Authorization: Bearer $PULP_TOKEN" -G \ + --data-urlencode "$filter=$resource" \ + --data-urlencode "state__in=running,canceling,waiting" \ + --data-urlencode "pulp_created__lt=$created" \ + --data-urlencode "ordering=pulp_created" \ + --data-urlencode "fields=pulp_href,name,state,pulp_created,started_at" \ + --data-urlencode "limit=20" \ + "$PULP_URL/$PULP_DOMAIN/api/v3/tasks/" 2>/dev/null | + jq -r '.results[] | " \(.state) \(.name) (created \(.pulp_created), started \(.started_at // "-")) \(.pulp_href)"' 2>/dev/null + ) || blockers="" + echo "[INFO] Tasks ahead on $resource:" + echo "${blockers:- none visible}" + done < <(echo "$body" | jq -r '.reserved_resources_record[]? + | if startswith("shared:") then "exclusive_resources \(ltrimstr("shared:"))" else "reserved_resources \(.)" end' 2>/dev/null) } # check that a named pulp resource exists through the REST list api, @@ -237,6 +308,29 @@ content_curl() { curl -H "Authorization: Bearer $PULP_TOKEN" "$@" } +# whether the repository's latest version already has a publication (a +# structured one with --structured); any lookup failure answers no +latest_version_published() { + local plugin=$1 publication_path=$2 repository=$3 latest count="" structured=false + shift 3 + [[ "$*" == *"--structured"* ]] && structured=true + refresh_pulp_token + latest=$(pulp "$plugin" repository show --name "$repository" 2>/dev/null | jq -r '.latest_version_href // empty') || latest="" + if [[ -n "$latest" ]]; then + count=$( + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Bearer $PULP_TOKEN" -G \ + --data-urlencode "repository_version=$latest" --data-urlencode "limit=100" \ + "$PULP_URL/$PULP_DOMAIN/api/v3/publications/$publication_path/" 2>/dev/null | + jq --argjson structured "$structured" '[.results[] | select(($structured | not) or .structured == true)] | length' 2>/dev/null + ) || count="" + fi + if [[ ! "$count" =~ ^[0-9]+$ ]]; then + echo "[INFO] Cannot check the publication state of $repository, publishing" + return 1 + fi + ((count > 0)) +} + create_publication() { local plugin=$1 repository=$2 shift 2 @@ -250,6 +344,12 @@ create_publication() { *) echo "::error::create_publication: unsupported plugin $plugin" >&2; return 1 ;; esac local repo_href body_json response code body task attempt outer rc + # --skip-if-published: only if this run did not modify $repository, since a + # stale read right after a modify could return the previous, published version + if [[ "$*" == *"--skip-if-published"* ]] && latest_version_published "$plugin" "$publication_path" "$repository" "$@"; then + echo "[INFO] The latest version of $repository is already published, skipping the publication" + return 0 + fi # outer retry: the publication task itself can die server-side for # retryable reasons (task worker terminated mid-task, version race) for outer in 1 2 3; do diff --git a/pulp/promote-to-stable/promote-deb.sh b/pulp/promote-to-stable/promote-deb.sh index 522c391..124ec62 100755 --- a/pulp/promote-to-stable/promote-deb.sh +++ b/pulp/promote-to-stable/promote-deb.sh @@ -255,7 +255,7 @@ lookup_prcs() { # candidates already published in the stable suite need no new association; # when EVERYTHING already reached stable (a rerun after the modify), only the -# publication may be missing: republish and stop. +# publication may be missing: publish it if needed and stop. UNPROMOTED_HREFS=() while read -r PACKAGE; do sha=$(echo "$PACKAGE" | jq -r '.sha256') @@ -265,7 +265,7 @@ while read -r PACKAGE; do done < <(echo "$PACKAGES" | jq -c '.[]') rm -f "$STABLE_SHAS_FILE" if ((${#UNPROMOTED_HREFS[@]} == 0)); then - echo "[INFO] All $PACKAGES_COUNT package(s) are already promoted to $STABLE_SUITE; republishing only" + echo "[INFO] All $PACKAGES_COUNT package(s) are already promoted to $STABLE_SUITE; publishing only if needed" while read -r PACKAGE; do manifest_add "$(echo "$PACKAGE" | jq -c \ --arg repository "$STABLE_REPOSITORY_NAME" --arg base_path "${LEGACY_STABLE_BASE_PATH:-$PULP_STABLE_DOMAIN/$STABLE_BASE_PATH}" \ @@ -273,7 +273,7 @@ if ((${#UNPROMOTED_HREFS[@]} == 0)); then '{filename: (.relative_path | sub(".*/"; "")), name: .package, version, arch: .architecture, sha256, repository: $repository, base_path: $base_path, suite: $suite, relative_path}')" done < <(echo "$PACKAGES" | jq -c '.[]') ensure_legacy_suite_associations - create_publication deb "$STABLE_REPOSITORY_NAME" --structured + create_publication deb "$STABLE_REPOSITORY_NAME" --structured --skip-if-published echo "::notice::Packages are available with: deb $PULP_CONTENT_URL/${LEGACY_STABLE_BASE_PATH:-$PULP_STABLE_DOMAIN/$STABLE_BASE_PATH}/ $STABLE_SUITE main" manifest_write "$MODULE_NAME" "${DISTRIB:-}" "deb" "$STABILITY" "promote" "$PULP_CONTENT_URL" exit 0 From 7eb237c294f19bd80c39f563a0b9f308a75b9d20 Mon Sep 17 00:00:00 2001 From: Kevin Duret <kduret@centreon.com> Date: Thu, 19 Nov 2026 15:35:23 +0100 Subject: [PATCH 11/15] perf(pulp): run the delivery verification requests in parallel Port of centreon-plugins #6518, which the centreon copy lacked: content HEADs and per-package presence lookups run 8 at a time, each Packages index is read once into a map, and the rpm presence check follows. The served-build check of the centreon copy is kept on top of the parallel HEADs. Without it a full plugins delivery spent 8 to 9 minutes on sequential HEADs. Refs: MON-215510 --- pulp/lib/check-common.sh | 54 ++++++++++++++++++++++------- pulp/lib/check-deb.sh | 71 +++++++++++++++++++++------------------ pulp/lib/check-rpm.sh | 49 +++++++++++---------------- pulp/tests/check-deb.bats | 24 ++++++++++++- 4 files changed, 124 insertions(+), 74 deletions(-) diff --git a/pulp/lib/check-common.sh b/pulp/lib/check-common.sh index 3fa79a1..8556d38 100644 --- a/pulp/lib/check-common.sh +++ b/pulp/lib/check-common.sh @@ -19,6 +19,15 @@ switch_pulp_domain "$PULP_DOMAIN" || exit 1 # content-app cache TTL (600s), the window must survive that worst case METADATA_TIMEOUT="${METADATA_TIMEOUT:-900}" METADATA_INTERVAL="${METADATA_INTERVAL:-15}" +# concurrent per-package requests (presence lookups, content HEADs) +[[ "${CHECK_MAX_PARALLEL:-}" =~ ^[1-9][0-9]*$ ]] || CHECK_MAX_PARALLEL=8 + +# wait_for_slot — block until fewer than CHECK_MAX_PARALLEL background jobs run +wait_for_slot() { + while (($(jobs -rp | wc -l) >= CHECK_MAX_PARALLEL)); do + wait -n || true + done +} # latest_version_href <rpm|deb> <repository>: print the repository's latest version # href (empty when it has none). The lookup is retried: a transient API error must @@ -120,23 +129,43 @@ served_artifact_sha256() { fi } -# check_fetchable_and_record — HEAD each resolved package on the content url and -# record every package's result row. Uses the E_FILENAME/E_ARCH/E_BASEPATH/E_SHA256, -# PRESENT_IDX, META_IDX and RESOLVED_IDX arrays filled by the sourcing script. +# check_fetchable_and_record — HEAD each resolved package on the content url +# (CHECK_MAX_PARALLEL at a time), then record every package's result row in +# order. Uses the E_FILENAME/E_ARCH/E_BASEPATH/E_SHA256, PRESENT_IDX, META_IDX +# and RESOLVED_IDX arrays filled by the sourcing script. check_fetchable_and_record() { - local i url out code served retry_served fetchable attempt digest_deadline="" + local i url out code served retry_served fetchable attempt digest_deadline="" results_dir + results_dir=$(mktemp -d) + for i in "${!E_FILENAME[@]}"; do + [[ "${META_IDX[$i]}" == "true" ]] || continue + url="${PULP_CONTENT_URL}/${E_BASEPATH[$i]}/${RESOLVED_IDX[$i]}" + # token refreshed in this shell: a refresh inside the job would be lost + refresh_pulp_token + ( + curl -fsSL -H "Authorization: Bearer $PULP_TOKEN" -o /dev/null \ + -w '%{http_code} %{url_effective}' -I "$url" 2>/dev/null > "$results_dir/$i" || echo 000 > "$results_dir/$i" + ) & + wait_for_slot + done + wait for i in "${!E_FILENAME[@]}"; do fetchable=false if [[ "${META_IDX[$i]}" == "true" ]]; then url="${PULP_CONTENT_URL}/${E_BASEPATH[$i]}/${RESOLVED_IDX[$i]}" - # retry: one flaky HEAD out of hundreds (content-app/S3 hiccup) must not - # fail the whole verification - for attempt in 1 2 3; do - out=$(content_curl -fsSL -o /dev/null -w '%{http_code} %{url_effective}' -I "$url" 2>/dev/null || echo 000) - code=${out%% *} - [[ "$code" == "200" ]] && { fetchable=true; break; } - sleep 2 - done + out=$(cat "$results_dir/$i" 2>/dev/null || echo 000) + code=${out%% *} + if [[ "$code" == "200" ]]; then + fetchable=true + else + # retry: one flaky HEAD out of hundreds (content-app/S3 hiccup) must not + # fail the whole verification + for attempt in 2 3; do + sleep 2 + out=$(content_curl -fsSL -o /dev/null -w '%{http_code} %{url_effective}' -I "$url" 2>/dev/null || echo 000) + code=${out%% *} + [[ "$code" == "200" ]] && { fetchable=true; break; } + done + fi # the legacy front resolves pool paths stable first: a same-named build # published elsewhere is served instead of the delivered one served=$(served_artifact_sha256 "${out#* }") @@ -164,6 +193,7 @@ check_fetchable_and_record() { fi record_row "${E_FILENAME[$i]}" "${E_ARCH[$i]}" "${PRESENT_IDX[$i]}" "${META_IDX[$i]}" "$fetchable" done + rm -rf "$results_dir" } # render_summary — write the per-OS table to the step summary (and stdout), then diff --git a/pulp/lib/check-deb.sh b/pulp/lib/check-deb.sh index 4bcf016..d6eb70f 100755 --- a/pulp/lib/check-deb.sh +++ b/pulp/lib/check-deb.sh @@ -7,7 +7,7 @@ # packages under the canonical pool layout, so the published Filename is resolved # from the suite's Packages index by sha256. Architecture "all" packages are # listed under every binary-<arch>, so they are looked up across the suite's -# architectures. +# architectures. Each index is read once into a sha256 -> Filename map. # # The expected list is the manifest emitted by the delivery/promote step. The # script never exits early — see check-common.sh. @@ -66,11 +66,13 @@ if [[ "${CHECK_MODE:-delivery}" == "delivery" && -n "${GITHUB_RUN_ID:-}" ]]; the done fi -# one lookup per package the listing did not return (content that already +# one lookup per package the listing did not return, CHECK_MAX_PARALLEL at a +# time (content that already # existed keeps the labels of the run that uploaded it, and a promotion # uploads nothing), by sha256 within the repository version. --retry covers # transient 5xx, which would otherwise read as "absent". declare -A PRESENT_IDX +PRESENCE_DIR=$(mktemp -d) lookups=0 for i in "${!E_FILENAME[@]}"; do PRESENT_IDX[$i]=false @@ -81,44 +83,41 @@ for i in "${!E_FILENAME[@]}"; do continue fi lookups=$((lookups + 1)) - refresh_pulp_token url="$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/packages/?$( printf 'repository_version=%s&sha256=%s&fields=pulp_href&limit=1' \ "$(jq -rn --arg v "$version_href" '$v | @uri')" "${E_SHA256[$i]}" )" - count=$(curl -fsSL --retry 3 --retry-delay 5 \ - -H "Authorization: Bearer $PULP_TOKEN" "$url" | jq -r '.count // 0') || { - echo "[WARN] presence lookup failed for ${E_FILENAME[$i]} in ${E_REPOSITORY[$i]} ($url)" >&2 - continue - } - [[ "${count:-0}" -gt 0 ]] && PRESENT_IDX[$i]=true + # token refreshed in this shell: a refresh inside the job would be lost + refresh_pulp_token + ( + count=$(curl -fsSL --retry 3 --retry-delay 5 \ + -H "Authorization: Bearer $PULP_TOKEN" "$url" | jq -r '.count // 0') || { + echo "[WARN] presence lookup failed for ${E_FILENAME[$i]} in ${E_REPOSITORY[$i]} ($url)" >&2 + exit 0 + } + [[ "${count:-0}" -gt 0 ]] && touch "$PRESENCE_DIR/$i" + ) & + wait_for_slot done +wait +for i in "${!E_FILENAME[@]}"; do + [[ -e "$PRESENCE_DIR/$i" ]] && PRESENT_IDX[$i]=true +done +rm -rf "$PRESENCE_DIR" echo "[INFO] Presence checked for ${#E_FILENAME[@]} package(s), $lookups through a per-package lookup" # --- metadata resolvability + fetchability, with a bounded retry window ----- -# resolve a package's published Filename from a suite Packages index by sha256 -resolve_filename() { - # read the Packages index from a file: awk exits on the first match, and a - # pipe writer would take a SIGPIPE ("printf: write error: Broken pipe" noise) - local packages_file=$1 sha=$2 - [[ -s "$packages_file" ]] || return 0 - awk -v sha="$sha" ' - BEGIN { RS = ""; FS = "\n" } - index($0, "SHA256: " sha) { - for (i = 1; i <= NF; i++) if ($i ~ /^Filename: /) { sub(/^Filename: /, "", $i); print $i; exit } - }' "$packages_file" -} - declare -A META_IDX # idx -> true|false declare -A RESOLVED_IDX # idx -> published Filename for i in "${!E_FILENAME[@]}"; do META_IDX[$i]=false; done -# one resolution round: fetch each suite's Packages indexes once, then resolve -# each pending package's published Filename by sha256 +# one resolution round: read each suite's Packages indexes once into a +# sha256 -> Filename map, then resolve each pending package from it resolve_pending() { - local -A pkg_cache=() # key: base_path|suite|arch -> Packages index file + local -A indexed=() # key: base_path|suite|arch -> set once read + local -A filename_by=() # key: base_path|suite|arch|sha256 -> Filename local -A arches_cache=() # key: base_path|suite -> space separated arches - local all_resolved=true i base_path suite arch search_arches sk ck a filename cache_file + local all_resolved=true i base_path suite arch search_arches sk ck a filename sha fname for i in "${!E_FILENAME[@]}"; do [[ "${META_IDX[$i]}" == "true" ]] && continue base_path=${E_BASEPATH[$i]}; suite=${E_SUITE[$i]}; arch=${E_ARCH[$i]} @@ -138,12 +137,21 @@ resolve_pending() { filename="" for a in $search_arches; do ck="$base_path|$suite|$a" - if [[ -z "${pkg_cache[$ck]+set}" ]]; then - cache_file=$(mktemp) - content_curl -fsSL "$PULP_CONTENT_URL/$base_path/dists/$suite/main/binary-$a/Packages" 2>/dev/null > "$cache_file" || true - pkg_cache[$ck]=$cache_file + if [[ -z "${indexed[$ck]+set}" ]]; then + indexed[$ck]=1 + while IFS=$'\t' read -r sha fname; do + filename_by["$ck|$sha"]=$fname + done < <(content_curl -fsSL "$PULP_CONTENT_URL/$base_path/dists/$suite/main/binary-$a/Packages" 2>/dev/null | + awk 'BEGIN { RS = ""; FS = "\n" } { + sha = ""; fname = "" + for (i = 1; i <= NF; i++) { + if ($i ~ /^SHA256: /) sha = substr($i, 9) + else if ($i ~ /^Filename: /) fname = substr($i, 11) + } + if (sha != "" && fname != "") print sha "\t" fname + }') fi - filename=$(resolve_filename "${pkg_cache[$ck]}" "${E_SHA256[$i]}") + filename=${filename_by["$ck|${E_SHA256[$i]}"]:-} [[ -n "$filename" ]] && break done @@ -154,7 +162,6 @@ resolve_pending() { all_resolved=false fi done - rm -f "${pkg_cache[@]}" [[ "$all_resolved" == "true" ]] } diff --git a/pulp/lib/check-rpm.sh b/pulp/lib/check-rpm.sh index 2369da2..f908fc8 100755 --- a/pulp/lib/check-rpm.sh +++ b/pulp/lib/check-rpm.sh @@ -95,45 +95,37 @@ declare -A META_IDX # idx -> true|false declare -A RESOLVED_IDX # idx -> published location href for i in "${!E_FILENAME[@]}"; do META_IDX[$i]=false; done -# resolve the published location href of the package whose sha256 checksum -# matches, from a primary.xml body: matching by checksum (not filename) binds -# the verification to the exact delivered content, like the DEB by-sha256 match -resolve_href() { - # read the primary.xml from a file: awk exits on the first match, and a pipe - # writer would take a SIGPIPE ("printf: write error: Broken pipe" noise) - local primary_file=$1 sha=$2 - [[ -s "$primary_file" ]] || return 0 - awk -v sha="$sha" ' - BEGIN { RS = "</package>" } - index($0, ">" sha "</checksum>") { - if (match($0, /<location[^>]*href="[^"]+"/)) { - s = substr($0, RSTART, RLENGTH) - sub(/.*href="/, "", s); sub(/"$/, "", s) - print s; exit - } - }' "$primary_file" -} - -# one resolution round: fetch each base_path's primary.xml once, then resolve -# each pending package's published href by sha256 +# one resolution round: read each base_path's primary.xml once into a +# sha256 -> location href map, then resolve each pending package from it; +# matching by checksum (not filename) binds the verification to the exact +# delivered content, like the DEB by-sha256 match resolve_pending() { - local -A primary_cache=() - local all_resolved=true i base_path repomd primary_href href cache_file + local -A indexed=() # key: base_path -> set once read + local -A href_by=() # key: base_path|sha256 -> location href + local all_resolved=true i base_path repomd primary_href href sha for i in "${!E_FILENAME[@]}"; do [[ "${META_IDX[$i]}" == "true" ]] && continue base_path=${E_BASEPATH[$i]} - if [[ -z "${primary_cache[$base_path]+set}" ]]; then - cache_file=$(mktemp) + if [[ -z "${indexed[$base_path]+set}" ]]; then + indexed[$base_path]=1 repomd=$(content_curl -fsSL "$PULP_CONTENT_URL/$base_path/repodata/repomd.xml" 2>/dev/null || true) primary_href=$(printf '%s' "$repomd" | grep -oP '<location href="\K[^"]+primary\.xml[^"]*' | head -1 || true) if [[ -n "$primary_href" ]]; then - content_curl -fsSL "$PULP_CONTENT_URL/$base_path/$primary_href" 2>/dev/null | gunzip -c 2>/dev/null > "$cache_file" || true + while IFS=$'\t' read -r sha href; do + href_by["$base_path|$sha"]=$href + done < <(content_curl -fsSL "$PULP_CONTENT_URL/$base_path/$primary_href" 2>/dev/null | gunzip -c 2>/dev/null | + awk 'BEGIN { RS = "</package>" } { + if (!match($0, /<checksum[^>]*>[0-9a-f]+<\/checksum>/)) next + sha = substr($0, RSTART, RLENGTH); sub(/<\/checksum>$/, "", sha); sub(/.*>/, "", sha) + if (!match($0, /<location[^>]*href="[^"]+"/)) next + href = substr($0, RSTART, RLENGTH); sub(/.*href="/, "", href); sub(/"$/, "", href) + print sha "\t" href + }') fi - primary_cache[$base_path]=$cache_file fi - href=$(resolve_href "${primary_cache[$base_path]}" "${E_SHA256[$i]}") + href=${href_by["$base_path|${E_SHA256[$i]}"]:-} if [[ -n "$href" ]]; then META_IDX[$i]=true RESOLVED_IDX[$i]=$href @@ -141,7 +133,6 @@ resolve_pending() { all_resolved=false fi done - rm -f "${primary_cache[@]}" [[ "$all_resolved" == "true" ]] } diff --git a/pulp/tests/check-deb.bats b/pulp/tests/check-deb.bats index bec6efc..d7221b0 100644 --- a/pulp/tests/check-deb.bats +++ b/pulp/tests/check-deb.bats @@ -41,7 +41,13 @@ case "$url" in ;; */Release) echo "Architectures: amd64" ;; */Packages) cat "${STUB_DIR}/Packages" ;; - *) [[ " $* " == *" -I "* ]] && printf '200 %s' "$url" ;; + *) + if [[ " $* " == *" -I "* ]]; then + echo "head" >> "${STUB_DIR}/calls" + [[ -n "${UNFETCHABLE:-}" && "$url" == *"${UNFETCHABLE}"* ]] && exit 22 + printf '200 %s' "$url" + fi + ;; esac STUB chmod +x "${STUB_DIR}/bin/pulp" "${STUB_DIR}/bin/curl" @@ -109,3 +115,19 @@ calls() { [ "$status" -ne 0 ] [[ "$output" == *"2/3 present"* ]] } + +@test "every package is fetched once when the content url answers" { + printf '%s\n' "${SHAS[@]}" > "${STUB_DIR}/batch_shas" + check_deb CHECK_MODE=delivery GITHUB_RUN_ID=42 + [ "$status" -eq 0 ] + [ "$(calls head)" -eq 3 ] +} + +@test "a package that cannot be fetched is retried, then fails the verification" { + printf '%s\n' "${SHAS[@]}" > "${STUB_DIR}/batch_shas" + check_deb CHECK_MODE=delivery GITHUB_RUN_ID=42 UNFETCHABLE=pkg1_1.0 + [ "$status" -ne 0 ] + [ "$(calls head)" -eq 5 ] + [[ "$output" == *"pkg1_1.0_all.deb"*"fetchable=❌"* ]] + [[ "$output" == *"pkg0_1.0_all.deb"*"fetchable=✅"* ]] +} From d7cea72e5fac8d098cc2c1571fad490d2126f0ce Mon Sep 17 00:00:00 2001 From: Kevin Duret <kduret@centreon.com> Date: Thu, 19 Nov 2026 16:07:28 +0100 Subject: [PATCH 12/15] ci: harden the release tooling and test the pulp task wait Validate a release tag before it reaches an api path, check that the scripts an action runs exist and are executable, serialize the release and the major tag move, run every step with pipefail, and skip the release until its app is configured. Document that refactor and revert changes are released, as release-please does. Add bats tests of the pulp task wait. Refs: MON-215510 --- .github/scripts/check-actions.sh | 10 +++ .github/scripts/tests/check-actions.bats | 16 ++++ .github/scripts/tests/update-major-tags.bats | 10 ++- .github/scripts/update-major-tags.sh | 22 ++++-- .github/workflows/lint.yml | 4 + .github/workflows/move-major-tag.yml | 9 +++ .github/workflows/release.yml | 9 ++- README.md | 25 +++--- pulp/package-delivery/action.yml | 2 +- pulp/promote-to-stable/action.yml | 4 +- pulp/setup-cli/action.yml | 2 +- pulp/tests/api.bats | 81 ++++++++++++++++++++ 12 files changed, 171 insertions(+), 23 deletions(-) create mode 100644 pulp/tests/api.bats diff --git a/.github/scripts/check-actions.sh b/.github/scripts/check-actions.sh index ed42f4a..feb6912 100755 --- a/.github/scripts/check-actions.sh +++ b/.github/scripts/check-actions.sh @@ -7,6 +7,7 @@ # an action of the same package is called with "uses: $/<package>/..."; # - no path or "$/" reference leaving the package: a change there would not # release it; +# - every script an action runs exists and is executable; # - every remote action is pinned to a full commit SHA with a version comment. # Usage: check-actions.sh [repository root] set -euo pipefail @@ -48,6 +49,15 @@ for action_file in "${action_files[@]}"; do error "$action_file" "line ${number}: '${ref}' must be pinned to a full commit SHA followed by a '# vX.Y.Z' comment" fi done < <(grep -nE '^[[:space:]-]*uses:' "$action_file" || true) + + while IFS= read -r script; do + script_path="${action_file%/*}/${script}" + if [[ ! -f "$script_path" ]]; then + error "$action_file" "script '${script}' does not exist" + elif [[ ! -x "$script_path" ]]; then + error "$action_file" "script '${script}' is not executable" + fi + done < <(grep -oE 'github\.action_path \}\}/[^[:space:]"]+' "$action_file" | sed 's|^github\.action_path }}/||' | sort -u || true) done while IFS= read -r package; do diff --git a/.github/scripts/tests/check-actions.bats b/.github/scripts/tests/check-actions.bats index 512bb7f..0f0c627 100644 --- a/.github/scripts/tests/check-actions.bats +++ b/.github/scripts/tests/check-actions.bats @@ -5,6 +5,8 @@ setup() { REPO="${BATS_TEST_TMPDIR}/repo" mkdir -p "${REPO}/tool/run" "${REPO}/tool/lib" echo '{"packages": {"tool": {"component": "tool"}}}' > "${REPO}/release-please-config.json" + printf '#!/bin/sh\n' > "${REPO}/tool/lib/run.sh" + chmod +x "${REPO}/tool/lib/run.sh" action 'uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5' \ 'run: ${{ github.action_path }}/../lib/run.sh' } @@ -85,3 +87,17 @@ action() { [ "$status" -ne 0 ] [[ "$output" == *"package 'ghost' holds no action.yml"* ]] } + +@test "a script that does not exist fails" { + action 'run: ${{ github.action_path }}/missing.sh' + run "$SCRIPT" "$REPO" + [ "$status" -ne 0 ] + [[ "$output" == *"script 'missing.sh' does not exist"* ]] +} + +@test "a script that is not executable fails" { + chmod -x "${REPO}/tool/lib/run.sh" + run "$SCRIPT" "$REPO" + [ "$status" -ne 0 ] + [[ "$output" == *"is not executable"* ]] +} diff --git a/.github/scripts/tests/update-major-tags.bats b/.github/scripts/tests/update-major-tags.bats index a305270..59c3aae 100644 --- a/.github/scripts/tests/update-major-tags.bats +++ b/.github/scripts/tests/update-major-tags.bats @@ -71,8 +71,10 @@ releases() { ! grep -qE -- "-X (PATCH|POST)" "$GH_CALLS" } -@test "rollback: a major tag is not a release tag" { - run "$SCRIPT" pulp/v1 - [ "$status" -ne 0 ] - ! grep -qE -- "-X (PATCH|POST)" "$GH_CALLS" +@test "rollback: anything but a release tag is refused before any api call" { + for tag in pulp/v1 v1.0.0 "pulp/v1.0.0/../../x" "../pulp/v1.0.0" "Pulp/v1.0.0"; do + run "$SCRIPT" "$tag" + [ "$status" -ne 0 ] || { echo "accepted: ${tag}"; return 1; } + done + [ ! -s "$GH_CALLS" ] } diff --git a/.github/scripts/update-major-tags.sh b/.github/scripts/update-major-tags.sh index a74f2fd..19485ec 100755 --- a/.github/scripts/update-major-tags.sh +++ b/.github/scripts/update-major-tags.sh @@ -7,14 +7,24 @@ # Environment: GITHUB_REPOSITORY, GH_TOKEN. set -euo pipefail +RELEASE_TAG_PATTERN='^([a-z0-9][a-z0-9-]*)/v([0-9]+)\.[0-9]+\.[0-9]+$' + +# print the major tag of a release tag, fail on anything else +major_tag_of() { + if [[ ! "$1" =~ $RELEASE_TAG_PATTERN ]]; then + echo "::error::'$1' is not a release tag (<package>/vX.Y.Z)" >&2 + return 1 + fi + echo "${BASH_REMATCH[1]}/v${BASH_REMATCH[2]}" +} + move_major_tag() { local tag=$1 sha=$2 major_tag - if [[ ! "$sha" =~ ^[0-9a-f]{40}$ || ! "$tag" =~ ^(.+)/v([0-9]+)\.[0-9]+\.[0-9]+$ ]]; then - echo "::error::unexpected release: tag '${tag}', sha '${sha}'" - exit 1 + major_tag=$(major_tag_of "$tag") + if [[ ! "$sha" =~ ^[0-9a-f]{40}$ ]]; then + echo "::error::unexpected commit '${sha}' for the release ${tag}" + return 1 fi - # BASH_REMATCH holds the tag match: it is the last regex evaluated above - major_tag="${BASH_REMATCH[1]}/v${BASH_REMATCH[2]}" if gh api "repos/${GITHUB_REPOSITORY}/git/ref/tags/${major_tag}" > /dev/null 2>&1; then gh api -X PATCH "repos/${GITHUB_REPOSITORY}/git/refs/tags/${major_tag}" -f "sha=${sha}" -F force=true > /dev/null @@ -27,6 +37,8 @@ move_major_tag() { if (($# > 0)); then for tag in "$@"; do + # validated before it reaches an api path: the tag is typed by a person + major_tag_of "$tag" > /dev/null if ! sha=$(gh api "repos/${GITHUB_REPOSITORY}/commits/tags/${tag}" --jq .sha); then echo "::error::release tag '${tag}' does not exist" exit 1 diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 84f985a..c015e66 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -12,6 +12,10 @@ on: permissions: {} +defaults: + run: + shell: bash + env: JACTIONLINT_VERSION: "2.6.0" BATS_VERSION: "1.13.0" diff --git a/.github/workflows/move-major-tag.yml b/.github/workflows/move-major-tag.yml index 0106657..aede058 100644 --- a/.github/workflows/move-major-tag.yml +++ b/.github/workflows/move-major-tag.yml @@ -13,6 +13,15 @@ on: permissions: {} +# shared with release: tags are never moved by two runs at once +concurrency: + group: release-tags + cancel-in-progress: false + +defaults: + run: + shell: bash + jobs: move-major-tag: name: move-major-tag diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c2b08a3..8263aa5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -4,8 +4,9 @@ name: release # CHANGELOG.md); merging it creates the <package>/vX.Y.Z tag and its GitHub # release, then the floating <package>/vX tag is moved to it. See the README. +# shared with move-major-tag: tags are never moved by two runs at once concurrency: - group: ${{ github.workflow }} + group: release-tags cancel-in-progress: false on: @@ -15,9 +16,15 @@ on: permissions: {} +defaults: + run: + shell: bash + jobs: release: name: release + # nothing to do until the release app is configured (see the README) + if: vars.RELEASE_APP_CLIENT_ID != '' runs-on: ubuntu-24.04 timeout-minutes: 10 # the environment only allows main and holds the release app key diff --git a/README.md b/README.md index 327feb3..97a083e 100644 --- a/README.md +++ b/README.md @@ -51,12 +51,16 @@ Each package follows [Semantic Versioning](https://semver.org/). Releases are ma [release-please](https://github.com/googleapis/release-please) from the titles of the pull requests merged on `main` (squash merge, so the title is the commit): -| Change to a package | Pull request title | Release | -| ---------------------------------------------------------------------------------------------- | ---------------------------- | ------- | -| Removed or renamed input/output, new required input, changed default or behavior a caller sees | `feat(pulp)!: …` (or `fix!`) | major | -| New action, new optional input or output | `feat(pulp): …` | minor | -| Bug fix, pinned dependency bump, performance | `fix(pulp): …`, `perf: …` | patch | -| Refactoring, documentation, tests, CI | `refactor:`, `docs:`, `ci:`… | none | +| Change to a package | Pull request title | Release | +| ---------------------------------------------------------------------------------------------- | ----------------------------------- | ------- | +| Removed or renamed input/output, new required input, changed default or behavior a caller sees | `feat(pulp)!: …` (or `fix!`) | major | +| New action, new optional input or output | `feat(pulp): …` | minor | +| Bug fix, pinned dependency bump, performance, refactoring, revert | `fix`, `perf`, `refactor`, `revert` | patch | +| Documentation, tests, CI, chores | `docs`, `test`, `ci`, `chore` | none | + +Anything that changes what an action runs needs a releasing type, or callers never get +it: `docs`, `test`, `ci` and `chore` changes to a package only ship with its next +release. A commit counts for the packages whose directory it changes. @@ -87,6 +91,9 @@ workflow with the previous release (for example `pulp/v1.4.1`): the major tag po back at it within seconds. Then fix or revert on `main`; the next release moves the tag forward again. +The same workflow repairs a release whose tag was created but whose major tag was not +moved (a failed `release` run): give it the new release. + ## Adding or changing an action - Put a new action in an existing package when it shares code with it, otherwise create @@ -164,9 +171,9 @@ deliver-pulp: Kept outside the code, set by a repository admin: - **Release app**: a GitHub App installed on this repository only, with `contents`, - `pull-requests` and `issues` write access; its client id in the - `RELEASE_APP_CLIENT_ID` variable and its private key in the `RELEASE_APP_PRIVATE_KEY` - secret of the `release` environment. + `pull-requests` and `issues` write access; its client id in the `RELEASE_APP_CLIENT_ID` + repository variable and its private key in the `RELEASE_APP_PRIVATE_KEY` secret of the + `release` environment. The `release` workflow does nothing until the variable is set. - **`release` environment**: deployment branches limited to `main`. - **`Release tags` ruleset**: the release app is the only bypass actor, so no one else can create, move or delete a tag. diff --git a/pulp/package-delivery/action.yml b/pulp/package-delivery/action.yml index 9eea6e4..9807203 100644 --- a/pulp/package-delivery/action.yml +++ b/pulp/package-delivery/action.yml @@ -1,4 +1,4 @@ -name: "package-delivery" +name: "Pulp package delivery" description: "Deliver rpm and deb packages to pulp" inputs: module_name: diff --git a/pulp/promote-to-stable/action.yml b/pulp/promote-to-stable/action.yml index af64194..8a6fb71 100644 --- a/pulp/promote-to-stable/action.yml +++ b/pulp/promote-to-stable/action.yml @@ -1,5 +1,5 @@ -name: "promote testing to stable" -description: "Promote testing packages to stable." +name: "Pulp promote to stable" +description: "Promote testing packages to the stable pulp repository" inputs: module_name: description: "Module name" diff --git a/pulp/setup-cli/action.yml b/pulp/setup-cli/action.yml index 26ccde7..48c0850 100644 --- a/pulp/setup-cli/action.yml +++ b/pulp/setup-cli/action.yml @@ -1,4 +1,4 @@ -name: "setup-pulp-cli" +name: "Pulp setup-cli" description: "Install pulp-cli and authenticate to pulp with a GitHub OIDC token" inputs: pulp_url: diff --git a/pulp/tests/api.bats b/pulp/tests/api.bats new file mode 100644 index 0000000..f5fe3d4 --- /dev/null +++ b/pulp/tests/api.bats @@ -0,0 +1,81 @@ +#!/usr/bin/env bats +# Outcome of a pulp task as the delivery scripts read it, against a curl +# stand-in that serves a scripted sequence of task states. + +setup() { + export STUB_DIR="${BATS_TEST_TMPDIR}/stub" + mkdir -p "${STUB_DIR}/bin" + # each poll consumes the first line of the sequence; the last one repeats + cat > "${STUB_DIR}/bin/curl" <<'STUB' +#!/usr/bin/env bash +body=$(head -n 1 "${STUB_DIR}/tasks") +if (($(wc -l < "${STUB_DIR}/tasks") > 1)); then + sed -i 1d "${STUB_DIR}/tasks" +fi +[[ "$body" == "unreachable" ]] && exit 7 +echo "$body" +STUB + chmod +x "${STUB_DIR}/bin/curl" +} + +# wait_task <budget env...> : poll a task with the scripted states, no real sleep +wait_task() { + run env PATH="${STUB_DIR}/bin:${PATH}" STUB_DIR="$STUB_DIR" PULP_URL=https://pulp.invalid PULP_DOMAIN=plugins \ + PULP_TOKEN=placeholder PULP_TOKEN_ISSUED_AT="$(date +%s)" "$@" \ + bash -c 'source "$1"; sleep() { command sleep 0.2; }; wait_task_race /pulp/api/v3/tasks/1/' _ "${BATS_TEST_DIRNAME}/../lib/api.sh" +} + +states() { + printf '%s\n' "$@" > "${STUB_DIR}/tasks" +} + +@test "a completed task succeeds" { + states '{"state": "completed"}' + wait_task + [ "$status" -eq 0 ] +} + +@test "a task is polled through waiting and running until it completes" { + states '{"state": "waiting"}' '{"state": "running"}' '{"state": "completed"}' + wait_task + [ "$status" -eq 0 ] +} + +@test "a failed task fails with its error" { + states '{"state": "failed", "error": {"description": "bad package"}}' + wait_task + [ "$status" -eq 1 ] + [[ "$output" == *"bad package"* ]] +} + +@test "a lost repository-version race is reported as retryable" { + states '{"state": "failed", "error": {"description": "duplicate key value violates core_repositoryversion"}}' + wait_task + [ "$status" -eq 2 ] +} + +@test "a task whose worker went missing is reported as retryable" { + states '{"state": "canceled", "reason": "Worker has gone missing"}' + wait_task + [ "$status" -eq 2 ] +} + +@test "a failed poll does not fail the task" { + states unreachable '{"state": "completed"}' + wait_task + [ "$status" -eq 0 ] +} + +@test "a task that stays queued fails once its waiting budget is spent" { + states '{"state": "waiting", "pulp_created": "2026-01-01T00:00:00Z", "unblocked_at": "2026-01-01T00:00:01Z"}' + wait_task PULP_TASK_WAITING_TIMEOUT=1 + [ "$status" -eq 1 ] + [[ "$output" == *"still waiting for its resources"* ]] +} + +@test "a task that keeps running fails once its running budget is spent" { + states '{"state": "running"}' + wait_task PULP_TASK_RUNNING_TIMEOUT=1 + [ "$status" -eq 1 ] + [[ "$output" == *"still running"* ]] +} From ded1b1544645970f5b5275f724cb6b4bba12d9e3 Mon Sep 17 00:00:00 2001 From: Kevin Duret <kduret@centreon.com> Date: Thu, 19 Nov 2026 16:55:04 +0100 Subject: [PATCH 13/15] fix(pulp): recover an interrupted deb promotion on rerun A deb promotion whose publication failed after the repository modify could not be rerun: the suite association already existed, so the release component could not be deduced, and the fallback listing of release components is refused (403) to the CI identity, which aborted the script before the recovery that republishes and re-checks the suite. An empty result aborted too, on a grep under pipefail. Both now reach the recovery. Add bats tests of that rerun. Refs: MON-215510 --- pulp/promote-to-stable/promote-deb.sh | 9 +- pulp/tests/promote-deb.bats | 121 ++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 pulp/tests/promote-deb.bats diff --git a/pulp/promote-to-stable/promote-deb.sh b/pulp/promote-to-stable/promote-deb.sh index 124ec62..6bab9ea 100755 --- a/pulp/promote-to-stable/promote-deb.sh +++ b/pulp/promote-to-stable/promote-deb.sh @@ -156,7 +156,7 @@ ensure_legacy_suite_associations() { printf 'repository_version=%s&distribution=%s&component=main&limit=1' \ "$(jq -rn --arg v "$latest" '$v | @uri')" "$(jq -rn --arg v "$STABLE_LEGACY_SUITE" '$v | @uri')" )" | jq -r '.results[0].pulp_href // empty' - ) + ) || legacy_rc="" if [[ -z "$legacy_rc" ]]; then echo "::error::Cannot resolve the $STABLE_LEGACY_SUITE/main release component of $STABLE_LEGACY_REPOSITORY_NAME (provisioned by create-repos)" return 1 @@ -358,15 +358,18 @@ if ((${#BATCH_PACKAGES[@]} > 0)); then if [[ $(echo "$STABLE_RC_SET" | grep -c .) -ne 1 ]]; then refresh_pulp_token STABLE_LATEST=$(pulp deb repository show --name "$STABLE_REPOSITORY_NAME" | jq -r '.latest_version_href') + # this listing is refused (403) to non-admin identities: a failure must + # not abort, the interrupted-promotion recovery below handles it STABLE_RC_SET=$( curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Bearer $PULP_TOKEN" \ "$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/release_components/?$( printf 'repository_version=%s&distribution=%s&component=main&limit=1' \ "$(jq -rn --arg v "$STABLE_LATEST" '$v | @uri')" "$(jq -rn --arg v "$STABLE_SUITE" '$v | @uri')" )" | jq -r '.results[0].pulp_href // empty' - ) + ) || STABLE_RC_SET="" fi - STABLE_RC=$(echo "$STABLE_RC_SET" | grep . | head -1) + # first line; a grep on an empty set would abort under pipefail + STABLE_RC="${STABLE_RC_SET%%$'\n'*}" if [[ -z "$STABLE_RC" ]]; then # a promotion interrupted after its repository modify only misses the # publication: republish and re-check every candidate before giving up diff --git a/pulp/tests/promote-deb.bats b/pulp/tests/promote-deb.bats new file mode 100644 index 0000000..530dc44 --- /dev/null +++ b/pulp/tests/promote-deb.bats @@ -0,0 +1,121 @@ +#!/usr/bin/env bats +# Rerun of a deb promotion interrupted after its repository modify, against +# stand-ins for curl and pulp. + +SHAS=( + 1111111111111111111111111111111111111111111111111111111111111111 + 2222222222222222222222222222222222222222222222222222222222222222 +) + +setup() { + SCRIPT="${BATS_TEST_DIRNAME}/../promote-to-stable/promote-deb.sh" + export STUB_DIR="${BATS_TEST_TMPDIR}/stub" + mkdir -p "${STUB_DIR}/bin" + : > "${STUB_DIR}/calls" + : > "${STUB_DIR}/stable_Packages" + + # testing index and api listing of two packages of the module + local i + : > "${STUB_DIR}/testing_Packages" + : > "${STUB_DIR}/listing" + for i in 0 1; do + printf 'Package: pkg%s\nFilename: pool/pkg%s_1.0_all.deb\nSHA256: %s\n\n' "$i" "$i" "${SHAS[$i]}" \ + >> "${STUB_DIR}/testing_Packages" + jq -cn --arg n "pkg${i}" --arg s "${SHAS[$i]}" --arg h "/content/packages/${i}/" \ + '{pulp_href: $h, package: $n, version: "1.0", architecture: "all", sha256: $s, relative_path: ("pool/" + $n + "_1.0_all.deb")}' \ + >> "${STUB_DIR}/listing" + done + + cat > "${STUB_DIR}/bin/pulp" <<'STUB' +#!/usr/bin/env bash +if [[ "$*" == *"repository show"* ]]; then + echo '{"pulp_href": "/repositories/1/", "latest_version_href": "/repositories/1/versions/5/"}' +fi +STUB + + # curl stand-in: answers by url; the suite associations already exist (the + # interrupted run added them) and only a publication makes stable serve them + cat > "${STUB_DIR}/bin/curl" <<'STUB' +#!/usr/bin/env bash +url="${*: -1}" +out="" code=0 body="" +args=("$@") +for ((i = 0; i < ${#args[@]}; i++)); do + [[ "${args[$i]}" == "-o" ]] && out="${args[$((i + 1))]}" + [[ "${args[$i]}" == "-w" ]] && code=1 +done +status=200 +case "$url" in + */dists/*/Release) body="Architectures: all" ;; + */dists/testing-suite/*/Packages) body=$(cat "${STUB_DIR}/testing_Packages") ;; + */dists/stable-suite/*/Packages) body=$(cat "${STUB_DIR}/stable_Packages") ;; + */pool/*) body="deb" ;; + */api/v3/repositories/deb/apt/) body='{"count": 1}' ;; + */content/deb/release_components/\?*) + echo "release_components" >> "${STUB_DIR}/calls" + echo "curl: (22) The requested URL returned error: 403" >&2 + exit 22 + ;; + */content/deb/package_release_components/) + body='{"results": [{"release_component": "/content/release_components/1/"}]}' + ;; + */content/deb/packages/\?*) body=$(jq -sc '{next: null, results: .}' "${STUB_DIR}/listing") ;; + */content/deb/packages/) + if [[ " $* " == *" -F "* ]]; then + echo "upload" >> "${STUB_DIR}/calls" + body='{"task": "/tasks/upload/"}' + status=202 + else + body='{"results": [{"pulp_href": "/content/packages/0/"}]}' + fi + ;; + */publications/deb/apt/) + echo "publish" >> "${STUB_DIR}/calls" + [[ "${PUBLISH_SERVES:-1}" == 1 ]] && cp "${STUB_DIR}/testing_Packages" "${STUB_DIR}/stable_Packages" + body='{"task": "/tasks/publish/"}' + status=202 + ;; + */tasks/*) body='{"state": "completed"}' ;; + *) + echo "unexpected ${url}" >> "${STUB_DIR}/calls" + exit 22 + ;; +esac +if [[ -n "$out" ]]; then + printf '%s\n' "$body" > "$out" + [[ "$code" == 1 ]] && printf '%s' "$status" +elif [[ "$code" == 1 ]]; then + printf '%s\n%s' "$body" "$status" +else + printf '%s\n' "$body" +fi +exit 0 +STUB + chmod +x "${STUB_DIR}/bin/pulp" "${STUB_DIR}/bin/curl" + + export PATH="${STUB_DIR}/bin:${PATH}" + export GITHUB_WORKSPACE="${BATS_TEST_TMPDIR}" + export PULP_URL="https://pulp.invalid" PULP_CONTENT_URL="https://content.invalid" + export PULP_TOKEN="token" PULP_DOMAIN="business" PULP_STABLE_DOMAIN="business" + export MODULE_NAME="mod" DISTRIB="trixie" STABILITY="stable" + export REPOSITORY_NAME="apt-testing" BASE_PATH="apt-testing" TESTING_SUITE="testing-suite" + export STABLE_REPOSITORY_NAME="apt-stable" STABLE_BASE_PATH="apt-stable" STABLE_SUITE="stable-suite" + cd "${BATS_TEST_TMPDIR}" +} + +@test "an interrupted promotion is republished when the release component listing is refused" { + run "$SCRIPT" + [ "$status" -eq 0 ] + [[ "$output" == *"the interrupted promotion only missed the publication"* ]] + grep -qx "release_components" "${STUB_DIR}/calls" + [ "$(grep -cx "publish" "${STUB_DIR}/calls")" -eq 1 ] + ! grep -q "^unexpected" "${STUB_DIR}/calls" + [ "$(jq '.packages | length' "${BATS_TEST_TMPDIR}/pulp-delivery-manifest.json")" -eq 2 ] +} + +@test "the promotion fails when the republished suite still misses packages" { + PUBLISH_SERVES=0 run "$SCRIPT" + [ "$status" -eq 1 ] + [[ "$output" == *"2 candidate package(s) are missing from the published stable-suite suite"* ]] + ! grep -q "^unexpected" "${STUB_DIR}/calls" +} From c43ec2632cac0a1f1a8a80efe7aba446c83004be Mon Sep 17 00:00:00 2001 From: Kevin Duret <kduret@centreon.com> Date: Thu, 19 Nov 2026 17:07:44 +0100 Subject: [PATCH 14/15] fix(pulp): cover the code specific to the maintained branches The actions were imported from develop, but the maintained branches of centreon, centreon-collect and centreon-modules carry code develop lacks. Port it so every branch can switch: the 24.x legacy mirror of dev-24.10.x, which deduces the release component without the admin-only listing; the 24.04 routing of dev-26.10.x to the legacy stable repositories; and feature repositories that already exist no longer need the creation token, which dev-25.10.x and dev-24.10.x callers do not pass. Refs: MON-215510 --- pulp/lib/repository-properties.sh | 4 +- pulp/package-delivery/action.yml | 2 +- .../ensure-feature-repository.sh | 12 +-- pulp/promote-to-stable/promote-deb.sh | 76 ++++++++++++++++--- pulp/tests/ensure-feature-repository.bats | 61 +++++++++++++++ pulp/tests/promote-deb.bats | 20 ++++- pulp/tests/repository-properties.bats | 16 ++++ 7 files changed, 172 insertions(+), 19 deletions(-) create mode 100644 pulp/tests/ensure-feature-repository.bats diff --git a/pulp/lib/repository-properties.sh b/pulp/lib/repository-properties.sh index 3cb1f0a..b536cfa 100755 --- a/pulp/lib/repository-properties.sh +++ b/pulp/lib/repository-properties.sh @@ -205,10 +205,10 @@ else STABLE_REPOSITORY_NAME="${DEB_PREFIX}${DEB_INFIX}stable" STABLE_BASE_PATH="$STABLE_REPOSITORY_NAME" STABLE_SUITE="$DISTRIB-$VERSION-stable" - # 24.10 clients use dedicated legacy stable repositories with plain-codename + # 24.x clients use dedicated legacy stable repositories with plain-codename # suites (apt|ubuntu-standard-<major>-stable, apt-<major>-business-stable): # the promote mirrors its package associations into them - if [[ -z "$DEB_INFIX" && "$VERSION" == "24.10" ]]; then + if [[ -z "$DEB_INFIX" && ( "$VERSION" == "24.04" || "$VERSION" == "24.10" ) ]]; then if [[ "$REPO_BASE" == "standard" ]]; then STABLE_LEGACY_REPOSITORY_NAME="${DEB_PREFIX}standard-$VERSION-stable" STABLE_LEGACY_SUITE="$DISTRIB" diff --git a/pulp/package-delivery/action.yml b/pulp/package-delivery/action.yml index 9807203..0eb5017 100644 --- a/pulp/package-delivery/action.yml +++ b/pulp/package-delivery/action.yml @@ -46,7 +46,7 @@ inputs: description: "OIDC audience, must match the pulp server GITHUB_OIDC.audience" required: true centreon_technique_pat: - description: "Token used to request the feature repository creation (required when delivery_type is feature)" + description: "Token used to request the creation of a missing rpm feature repository" required: false default: "" diff --git a/pulp/package-delivery/ensure-feature-repository.sh b/pulp/package-delivery/ensure-feature-repository.sh index 8d57f0e..1d82af0 100755 --- a/pulp/package-delivery/ensure-feature-repository.sh +++ b/pulp/package-delivery/ensure-feature-repository.sh @@ -14,11 +14,6 @@ PULP_DOMAIN="${PULP_DOMAIN:-default}" MAX_ATTEMPTS=40 POLL_INTERVAL=15 -if [[ -z "${CENTREON_TECHNIQUE_PAT:-}" ]]; then - echo "::error::centreon_technique_pat input is required to request the feature repository creation" - exit 1 -fi - if [[ -z "${VERSION:-}" ]]; then echo "::error::version input is required for a feature delivery" exit 1 @@ -46,6 +41,13 @@ if feature_repositories_exist; then exit 0 fi +# only a missing repository needs the token: callers that provision their +# feature repositories themselves deliver without it +if [[ -z "${CENTREON_TECHNIQUE_PAT:-}" ]]; then + echo "::error::Feature repositories $REPOSITORY_PREFIX-{noarch,x86_64} do not exist in domain $PULP_DOMAIN and the centreon_technique_pat input, required to request their creation, is empty" + exit 1 +fi + echo "[INFO] Requesting the creation of the $FEATURE_TICKET feature repositories for $VERSION" curl --fail-with-body -sS -X POST \ -H "Authorization: Bearer $CENTREON_TECHNIQUE_PAT" \ diff --git a/pulp/promote-to-stable/promote-deb.sh b/pulp/promote-to-stable/promote-deb.sh index 6bab9ea..73fa533 100755 --- a/pulp/promote-to-stable/promote-deb.sh +++ b/pulp/promote-to-stable/promote-deb.sh @@ -136,6 +136,30 @@ if ! pulp_resource_exists "repositories/deb/apt" "$STABLE_REPOSITORY_NAME"; then fi STABLE_REPOSITORY_HREF=$(pulp deb repository show --name "$STABLE_REPOSITORY_NAME" | jq -r '.pulp_href') +# emit the release component a package is associated with in the latest version +# of the legacy repository: empty if none, error if ambiguous or unreadable +legacy_release_component() { + local package_href=$1 latest rcs + refresh_pulp_token + latest=$(pulp deb repository show --name "$STABLE_LEGACY_REPOSITORY_NAME" | jq -r '.latest_version_href') + if ! rcs=$( + curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Bearer $PULP_TOKEN" -G \ + --data-urlencode "package=$package_href" \ + --data-urlencode "repository_version=$latest" \ + --data-urlencode "limit=100" \ + "$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/package_release_components/" \ + | jq -r '[.results[].release_component] | unique | .[]' + ); then + echo "::error::Cannot list the $STABLE_LEGACY_REPOSITORY_NAME associations of $package_href" >&2 + return 1 + fi + if [[ $(echo "$rcs" | grep -c .) -gt 1 ]]; then + echo "::error::$package_href is associated with several release components of $STABLE_LEGACY_REPOSITORY_NAME: $rcs" >&2 + return 1 + fi + echo "$rcs" +} + # 24.x standard clients use the dedicated legacy repository form # (apt-standard-24.10-stable/ with a plain-codename suite, provisioned by # create-repos): mirror every candidate package association into that suite so @@ -147,20 +171,52 @@ ensure_legacy_suite_associations() { echo "::error::Dedicated legacy repository $STABLE_LEGACY_REPOSITORY_NAME does not exist. Pulp repositories are provisioned centrally by delivery-tooling create-repos; run create-repos before promoting." return 1 fi - local legacy_repo_href latest legacy_rc + local legacy_repo_href legacy_rc ref ref_sha ref_href ref_file legacy_attempt task_href rc legacy_repo_href=$(pulp deb repository show --name "$STABLE_LEGACY_REPOSITORY_NAME" | jq -r '.pulp_href') - latest=$(pulp deb repository show --name "$STABLE_LEGACY_REPOSITORY_NAME" | jq -r '.latest_version_href') - legacy_rc=$( - curl -fsSL --retry 3 --retry-delay 5 -H "Authorization: Bearer $PULP_TOKEN" \ - "$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/release_components/?$( - printf 'repository_version=%s&distribution=%s&component=main&limit=1' \ - "$(jq -rn --arg v "$latest" '$v | @uri')" "$(jq -rn --arg v "$STABLE_LEGACY_SUITE" '$v | @uri')" - )" | jq -r '.results[0].pulp_href // empty' - ) || legacy_rc="" + # listing release components is admin-only: deduce the legacy suite's one from + # the reference package's association in the legacy repository (a deb build + # targets a single distribution), legacy-uploading it there first if needed + ref=$(echo "$PACKAGES" | jq -c '.[0]') + ref_sha=$(echo "$ref" | jq -r '.sha256') + ref_href=$(lookup_deb_content "packages" "--data-urlencode sha256=$ref_sha") + if [[ -z "$ref_href" ]]; then + echo "::error::Cannot resolve the reference package (sha256 $ref_sha) for the $STABLE_LEGACY_SUITE mirror" + return 1 + fi + legacy_rc=$(legacy_release_component "$ref_href") || return 1 + if [[ -z "$legacy_rc" ]]; then + ref_file="promoted-packages/$(basename "$(echo "$ref" | jq -r '.relative_path')")" + [[ -s "$ref_file" ]] || download_testing_package "$ref_sha" "$(echo "$ref" | jq -r '.architecture')" "$ref_file" + for legacy_attempt in 1 2 3; do + echo "[INFO] Establishing $STABLE_LEGACY_SUITE/main in $STABLE_LEGACY_REPOSITORY_NAME [legacy path, attempt $legacy_attempt]" + task_href=$( + pulp_upload \ + -F "file=@\"$ref_file\"" \ + -F "relative_path=$(echo "$ref" | jq -r '.relative_path')" \ + -F "distribution=$STABLE_LEGACY_SUITE" \ + -F "component=main" \ + -F "repository=$legacy_repo_href" \ + -F "pulp_labels=$PULP_LABELS" \ + "$PULP_URL/$PULP_DOMAIN/api/v3/content/deb/packages/" + ) + wait_task_race "$task_href" && rc=0 || rc=$? + if [[ $rc -eq 0 ]]; then + break + elif [[ $rc -eq 2 && $legacy_attempt -lt 3 ]]; then + echo "[WARN] Legacy upload into $STABLE_LEGACY_REPOSITORY_NAME lost the repository-version race, retrying" + sleep $((legacy_attempt * 15)) + else + echo "::error::Legacy upload into $STABLE_LEGACY_REPOSITORY_NAME failed" + return 1 + fi + done + legacy_rc=$(legacy_release_component "$ref_href") || return 1 + fi if [[ -z "$legacy_rc" ]]; then - echo "::error::Cannot resolve the $STABLE_LEGACY_SUITE/main release component of $STABLE_LEGACY_REPOSITORY_NAME (provisioned by create-repos)" + echo "::error::Cannot resolve the $STABLE_LEGACY_SUITE/main release component of $STABLE_LEGACY_REPOSITORY_NAME" return 1 fi + echo "[INFO] Release component of $STABLE_LEGACY_REPOSITORY_NAME $STABLE_LEGACY_SUITE/main: $legacy_rc" echo "[INFO] Mirroring $PACKAGES_COUNT package association(s) into $STABLE_LEGACY_REPOSITORY_NAME $STABLE_LEGACY_SUITE/main" local units_file body_file sha href out code body prc units_file=$(mktemp) diff --git a/pulp/tests/ensure-feature-repository.bats b/pulp/tests/ensure-feature-repository.bats new file mode 100644 index 0000000..0edf523 --- /dev/null +++ b/pulp/tests/ensure-feature-repository.bats @@ -0,0 +1,61 @@ +#!/usr/bin/env bats +# Feature repository provisioning, against a curl stand-in for the pulp api +# and the delivery-tooling dispatch. + +setup() { + SCRIPT="${BATS_TEST_DIRNAME}/../package-delivery/ensure-feature-repository.sh" + export STUB_DIR="${BATS_TEST_TMPDIR}/stub" + mkdir -p "${STUB_DIR}/bin" + : > "${STUB_DIR}/calls" + # the repositories exist when REPOSITORIES_EXIST=1, or once a dispatch was sent + cat > "${STUB_DIR}/bin/curl" <<'STUB' +#!/usr/bin/env bash +url="${*: -1}" +case " $* " in + *"/dispatches "*) + echo "dispatch" >> "${STUB_DIR}/calls" + exit 0 + ;; +esac +count=0 +if [[ "${REPOSITORIES_EXIST:-0}" == 1 ]] || grep -qx dispatch "${STUB_DIR}/calls"; then + count=1 +fi +printf '{"count": %s}\n200' "$count" +STUB + chmod +x "${STUB_DIR}/bin/curl" +} + +ensure() { + run env PATH="${STUB_DIR}/bin:${PATH}" STUB_DIR="$STUB_DIR" PULP_URL=https://pulp.invalid PULP_DOMAIN=standard \ + PULP_TOKEN=placeholder PULP_TOKEN_ISSUED_AT="$(date +%s)" VERSION=25.10 GH_HEAD_REF=MON-1234-feature \ + REPOSITORY_PREFIX=rpm-feature-MON-1234-25.10-el9-unstable "$@" \ + bash -c 'sleep() { :; }; export -f sleep; "$1"' _ "$SCRIPT" +} + +@test "existing feature repositories need no token" { + ensure REPOSITORIES_EXIST=1 + [ "$status" -eq 0 ] + [[ "$output" == *"already exist"* ]] + [ ! -s "${STUB_DIR}/calls" ] +} + +@test "missing feature repositories are requested with the token, then awaited" { + ensure CENTREON_TECHNIQUE_PAT=placeholder + [ "$status" -eq 0 ] + [ "$(grep -cx dispatch "${STUB_DIR}/calls")" -eq 1 ] + [[ "$output" == *"are ready"* ]] +} + +@test "missing feature repositories without a token fail with the reason" { + ensure + [ "$status" -ne 0 ] + [[ "$output" == *"do not exist"*"centreon_technique_pat"* ]] + [ ! -s "${STUB_DIR}/calls" ] +} + +@test "a branch without ticket cannot have a feature repository" { + ensure GH_HEAD_REF=my-feature REPOSITORIES_EXIST=1 + [ "$status" -ne 0 ] + [[ "$output" == *"Cannot extract the feature ticket"* ]] +} diff --git a/pulp/tests/promote-deb.bats b/pulp/tests/promote-deb.bats index 530dc44..26433a5 100644 --- a/pulp/tests/promote-deb.bats +++ b/pulp/tests/promote-deb.bats @@ -57,7 +57,12 @@ case "$url" in exit 22 ;; */content/deb/package_release_components/) - body='{"results": [{"release_component": "/content/release_components/1/"}]}' + body='{"pulp_href": "/content/prc/1/", "results": [{"pulp_href": "/content/prc/1/", "release_component": "/content/release_components/1/"}]}' + ;; + */modify/) + echo "modify" >> "${STUB_DIR}/calls" + body='{"task": "/tasks/add/"}' + status=202 ;; */content/deb/packages/\?*) body=$(jq -sc '{next: null, results: .}' "${STUB_DIR}/listing") ;; */content/deb/packages/) @@ -119,3 +124,16 @@ STUB [[ "$output" == *"2 candidate package(s) are missing from the published stable-suite suite"* ]] ! grep -q "^unexpected" "${STUB_DIR}/calls" } + +@test "a 24.x promotion mirrors into the legacy repository without the admin-only listing" { + STABLE_LEGACY_SUITE=trixie STABLE_LEGACY_REPOSITORY_NAME=apt-standard-24.10-stable run "$SCRIPT" + [ "$status" -eq 0 ] + [[ "$output" == *"Release component of apt-standard-24.10-stable trixie/main"* ]] + [[ "$output" == *"Mirroring 2 package association(s) into apt-standard-24.10-stable trixie/main"* ]] + # only the stable suite lookup of the recovery still tries the listing + [ "$(grep -cx "release_components" "${STUB_DIR}/calls")" -eq 1 ] + # the mirror adds the associations to the legacy repository and publishes it + [ "$(grep -cx "modify" "${STUB_DIR}/calls")" -eq 1 ] + [ "$(grep -cx "publish" "${STUB_DIR}/calls")" -eq 2 ] + ! grep -q "^unexpected" "${STUB_DIR}/calls" +} diff --git a/pulp/tests/repository-properties.bats b/pulp/tests/repository-properties.bats index 2f39313..15709a7 100644 --- a/pulp/tests/repository-properties.bats +++ b/pulp/tests/repository-properties.bats @@ -99,3 +99,19 @@ output_of() { grep -qE "^${name}=" "$GITHUB_OUTPUT" || { echo "missing output: ${name}"; return 1; } done } + +@test "24.x stable promotions also target the dedicated legacy repository" { + for version in 24.04 24.10; do + : > "$GITHUB_OUTPUT" + properties MODULE_NAME=web DISTRIB=bookworm STABILITY=stable IS_CLOUD=false VERSION="$version" RELEASE_TYPE=release + [ "$status" -eq 0 ] + [ "$(output_of stable_legacy_repository_name)" = "apt-standard-${version}-stable" ] + [ "$(output_of stable_legacy_suite)" = "bookworm" ] + done +} + +@test "later versions have no legacy repository" { + properties MODULE_NAME=web DISTRIB=bookworm STABILITY=stable IS_CLOUD=false VERSION=25.10 RELEASE_TYPE=release + [ "$status" -eq 0 ] + [ -z "$(output_of stable_legacy_repository_name)" ] +} From f61a070932b8f2260f8ef237ec847228d496e4a2 Mon Sep 17 00:00:00 2001 From: Kevin Duret <kduret@centreon.com> Date: Thu, 19 Nov 2026 17:09:09 +0100 Subject: [PATCH 15/15] fix(pulp): promote families whose testing and stable suites share their name Port of the centreon-plugins case: plugins and connectors name both suites after the distribution, so the release component is shared and already associated before the reference upload, and the before/after diff that deduces it is empty. Limited to equal suite names: elsewhere a single association may be the testing one. Refs: MON-215510 --- pulp/promote-to-stable/promote-deb.sh | 7 +++++++ pulp/tests/promote-deb.bats | 14 ++++++++++++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/pulp/promote-to-stable/promote-deb.sh b/pulp/promote-to-stable/promote-deb.sh index 73fa533..2bc87d6 100755 --- a/pulp/promote-to-stable/promote-deb.sh +++ b/pulp/promote-to-stable/promote-deb.sh @@ -411,6 +411,13 @@ if ((${#BATCH_PACKAGES[@]} > 0)); then fi LEGACY_REF_AFTER=$(lookup_prcs "$LEGACY_REF_STABLE_HREF" | sort) STABLE_RC_SET=$(comm -13 <(echo "$LEGACY_REF_BEFORE") <(echo "$LEGACY_REF_AFTER") | grep . || true) + # unversioned families name their testing and stable suites alike: in a + # shared domain that is one release component, already associated before + # the legacy upload, so the diff is empty. Not applied when the names + # differ, where a single association may belong to the testing suite. + if [[ -z "$STABLE_RC_SET" && "$TESTING_SUITE" == "$STABLE_SUITE" && $(echo "$LEGACY_REF_AFTER" | grep -c .) -eq 1 ]]; then + STABLE_RC_SET=$LEGACY_REF_AFTER + fi if [[ $(echo "$STABLE_RC_SET" | grep -c .) -ne 1 ]]; then refresh_pulp_token STABLE_LATEST=$(pulp deb repository show --name "$STABLE_REPOSITORY_NAME" | jq -r '.latest_version_href') diff --git a/pulp/tests/promote-deb.bats b/pulp/tests/promote-deb.bats index 26433a5..50b535d 100644 --- a/pulp/tests/promote-deb.bats +++ b/pulp/tests/promote-deb.bats @@ -47,8 +47,8 @@ done status=200 case "$url" in */dists/*/Release) body="Architectures: all" ;; - */dists/testing-suite/*/Packages) body=$(cat "${STUB_DIR}/testing_Packages") ;; - */dists/stable-suite/*/Packages) body=$(cat "${STUB_DIR}/stable_Packages") ;; + */apt-testing/dists/*/Packages) body=$(cat "${STUB_DIR}/testing_Packages") ;; + */apt-stable/dists/*/Packages) body=$(cat "${STUB_DIR}/stable_Packages") ;; */pool/*) body="deb" ;; */api/v3/repositories/deb/apt/) body='{"count": 1}' ;; */content/deb/release_components/\?*) @@ -137,3 +137,13 @@ STUB [ "$(grep -cx "publish" "${STUB_DIR}/calls")" -eq 2 ] ! grep -q "^unexpected" "${STUB_DIR}/calls" } + +@test "a family whose testing and stable suites share their name needs no listing" { + TESTING_SUITE=trixie STABLE_SUITE=trixie run "$SCRIPT" + [ "$status" -eq 0 ] + [[ "$output" != *"Cannot deduce"* ]] + ! grep -qx "release_components" "${STUB_DIR}/calls" + [ "$(grep -cx "modify" "${STUB_DIR}/calls")" -eq 1 ] + [ "$(grep -cx "publish" "${STUB_DIR}/calls")" -eq 1 ] + ! grep -q "^unexpected" "${STUB_DIR}/calls" +}