ci: match dispatched release checks exactly and allow reruns

Release Please passes a per-attempt request id to each trusted check and waits only on the run carrying it, never with a baseline override. The checks run in their own job so re-running it re-dispatches all three and repaints Verify on the current head, and the delivery token is minted just before enabling auto-merge.

Author: kvlonge

.github/workflows/pull-request-metadata.yml

@@ -1,4 +1,7 @@
 name: Pull Request Metadata
+run-name: >-
+  ${{ inputs.request_id && format('{0} [{1}]', github.workflow, inputs.request_id)
+  || github.event.pull_request.title || github.event.head_commit.message || github.workflow }}
 
 on:
   pull_request:
@@ -9,6 +12,10 @@ on:
         description: Pull request number to validate
         required: true
         type: number
+      request_id:
+        description: "Release Please attempt that dispatched this run; leave empty for manual runs"
+        required: false
+        type: string
 
 permissions:
   contents: read

.github/workflows/release-performance.yml

@@ -1,4 +1,7 @@
 name: Release Performance
+run-name: >-
+  ${{ inputs.request_id && format('{0} [{1}]', github.workflow, inputs.request_id)
+  || github.event.pull_request.title || github.event.head_commit.message || github.workflow }}
 
 # Release Please dispatches this workflow for each release pull request and reports its result
 # through the required Verify status. Pull requests that change the comparison itself run it too.
@@ -9,6 +12,10 @@ on:
         description: "Published sqlbuild version to compare against (default: the release before the candidate)"
         required: false
         type: string
+      request_id:
+        description: "Release Please attempt that dispatched this run; leave empty for manual runs"
+        required: false
+        type: string
   pull_request:
     paths:
       - .github/workflows/release-performance.yml
@@ -51,7 +58,12 @@ jobs:
       - name: Compare candidate and previous release on this runner
         env:
           BASELINE: ${{ inputs.baseline }}
+          REQUEST_ID: ${{ inputs.request_id }}
         run: |
+          if [[ -n "${REQUEST_ID}" && -n "${BASELINE}" ]]; then
+            echo "Release Please runs always compare with the previous release; drop baseline" >&2
+            exit 1
+          fi
           wheels=("${RUNNER_TEMP}"/candidate/sqlbuild-*.whl)
           args=(
             --candidate "${wheels[0]}"

.github/workflows/release-please.yml

@@ -19,6 +19,10 @@ concurrency:
 jobs:
   release-please:
     runs-on: ubuntu-24.04
+    outputs:
+      prs_created: ${{ steps.release.outputs.prs_created }}
+      release_branch: ${{ steps.release_branch.outputs.name }}
+      release_pr: ${{ steps.release_branch.outputs.pr }}
     steps:
       - name: Create delivery token
         id: delivery-token
@@ -56,11 +60,13 @@ jobs:
           RELEASE_PRS: ${{ steps.release.outputs.prs }}
         run: |
           release_branch="$(jq -r '.[0].headBranchName // empty' <<< "${RELEASE_PRS}")"
-          if [[ -z "${release_branch}" ]]; then
-            echo "Release Please reported a created PR without a head branch" >&2
+          release_pr="$(jq -r '.[0].number // empty' <<< "${RELEASE_PRS}")"
+          if [[ -z "${release_branch}" || -z "${release_pr}" ]]; then
+            echo "Release Please reported a created PR without a head branch or number" >&2
             exit 1
           fi
           echo "name=${release_branch}" >> "${GITHUB_OUTPUT}"
+          echo "pr=${release_pr}" >> "${GITHUB_OUTPUT}"
 
       - uses: actions/checkout@v4
         if: steps.release.outputs.prs_created == 'true'
@@ -99,93 +105,114 @@ jobs:
           git commit -m "chore: refresh release lockfiles"
           git push origin "HEAD:${RELEASE_BRANCH}"
 
+  # A separate job so that "Re-run failed jobs" re-dispatches and re-evaluates every trusted check
+  # and repaints their statuses and Verify on the current head of the release branch.
+  release-checks:
+    name: Release pull request checks
+    needs: release-please
+    if: needs.release-please.outputs.prs_created == 'true'
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ needs.release-please.outputs.release_branch }}
+
       - name: Run release pull request checks
-        if: steps.release.outputs.prs_created == 'true'
         env:
           GH_TOKEN: ${{ github.token }}
-          RELEASE_BRANCH: ${{ steps.release_branch.outputs.name }}
-          RELEASE_PRS: ${{ steps.release.outputs.prs }}
+          RELEASE_BRANCH: ${{ needs.release-please.outputs.release_branch }}
+          RELEASE_PR: ${{ needs.release-please.outputs.release_pr }}
+          REQUEST_ID: ${{ github.run_id }}-${{ github.run_attempt }}
         run: |
-          release_pr="$(jq -r '.[0].number // empty' <<< "${RELEASE_PRS}")"
-          if [[ -z "${release_pr}" ]]; then
-            echo "Release Please reported a created PR without a number" >&2
-            exit 1
-          fi
           head_sha="$(git rev-parse HEAD)"
+          recovery="rerun the failed Release Please job to re-check"
+          post_status() {
+            gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \
+              -f "state=$1" -f "context=$2" -f "description=$3" ${4:+-f "target_url=$4"} >/dev/null
+          }
+          fail_verify() {
+            post_status failure Verify "$1; ${recovery}" "${2:-}"
+            {
+              echo "### Release pull request checks failed"
+              echo
+              echo "$1. After fixing or confirming a transient cause, use **Re-run failed jobs**"
+              echo "on this workflow run: it re-dispatches every trusted check and repaints Verify."
+            } >> "${GITHUB_STEP_SUMMARY}"
+            exit 1
+          }
 
           # Verify is the required status, so it only succeeds after every trusted check passes,
-          # including the same-runner comparison with the previous release.
+          # including the same-runner comparison with the previous release. Each dispatched run
+          # carries this attempt's request id in its run name and is matched on it exactly.
           workflows=(pull-request-metadata.yml version-guard.yml release-performance.yml)
           contexts=("Pull Request Metadata" "Block automatic 1.0 releases" "Release performance")
           for context in "${contexts[@]}"; do
-            gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \
-              -f state=pending -f "context=${context}" \
-              -f description="Trusted release check is running" >/dev/null
+            post_status pending "${context}" "Trusted release check is running"
           done
-          gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \
-            -f state=pending -f context=Verify \
-            -f description="Validating trusted release metadata and performance" >/dev/null
+          post_status pending Verify "Validating trusted release metadata and performance"
 
+          dispatched_after="$(date -u -d '-2 minutes' +%Y-%m-%dT%H:%M:%SZ)"
           gh workflow run pull-request-metadata.yml --ref "${RELEASE_BRANCH}" \
-            -f "pr_number=${release_pr}"
-          gh workflow run version-guard.yml --ref "${RELEASE_BRANCH}"
-          gh workflow run release-performance.yml --ref "${RELEASE_BRANCH}"
+            -f "pr_number=${RELEASE_PR}" -f "request_id=${REQUEST_ID}"
+          gh workflow run version-guard.yml --ref "${RELEASE_BRANCH}" \
+            -f "request_id=${REQUEST_ID}"
+          gh workflow run release-performance.yml --ref "${RELEASE_BRANCH}" \
+            -f "request_id=${REQUEST_ID}"
           for index in "${!workflows[@]}"; do
             workflow="${workflows[${index}]}"
             context="${contexts[${index}]}"
             run_id=""
             for _ in {1..30}; do
               run_id="$(gh run list --workflow "${workflow}" --branch "${RELEASE_BRANCH}" \
-                --event workflow_dispatch --limit 20 --json databaseId,headSha \
-                --jq ".[] | select(.headSha == \"${head_sha}\") | .databaseId" | head -n 1)"
+                --event workflow_dispatch --limit 50 \
+                --json databaseId,headSha,displayTitle,createdAt \
+                --jq ".[] | select(.headSha == \"${head_sha}\"
+                  and (.displayTitle | endswith(\" [${REQUEST_ID}]\"))
+                  and .createdAt >= \"${dispatched_after}\") | .databaseId" | head -n 1)"
               [[ -n "${run_id}" ]] && break
               sleep 2
             done
             if [[ -z "${run_id}" ]]; then
-              gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \
-                -f state=failure -f "context=${context}" \
-                -f description="Trusted release check did not start" >/dev/null
-              gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \
-                -f state=failure -f context=Verify \
-                -f description="${context} did not start" >/dev/null
-              exit 1
+              post_status failure "${context}" "Trusted release check did not start"
+              fail_verify "${context} did not start"
             fi
             run_url="$(gh run view "${run_id}" --json url --jq .url)"
             if gh run watch "${run_id}" --exit-status --interval 30; then
-              gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \
-                -f state=success -f "context=${context}" -f target_url="${run_url}" \
-                -f description="Trusted release check passed" >/dev/null
+              post_status success "${context}" "Trusted release check passed" "${run_url}"
             else
-              gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \
-                -f state=failure -f "context=${context}" -f target_url="${run_url}" \
-                -f description="Trusted release check failed" >/dev/null
-              gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \
-                -f state=failure -f context=Verify -f target_url="${run_url}" \
-                -f description="${context} failed" >/dev/null
-              exit 1
+              post_status failure "${context}" "Trusted release check failed" "${run_url}"
+              fail_verify "${context} failed" "${run_url}"
             fi
           done
 
-          unexpected_files="$(gh pr view "${release_pr}" --repo "${GITHUB_REPOSITORY}" \
+          unexpected_files="$(gh pr view "${RELEASE_PR}" --repo "${GITHUB_REPOSITORY}" \
             --json files --jq '[.files[].path | select(. != ".release-please-manifest.json" and . != "CHANGELOG.md" and . != "pyproject.toml" and . != "uv.lock" and . != "Cargo.lock" and . != "crates/sqlbuild-rules-native/Cargo.toml")] | join("\n")')"
           if [[ -n "${unexpected_files}" ]]; then
             echo "Release pull request contains non-metadata changes:" >&2
             echo "${unexpected_files}" >&2
-            gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \
-              -f state=failure -f context=Verify \
-              -f description="Release pull request contains non-metadata changes" >/dev/null
+            post_status failure Verify "Release pull request contains non-metadata changes"
             exit 1
           fi
-          gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \
-            -f state=success -f context=Verify \
-            -f description="Product changes passed verification before release metadata update" >/dev/null
+          post_status success Verify \
+            "Product changes passed verification before release metadata update"
+
+      # The checks can wait longer than an installation token lives, so mint one just before use.
+      - name: Create delivery token
+        id: delivery-token
+        uses: actions/create-github-app-token@v2
+        with:
+          app-id: ${{ vars.CHIO_DELIVERY_APP_ID }}
+          private-key: ${{ secrets.CHIO_DELIVERY_PRIVATE_KEY }}
+          owner: ${{ github.repository_owner }}
+          repositories: ${{ github.event.repository.name }}
+          permission-actions: write
+          permission-contents: write
+          permission-issues: write
+          permission-pull-requests: write
 
       - name: Enable release pull request auto-merge
-        if: steps.release.outputs.prs_created == 'true'
         env:
           # The delivery app merges, so the resulting push to main starts the tag and publish runs.
           GH_TOKEN: ${{ steps.delivery-token.outputs.token }}
-          RELEASE_PRS: ${{ steps.release.outputs.prs }}
-        run: |
-          release_pr="$(jq -r '.[0].number // empty' <<< "${RELEASE_PRS}")"
-          gh pr merge "${release_pr}" --repo "${GITHUB_REPOSITORY}" --auto --squash
+          RELEASE_PR: ${{ needs.release-please.outputs.release_pr }}
+        run: gh pr merge "${RELEASE_PR}" --repo "${GITHUB_REPOSITORY}" --auto --squash

.github/workflows/version-guard.yml

@@ -1,4 +1,7 @@
 name: Version Guard
+run-name: >-
+  ${{ inputs.request_id && format('{0} [{1}]', github.workflow, inputs.request_id)
+  || github.event.pull_request.title || github.event.head_commit.message || github.workflow }}
 
 on:
   pull_request:
@@ -8,6 +11,11 @@ on:
     branches:
       - main
   workflow_dispatch:
+    inputs:
+      request_id:
+        description: "Release Please attempt that dispatched this run; leave empty for manual runs"
+        required: false
+        type: string
 
 permissions:
   contents: read

tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md

@@ -103,16 +103,20 @@ the medians of wall time, CPU time (user+sys) and peak RSS:
 
 A command fails when its candidate median exceeds the baseline by more than 25% and by more than
 0.5 s (wall and CPU) or 32 MiB (peak RSS); the limits live in
-`scripts/release_performance/constants.py`. The baseline defaults to the highest published or
-release-tagged version below the candidate. A freshly tagged baseline is awaited until its wheels
+`scripts/release_performance/constants.py`. The baseline defaults to the highest version below
+the candidate that is installable from PyPI, or release-tagged but not yet on PyPI; yanked releases
+never count. A freshly tagged baseline is awaited until its wheels
 are on PyPI, installing from the uncompressed simple index's wheel URL when the CDN still lags. A
 command that needs a feature newer than a compared version declares `minimum_version` and appears
 as skipped, with the reason, in the job summary; any other failure of either version fails the
 check.
 
 Release Please dispatches `.github/workflows/release-performance.yml` with the metadata and version
 checks and posts the required `Verify` status only after it passes, so a regressed release pull
-request cannot auto-merge. Run it manually with an optional `baseline` input, or locally:
+request cannot auto-merge. Each dispatch carries a request id in its run name, and Release Please
+waits only on the runs it started. After a transient failure, use **Re-run failed jobs** on the
+Release Please run: it re-dispatches all three checks and repaints `Verify` on the current head.
+Run the comparison manually with an optional `baseline` input, or locally:
 
 ```bash
 uv run python -m scripts.compare_release_performance --candidate 0.121.0 --baseline 0.119.1