feat(verify): replay only changed cases and run one required Verify check

.github/workflows/ci.yml

@@ -1,33 +1,83 @@
 name: CI
 
+# The required check is the Verify job, which passes only when every job the change needs passed.
+# A pull request runs what its changed files need (scripts/replay_scope decides):
+# - documentation and website changes build and check the site only;
+# - code changes run check-ci, the Rust and Python tests, and the replay of the recorded cases,
+#   which reuses main's judgements for every case whose conversion and recording are unchanged;
+# - a change under a full-replay path replays every case.
+# The nightly run on main replays every case, refreshes the replay cache pull requests reuse,
+# converts DuckShop within its time budget and fuzzes. Release Please dispatches a full run on
+# each release pull request's branch, so every release replays every case first.
 on:
   pull_request:
-  push:
-    branches: [main]
+  schedule:
+    - cron: "17 3 * * *"
   workflow_dispatch:
 
 permissions:
   contents: read
 
 concurrency:
-  group: ci-${{ github.ref }}
+  group: ci-${{ github.ref }}-${{ github.event_name }}
   cancel-in-progress: ${{ github.event_name == 'pull_request' }}
 
+env:
+  CARGO_TERM_COLOR: always
+  CARGO_INCREMENTAL: "0"
+
 jobs:
-  verify:
-    name: Verify
+  changes:
+    name: Classify the change
+    runs-on: ubuntu-24.04
+    outputs:
+      replay: ${{ steps.scope.outputs.replay }}
+      code: ${{ steps.scope.outputs.code }}
+      site: ${{ steps.scope.outputs.site }}
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      # The repository's scripts need Python 3.12.
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+      # Every pull request, documentation only or not, keeps agent files and private paths out.
+      - name: Run the clutter guard
+        run: python3 -m scripts.check_clutter_guard
+      - name: Select the checks the change needs
+        id: scope
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+        run: |
+          if [[ "${EVENT_NAME}" == "pull_request" ]]; then
+            python3 -m scripts.select_replay scope --base "${BASE_SHA}" >> "${GITHUB_OUTPUT}"
+          else
+            printf 'replay=full\ncode=true\nsite=true\n' >> "${GITHUB_OUTPUT}"
+          fi
+
+  site:
+    name: Website
+    needs: changes
+    if: needs.changes.outputs.site == 'true'
+    uses: ./.github/workflows/website-build.yml
+
+  checks:
+    name: Checks, tests and replay
+    needs: changes
+    if: needs.changes.outputs.code == 'true'
     runs-on: ubuntu-24.04
-    timeout-minutes: 20
+    # A cold build of the workspace takes most of this; the cached dependency builds cut it.
+    timeout-minutes: 120
     env:
-      CARGO_TERM_COLOR: always
-      CARGO_INCREMENTAL: "0"
       # The Snowflake recorder is never built or run in CI.
       RECORDER: "0"
       # `uv run` would otherwise sync the default groups, recorder included.
       UV_NO_SYNC: "1"
-      # The evidence pin check and the result baselines compare with the pull request base or the
-      # previous main commit.
-      EVIDENCE_BASE: ${{ github.event.pull_request.base.sha || github.event.before || 'origin/main' }}
+      # The evidence pin check and the result baselines compare with the pull request's base.
+      EVIDENCE_BASE: ${{ github.event.pull_request.base.sha || 'origin/main' }}
+      REPLAY: ${{ needs.changes.outputs.replay }}
     steps:
       - uses: actions/checkout@v4
         with:
@@ -39,77 +89,115 @@ jobs:
           cache-dependency-glob: uv.lock
       - name: Install the pinned Rust toolchain
         run: rustup toolchain install
-      # Caches the dependency builds, polyglot above all, keyed by Cargo.lock and the toolchain.
+      # The gate build and its tests outgrow the runner's free disk with the preinstalled SDKs.
+      - name: Free disk space
+        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
+      # The dependency builds, keyed by Cargo.lock and the toolchain; pull requests only read them.
       - uses: Swatinem/rust-cache@v2
         with:
-          shared-key: verify
+          shared-key: gate
+          save-if: ${{ github.event_name != 'pull_request' }}
+      - name: Download the DuckDB library again if the restored cache lacks it
+        run: scripts/refetch_duckdb_library.sh
+      # The evidence snapshot evidence.toml pins; scripts/evidence.sh verifies it after a restore.
+      - uses: actions/cache@v4
+        with:
+          path: .evidence/pinned
+          key: evidence-${{ hashFiles('evidence.toml') }}
       - name: Install Python
         run: uv python install 3.12
       - name: Install development tools without the recorder
         run: uv sync --locked --no-group recorder
       # The instruction-count guard in check-ci runs its workloads under callgrind.
       - name: Install valgrind
         run: sudo apt-get update -q && sudo apt-get install -y -q --no-install-recommends valgrind
-      - name: Run static checks
-        run: make check-ci
-      - name: Run tests
-        run: make test
-      # `make verify-recorded`, with the same verify flags as `make gate`, then the measure and SQL
-      # size checks.
+      # One gate build serves every step: check-ci, the Rust tests, the Python tests and the replay.
+      - name: Run static checks and tests
+        run: make check-merge test-python
+      - name: Find the replay cache's scope
+        id: replay
+        if: env.REPLAY != 'none'
+        run: echo "scope=$(uv run python -m scripts.select_replay digest)" >> "${GITHUB_OUTPUT}"
+      # Judgements made under the same full-replay paths, most recently by main's nightly run.
+      - name: Restore the replay cache
+        if: env.REPLAY == 'changed'
+        uses: actions/cache/restore@v4
+        with:
+          path: target/replay-cache.json
+          key: replay-${{ steps.replay.outputs.scope }}-${{ github.run_id }}
+          restore-keys: replay-${{ steps.replay.outputs.scope }}-
       - name: Replay recorded translations on DuckDB and check their baselines
-        run: >-
-          make check-convert DIALECT=snowflake VERIFY_PROFILE=dev RESULTS_BASE="$EVIDENCE_BASE"
+        if: env.REPLAY != 'none'
+        run: make verify-recorded RESULTS_BASE="${EVIDENCE_BASE}" REPLAY_CACHE=target/replay-cache.json
+      - name: Report the replay
+        if: env.REPLAY == 'none'
+        run: echo "No changed file is in a crate the sqlquack binary is built from, so nothing replays."
+      # Pull requests restore the cache saved on their base branch, main's from the nightly run.
+      - name: Save the replay cache
+        if: env.REPLAY != 'none' && github.event_name != 'pull_request'
+        uses: actions/cache/save@v4
+        with:
+          path: target/replay-cache.json
+          key: replay-${{ steps.replay.outputs.scope }}-${{ github.run_id }}
+
+  verify:
+    name: Verify
+    if: always()
+    needs: [changes, site, checks]
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Confirm every job the change needs passed
+        env:
+          CHANGES: ${{ needs.changes.result }}
+          SITE: ${{ needs.site.result }}
+          SITE_NEEDED: ${{ needs.changes.outputs.site }}
+          CHECKS: ${{ needs.checks.result }}
+          CHECKS_NEEDED: ${{ needs.changes.outputs.code }}
+        run: |
+          failed=0
+          expect() {
+            local name=$1 result=$2 needed=$3
+            if [[ "${result}" == "success" || ( "${needed}" != "true" && "${result}" == "skipped" ) ]]; then
+              echo "${name}: ${result}"
+            else
+              echo "${name}: ${result}, but the change needs it" >&2
+              failed=1
+            fi
+          }
+          expect "Classify the change" "${CHANGES}" true
+          expect "Website" "${SITE}" "${SITE_NEEDED}"
+          expect "Checks, tests and replay" "${CHECKS}" "${CHECKS_NEEDED}"
+          exit "${failed}"
 
-  polyglot-check:
-    name: Exact parser check on DuckShop
+  duckshop:
+    name: Convert DuckShop within the time budget
+    if: github.event_name != 'pull_request'
     runs-on: ubuntu-24.04
     timeout-minutes: 30
-    env:
-      CARGO_TERM_COLOR: always
-      CARGO_INCREMENTAL: "0"
-      # The DuckShop benchmarks and the SQLBuild release that compiles them, pinned together.
-      SQLBUILD_REF: a7bc60e0abee5bb19d67e5bdc948e048a1df82e0
-      SQLBUILD_VERSION: 0.187.2
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/checkout@v4
-        with:
-          repository: sqlstack-io/sqlbuild
-          ref: ${{ env.SQLBUILD_REF }}
-          path: sqlbuild
-          sparse-checkout: benchmarks
       - uses: astral-sh/setup-uv@v5
         with:
           version: "0.12.3"
       - name: Install the pinned Rust toolchain
         run: rustup toolchain install
       - uses: Swatinem/rust-cache@v2
         with:
-          shared-key: verify
-          save-if: false
+          shared-key: duckshop
+          save-if: ${{ github.event_name != 'pull_request' }}
+      - name: Download the DuckDB library again if the restored cache lacks it
+        run: scripts/refetch_duckdb_library.sh
       - name: Compile DuckShop and DuckShop 10K
-        run: |
-          uv venv -p 3.12 "$RUNNER_TEMP/sqlbuild"
-          VIRTUAL_ENV="$RUNNER_TEMP/sqlbuild" uv pip install "sqlbuild==$SQLBUILD_VERSION"
-          python3 sqlbuild/benchmarks/duckshop-10k/generate.py --out "$RUNNER_TEMP/duckshop-10k"
-          (cd sqlbuild/benchmarks/duckshop/project && "$RUNNER_TEMP/sqlbuild/bin/sqb" compile)
-          (cd "$RUNNER_TEMP/duckshop-10k" && "$RUNNER_TEMP/sqlbuild/bin/sqb" compile)
-      - name: Compare with polyglot
-        run: |
-          cargo run --locked --release -p sqlquack-polyglot-check -- \
-            directory sqlbuild/benchmarks/duckshop/project/target/compiled --dialect snowflake
-          cargo run --locked --release -p sqlquack-polyglot-check -- \
-            directory "$RUNNER_TEMP/duckshop-10k/target/compiled" --dialect snowflake
-      - name: Convert DuckShop within the time budget
-        run: |
-          make census-duckshop DUCKSHOP_COMPILED="sqlbuild/benchmarks/duckshop/project/target/compiled $RUNNER_TEMP/duckshop-10k/target/compiled"
+        run: make duckshop-compiled
+      - name: Convert them
+        run: make census-duckshop
 
   fuzz:
     name: Fuzz tolerant parsing
+    if: github.event_name != 'pull_request'
     runs-on: ubuntu-24.04
     timeout-minutes: 15
     env:
-      CARGO_TERM_COLOR: always
       # Seconds of fuzzing after the build.
       FUZZ_SECONDS: "90"
     steps:

.github/workflows/distributions.yml

@@ -1,9 +1,9 @@
 name: Distributions
 
-# Builds and smoke tests every wheel and sdist when a pull request changes what goes into them, and
-# on every push to main the linux-x86_64 wheels only, so a distribution-only failure shows at the
-# landing rather than at the next release. Both also run the command line's convert tests with
-# DuckDB compiled in. Neither blocks a landing.
+# Builds and smoke tests the linux-x86_64 wheels when a pull request changes what goes into them,
+# and on every push to main, so a distribution-only failure shows at the landing rather than at the
+# next release. Both also run the command line's convert tests with DuckDB compiled in. Every
+# platform and the sdists build nightly and for each release. Neither blocks a landing.
 on:
   push:
     branches: [main]
@@ -31,8 +31,8 @@ jobs:
     name: Distributions
     uses: ./.github/workflows/build-distributions.yml
     with:
-      platforms: ${{ github.event_name == 'push' && 'linux-x86_64' || '' }}
-      sdist: ${{ github.event_name != 'push' }}
+      platforms: ${{ github.event_name == 'workflow_dispatch' && '' || 'linux-x86_64' }}
+      sdist: ${{ github.event_name == 'workflow_dispatch' }}
 
   # The command line built with `bundled` alone, as the sqlquack wheel compiles DuckDB in: no system
   # DuckDB.

.github/workflows/pull-request-metadata.yml

@@ -0,0 +1,57 @@
+name: Pull Request Metadata
+run-name: >-
+  ${{ inputs.request_id && format('{0} [{1}]', github.workflow, inputs.request_id)
+  || github.event.pull_request.title || github.event.head_commit.message
+  || github.event.merge_group.head_commit.message || github.workflow }}
+
+on:
+  pull_request:
+    types: [opened, edited, synchronize, reopened]
+  merge_group:
+  workflow_dispatch:
+    inputs:
+      pr_number:
+        description: Pull request number to validate
+        required: true
+        type: number
+      request_id:
+        description: "Release Please attempt that dispatched this run; leave empty for manual runs"
+        required: false
+        type: string
+
+permissions:
+  contents: read
+  pull-requests: read
+
+jobs:
+  validate:
+    name: Pull Request Metadata
+    runs-on: ubuntu-24.04
+    steps:
+      # The queue only admits pull requests whose metadata check already passed, so a merge group
+      # reports success instead of skipping: a required check must report for the queue to merge.
+      - name: Accept metadata validated on the pull request
+        if: github.event_name == 'merge_group'
+        run: echo "Pull request metadata was validated on the pull request before it was queued."
+
+      - uses: actions/checkout@v4
+        if: github.event_name != 'merge_group'
+      - name: Resolve pull request metadata
+        if: github.event_name != 'merge_group'
+        env:
+          EVENT_NAME: ${{ github.event_name }}
+          GH_TOKEN: ${{ github.token }}
+          PR_NUMBER: ${{ inputs.pr_number }}
+        run: |
+          if [[ "${EVENT_NAME}" == "pull_request" ]]; then
+            cp "${GITHUB_EVENT_PATH}" "${RUNNER_TEMP}/pull-request.json"
+          else
+            gh api "repos/${GITHUB_REPOSITORY}/pulls/${PR_NUMBER}" \
+              > "${RUNNER_TEMP}/pull-request.json"
+          fi
+
+      - name: Validate branch, title, and description
+        if: github.event_name != 'merge_group'
+        env:
+          PR_METADATA_PATH: ${{ runner.temp }}/pull-request.json
+        run: python -m scripts.validate_pr_metadata --metadata-file "${PR_METADATA_PATH}"

.github/workflows/release-please.yml

@@ -1,8 +1,8 @@
 name: Release Please
 
 # On every push to main, opens or updates the release pull request from the Conventional Commits
-# since the last release and dispatches Release Distributions to build its distributions. Once the
-# release pull request merges, it tags the release and dispatches Publish for that tag.
+# since the last release, and dispatches its required checks and Release Distributions on its
+# branch. Once the release pull request merges, it tags the release and dispatches Publish for it.
 on:
   push:
     branches:
@@ -37,16 +37,23 @@ jobs:
           gh workflow run publish.yml --repo "${GITHUB_REPOSITORY}" --ref main \
             -f "tag=${RELEASE_TAG}"
 
-      - name: Build the release pull request's distributions
+      # A pull request this workflow opens starts no pull_request runs, so the release pull
+      # request's required checks are dispatched on its branch: CI replays every recorded case
+      # before the release, alongside the distribution build.
+      - name: Check the release pull request and build its distributions
         if: steps.release.outputs.prs_created == 'true'
         env:
           GH_TOKEN: ${{ github.token }}
           RELEASE_PRS: ${{ steps.release.outputs.prs }}
         run: |
           release_branch="$(jq -r '.[0].headBranchName // empty' <<< "${RELEASE_PRS}")"
-          if [[ -z "${release_branch}" ]]; then
-            echo "Release Please reported a created pull request without a head branch" >&2
+          release_pr="$(jq -r '.[0].number // empty' <<< "${RELEASE_PRS}")"
+          if [[ -z "${release_branch}" || -z "${release_pr}" ]]; then
+            echo "Release Please reported a created pull request without a head branch or number" >&2
             exit 1
           fi
-          gh workflow run release-distributions.yml --repo "${GITHUB_REPOSITORY}" \
-            --ref "${release_branch}"
+          run() { gh workflow run "$1" --repo "${GITHUB_REPOSITORY}" --ref "${release_branch}" "${@:2}"; }
+          run release-distributions.yml
+          run ci.yml
+          run version-guard.yml
+          run pull-request-metadata.yml -f "pr_number=${release_pr}"