ci.yml

name: CI

# The required check is the Verify job, which passes only when every job the change needs passed.
# A pull request runs what its changed files need (scripts/replay_scope decides):
# - documentation and website changes build and check the site only;
# - code changes run check-ci, the Rust and Python tests, and the replay of the recorded cases,
#   which reuses main's judgements for every case whose conversion and recording are unchanged;
# - a change under a full-replay path replays every case.
# The nightly run on main replays every case, refreshes the replay cache pull requests reuse,
# converts DuckShop within its time budget and fuzzes. Release Please dispatches a full run on
# each release pull request's branch, so every release replays every case first.
# A merge queue group runs the same checks as a pull request, scoped against the group's base.
on:
  pull_request:
  merge_group:
  schedule:
    - cron: "17 3 * * *"
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: ci-${{ github.ref }}-${{ github.event_name }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

env:
  CARGO_TERM_COLOR: always
  CARGO_INCREMENTAL: "0"

jobs:
  changes:
    name: Classify the change
    runs-on: ubuntu-24.04
    outputs:
      replay: ${{ steps.scope.outputs.replay }}
      code: ${{ steps.scope.outputs.code }}
      site: ${{ steps.scope.outputs.site }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      # The repository's scripts need Python 3.12.
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      # Every pull request, documentation only or not, keeps agent files and private paths out.
      - name: Run the clutter guard
        run: python3 -m scripts.check_clutter_guard
      - name: Select the checks the change needs
        id: scope
        env:
          EVENT_NAME: ${{ github.event_name }}
          BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }}
        run: |
          if [[ "${EVENT_NAME}" == "pull_request" || "${EVENT_NAME}" == "merge_group" ]]; then
            python3 -m scripts.select_replay scope --base "${BASE_SHA}" >> "${GITHUB_OUTPUT}"
          else
            printf 'replay=full\ncode=true\nsite=true\n' >> "${GITHUB_OUTPUT}"
          fi

  site:
    name: Website
    needs: changes
    if: needs.changes.outputs.site == 'true'
    uses: ./.github/workflows/website-build.yml

  checks:
    name: Checks, tests and replay
    needs: changes
    if: needs.changes.outputs.code == 'true'
    runs-on: ubuntu-24.04
    # A cold build of the workspace takes most of this; the cached dependency builds cut it.
    timeout-minutes: 120
    env:
      # The Snowflake recorder is never built or run in CI.
      RECORDER: "0"
      # `uv run` would otherwise sync the default groups, recorder included.
      UV_NO_SYNC: "1"
      # The evidence pin check and the result baselines compare with the pull request's base.
      EVIDENCE_BASE: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || 'origin/main' }}
      REPLAY: ${{ needs.changes.outputs.replay }}
      # The Python tests and the replay take the module and binary from check-merge's gate build.
      PYTHON_MODULE_BUILD: gate-build
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: astral-sh/setup-uv@v5
        with:
          version: "0.12.3"
          enable-cache: true
          cache-dependency-glob: uv.lock
      - name: Install the pinned Rust toolchain
        run: rustup toolchain install
      # The gate build and its tests outgrow the runner's free disk with the preinstalled SDKs.
      - name: Free disk space
        run: sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc /opt/hostedtoolcache/CodeQL
      # The dependency builds, keyed by Cargo.lock and the toolchain; pull requests only read them.
      - uses: Swatinem/rust-cache@v2
        with:
          shared-key: gate
          save-if: ${{ github.event_name != 'pull_request' && github.event_name != 'merge_group' }}
      - name: Download the DuckDB library again if the restored cache lacks it
        run: scripts/refetch_duckdb_library.sh
      # The evidence snapshot evidence.toml pins; scripts/evidence.sh verifies it after a restore.
      - uses: actions/cache@v4
        with:
          path: .evidence/pinned
          key: evidence-${{ hashFiles('evidence.toml') }}
      - name: Install Python
        run: uv python install 3.12
      - name: Install development tools without the recorder
        run: uv sync --locked --no-group recorder
      # The instruction-count guard in check-ci runs its workloads under callgrind.
      # The runner's Azure Ubuntu mirror sometimes stops resolving, so apt uses archive.ubuntu.com
      # directly. apt still checks the archive's signatures.
      - name: Install valgrind
        run: |
          { sudo grep -rlE 'mirror\+file:|azure\.archive\.ubuntu\.com' /etc/apt || true; } | xargs -r sudo sed -i -E \
            -e 's#mirror\+file:/etc/apt/apt-mirrors\.txt#http://archive.ubuntu.com/ubuntu/#g' \
            -e 's#https?://azure\.archive\.ubuntu\.com/ubuntu/?#http://archive.ubuntu.com/ubuntu/#g'
          for attempt in 1 2 3; do
            if sudo apt-get update -q -o Acquire::Retries=3 &&
              sudo apt-get install -y -q --no-install-recommends -o Acquire::Retries=3 valgrind; then
              exit 0
            fi
            sleep $((attempt * 15))
          done
          exit 1
      # One gate build serves every step: check-ci, the Rust tests, the Python tests and the replay.
      - name: Run static checks and tests
        run: make check-merge test-python
      - name: Find the replay cache's scope
        id: replay
        if: env.REPLAY != 'none'
        run: echo "scope=$(uv run python -m scripts.select_replay digest)" >> "${GITHUB_OUTPUT}"
      # Judgements made under the same full-replay paths, most recently by main's nightly run.
      - name: Restore the replay cache
        if: env.REPLAY == 'changed'
        uses: actions/cache/restore@v4
        with:
          path: target/replay-cache.json
          key: replay-${{ steps.replay.outputs.scope }}-${{ github.run_id }}
          restore-keys: replay-${{ steps.replay.outputs.scope }}-
      - name: Replay recorded translations on DuckDB and check their baselines
        if: env.REPLAY != 'none'
        run: make verify-recorded RESULTS_BASE="${EVIDENCE_BASE}" REPLAY_CACHE=target/replay-cache.json
      - name: Report the replay
        if: env.REPLAY == 'none'
        run: echo "No changed file is in a crate the sqlquack binary is built from, so nothing replays."
      # Pull requests restore the cache saved on their base branch, main's from the nightly run.
      - name: Save the replay cache
        if: env.REPLAY != 'none' && github.event_name != 'pull_request' && github.event_name != 'merge_group'
        uses: actions/cache/save@v4
        with:
          path: target/replay-cache.json
          key: replay-${{ steps.replay.outputs.scope }}-${{ github.run_id }}

  verify:
    name: Verify
    if: always()
    needs: [changes, site, checks]
    runs-on: ubuntu-24.04
    steps:
      - name: Confirm every job the change needs passed
        env:
          CHANGES: ${{ needs.changes.result }}
          SITE: ${{ needs.site.result }}
          SITE_NEEDED: ${{ needs.changes.outputs.site }}
          CHECKS: ${{ needs.checks.result }}
          CHECKS_NEEDED: ${{ needs.changes.outputs.code }}
        run: |
          failed=0
          expect() {
            local name=$1 result=$2 needed=$3
            if [[ "${result}" == "success" || ( "${needed}" != "true" && "${result}" == "skipped" ) ]]; then
              echo "${name}: ${result}"
            else
              echo "${name}: ${result}, but the change needs it" >&2
              failed=1
            fi
          }
          expect "Classify the change" "${CHANGES}" true
          expect "Website" "${SITE}" "${SITE_NEEDED}"
          expect "Checks, tests and replay" "${CHECKS}" "${CHECKS_NEEDED}"
          exit "${failed}"

  duckshop:
    name: Convert DuckShop within the time budget
    if: github.event_name != 'pull_request' && github.event_name != 'merge_group'
    runs-on: ubuntu-24.04
    timeout-minutes: 30
    steps:
      - uses: actions/checkout@v4
      - uses: astral-sh/setup-uv@v5
        with:
          version: "0.12.3"
      - name: Install the pinned Rust toolchain
        run: rustup toolchain install
      - uses: Swatinem/rust-cache@v2
        with:
          shared-key: duckshop
          save-if: ${{ github.event_name != 'pull_request' && github.event_name != 'merge_group' }}
      - name: Download the DuckDB library again if the restored cache lacks it
        run: scripts/refetch_duckdb_library.sh
      - name: Compile DuckShop and DuckShop 10K
        run: make duckshop-compiled
      - name: Convert them
        run: make census-duckshop

  fuzz:
    name: Fuzz tolerant parsing
    if: github.event_name != 'pull_request' && github.event_name != 'merge_group'
    runs-on: ubuntu-24.04
    timeout-minutes: 15
    env:
      # Seconds of fuzzing after the build.
      FUZZ_SECONDS: "90"
    steps:
      - uses: actions/checkout@v4
      - name: Install nightly Rust
        run: rustup toolchain install nightly --profile minimal
      - uses: taiki-e/install-action@v2
        with:
          tool: cargo-fuzz
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: fuzz
          shared-key: fuzz
      - name: Seed the corpus with the corpora's statements
        run: python3 -m scripts.seed_fuzz_corpus --out fuzz/corpus/tolerant
      - name: Fuzz
        working-directory: fuzz
        # The prebuilt cargo-fuzz is a musl binary and would default to its own target.
        run: >-
          cargo +nightly fuzz run --target x86_64-unknown-linux-gnu tolerant corpus/tolerant --
          -max_total_time="$FUZZ_SECONDS" -max_len=4096 -timeout=10 -rss_limit_mb=2048
      - name: Keep any failing input
        if: failure()
        uses: actions/upload-artifact@v4
        with:
          name: fuzz-artifacts
          path: fuzz/artifacts
          if-no-files-found: ignore