build-distributions.yml

name: Build Distributions

# The single owner of the wheel platform matrix, the maturin settings and the distribution smoke
# tests. Verify builds the wheels for pull requests that touch the native build, Release
# Distributions builds and keeps every distribution for a release pull request, and Publish either
# smoke tests and reuses those distributions or builds them here from the release tag.
on:
  workflow_call:
    inputs:
      ref:
        description: "Commit, tag or branch to build; empty builds the triggering commit"
        required: false
        type: string
        default: ""
      sdist:
        description: "Also build the source distributions"
        required: false
        type: boolean
        default: true
      upload:
        description: "Upload the distributions and a manifest recording their source commit and tree"
        required: false
        type: boolean
        default: false
      prebuilt-run-id:
        description: >-
          Release Distributions run whose distributions are smoke tested and uploaded instead of
          building; its manifest must record the tree of ref. The caller grants actions: read.
        required: false
        type: string
        default: ""
      retention-days:
        description: "Retention of uploaded artifacts in days; 0 uses the repository default"
        required: false
        type: number
        default: 0
      cargo-profile:
        description: "Cargo profile of the native builds: dist for released wheels, release for quicker checks"
        required: false
        type: string
        default: dist
      dependency-cache-writer:
        description: >-
          Build only the platforms that use the dependency compile cache and save it; it is saved
          only from scheduled runs and pushes on main
        required: false
        type: boolean
        default: false
    outputs:
      sha:
        description: "Commit the distributions were built from"
        value: ${{ jobs.source.outputs.sha }}
      tree:
        description: "Tree of the commit the distributions were built from"
        value: ${{ jobs.source.outputs.tree }}

jobs:
  source:
    name: Resolve source
    runs-on: ubuntu-24.04
    outputs:
      sha: ${{ steps.source.outputs.sha }}
      tree: ${{ steps.source.outputs.tree }}
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref }}
      - name: Pin the source commit
        id: source
        run: |
          sha="$(git rev-parse HEAD)"
          tree="$(git rev-parse 'HEAD^{tree}')"
          echo "sha=${sha}" >> "${GITHUB_OUTPUT}"
          echo "tree=${tree}" >> "${GITHUB_OUTPUT}"
          echo "Distributions for commit ${sha} with tree ${tree}" >> "${GITHUB_STEP_SUMMARY}"
      - name: Download the prebuilt manifest
        if: inputs.prebuilt-run-id != ''
        uses: actions/download-artifact@v4
        with:
          name: release-manifest
          path: ${{ runner.temp }}/prebuilt
          run-id: ${{ inputs.prebuilt-run-id }}
          github-token: ${{ github.token }}
      - name: Require prebuilt distributions from this exact tree
        if: inputs.prebuilt-run-id != ''
        env:
          MANIFEST: ${{ runner.temp }}/prebuilt/release-manifest.json
          PREBUILT_RUN_ID: ${{ inputs.prebuilt-run-id }}
          TREE: ${{ steps.source.outputs.tree }}
        run: |
          recorded="$(jq -r '.tree // empty' "${MANIFEST}")"
          if [[ "${recorded}" != "${TREE}" ]]; then
            echo "Run ${PREBUILT_RUN_ID} built tree ${recorded:-<none>}, not ${TREE}" >&2
            exit 1
          fi
          echo "Reusing distributions of run ${PREBUILT_RUN_ID}, built from tree ${TREE}" \
            >> "${GITHUB_STEP_SUMMARY}"

  wheels:
    name: Build wheel (${{ matrix.platform.name }})
    needs: source
    runs-on: ${{ matrix.platform.runner }}
    # Cold dist builds take up to about 2x the 17-31 minute release-profile jobs on Windows and macOS,
    # and a cold macOS arm64 dist build has taken an hour.
    timeout-minutes: 150
    strategy:
      fail-fast: false
      matrix:
        # dependency-cache marks the slowest platforms, whose dist builds restore compiled
        # dependencies. Linux builds run in maturin's manylinux container, where the host cache
        # does not apply.
        platform:
          - name: linux-x86_64
            runner: ubuntu-24.04
            target: x86_64
            dependency-cache: none
          - name: linux-aarch64
            runner: ubuntu-24.04-arm
            target: aarch64
            dependency-cache: none
          - name: macos-x86_64
            runner: macos-15
            target: x86_64
            python-architecture: x64
            smoke-arch: x86_64
            dependency-cache: restore
          - name: macos-arm64
            runner: macos-15
            target: aarch64
            dependency-cache: restore
          - name: windows-x86_64
            runner: windows-2025
            target: x86_64
            dependency-cache: none
        exclude:
          - platform:
              dependency-cache: ${{ inputs.dependency-cache-writer && 'none' || 'never' }}
    steps:
      - uses: actions/checkout@v4
        if: inputs.prebuilt-run-id == ''
        with:
          ref: ${{ needs.source.outputs.sha }}
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          architecture: ${{ matrix.platform.python-architecture }}
      - uses: dtolnay/rust-toolchain@master
        if: inputs.prebuilt-run-id == ''
        with:
          toolchain: stable
      # Compiled dependencies of dist builds, keyed on the lockfile, toolchain, profile and target.
      # Only Dist Dependency Cache saves them, from main, so release builds restore a cache built
      # from main and never write one.
      - uses: Swatinem/rust-cache@v2
        if: >-
          inputs.prebuilt-run-id == '' && inputs.cargo-profile == 'dist'
          && matrix.platform.dependency-cache != 'none'
        with:
          shared-key: dist-${{ matrix.platform.name }}-${{ inputs.cargo-profile }}
          save-if: >-
            ${{ inputs.dependency-cache-writer && github.ref == 'refs/heads/main'
            && (github.event_name == 'push' || github.event_name == 'schedule') }}
      - uses: actions/setup-node@v4
        if: inputs.prebuilt-run-id == ''
        with:
          node-version: "22"
          cache: npm
          cache-dependency-path: ui/package-lock.json
      - name: Build the web UI that sqb server embeds
        if: inputs.prebuilt-run-id == ''
        working-directory: ui
        run: |
          npm ci
          npm run build
      # A Linux dist build can need more memory than the hosted runner has; see the action.
      - name: Add swap and limit cargo jobs for the Linux dist build
        if: inputs.prebuilt-run-id == '' && runner.os == 'Linux' && inputs.cargo-profile == 'dist'
        uses: ./.github/actions/dist-build-memory
        with:
          step: start
      # maturin-action downloads maturin from GitHub releases, which can fail on its own. Each
      # build retries once; macOS and Windows then fall back to the same maturin from PyPI. Linux
      # builds need the action's manylinux container, so they have no fallback.
      - id: extension
        uses: PyO3/maturin-action@v1
        if: inputs.prebuilt-run-id == ''
        continue-on-error: true
        with:
          command: build
          # One platform records a cargo build timing report for build time work.
          args: >-
            --profile ${{ inputs.cargo-profile }} --out dist --features extension-module
            ${{ matrix.platform.name == 'linux-x86_64' && '--timings' || '' }}
          target: ${{ matrix.platform.target }}
          manylinux: auto
      - id: extension-retry
        name: Retry the extension build
        uses: PyO3/maturin-action@v1
        if: inputs.prebuilt-run-id == '' && steps.extension.outcome == 'failure'
        continue-on-error: ${{ runner.os != 'Linux' }}
        with:
          command: build
          # One platform records a cargo build timing report for build time work.
          args: >-
            --profile ${{ inputs.cargo-profile }} --out dist --features extension-module
            ${{ matrix.platform.name == 'linux-x86_64' && '--timings' || '' }}
          target: ${{ matrix.platform.target }}
          manylinux: auto
      - name: Build the extension with maturin from PyPI
        if: >-
          inputs.prebuilt-run-id == '' && runner.os != 'Linux'
          && steps.extension.outcome == 'failure' && steps.extension-retry.outcome == 'failure'
        shell: bash
        env:
          MATURIN_ARGS: >-
            --profile ${{ inputs.cargo-profile }} --out dist --features extension-module
          TARGET: ${{ matrix.platform.target }}
        run: scripts/maturin_from_pypi.sh build ${MATURIN_ARGS}
      - id: cli
        uses: PyO3/maturin-action@v1
        if: inputs.prebuilt-run-id == ''
        continue-on-error: true
        # On macOS maturin links the extension with these flags for every crate, which changes
        # their build hash. Passing the same flags here lets sqb reuse the crates the extension
        # build just compiled instead of compiling them again; sqb has no undefined symbols, and
        # the smoke test below runs it.
        env:
          RUSTFLAGS: >-
            ${{ runner.os == 'macOS' && '-C link-arg=-undefined -C link-arg=dynamic_lookup' || '' }}
          # A wheel must never ship a sqb server without its UI.
          SQLBUILD_UI_REQUIRED: "1"
        with:
          command: build
          args: >-
            --profile ${{ inputs.cargo-profile }} --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml --features ui
            ${{ matrix.platform.name == 'linux-x86_64' && '--timings' || '' }}
          target: ${{ matrix.platform.target }}
          manylinux: auto
      - id: cli-retry
        name: Retry the sqb build
        uses: PyO3/maturin-action@v1
        if: inputs.prebuilt-run-id == '' && steps.cli.outcome == 'failure'
        continue-on-error: ${{ runner.os != 'Linux' }}
        env:
          RUSTFLAGS: >-
            ${{ runner.os == 'macOS' && '-C link-arg=-undefined -C link-arg=dynamic_lookup' || '' }}
          # A wheel must never ship a sqb server without its UI.
          SQLBUILD_UI_REQUIRED: "1"
        with:
          command: build
          args: >-
            --profile ${{ inputs.cargo-profile }} --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml --features ui
            ${{ matrix.platform.name == 'linux-x86_64' && '--timings' || '' }}
          target: ${{ matrix.platform.target }}
          manylinux: auto
      - name: Build sqb with maturin from PyPI
        if: >-
          inputs.prebuilt-run-id == '' && runner.os != 'Linux'
          && steps.cli.outcome == 'failure' && steps.cli-retry.outcome == 'failure'
        shell: bash
        env:
          RUSTFLAGS: >-
            ${{ runner.os == 'macOS' && '-C link-arg=-undefined -C link-arg=dynamic_lookup' || '' }}
          # A wheel must never ship a sqb server without its UI.
          SQLBUILD_UI_REQUIRED: "1"
          MATURIN_ARGS: >-
            --profile ${{ inputs.cargo-profile }} --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml --features ui
          TARGET: ${{ matrix.platform.target }}
        run: scripts/maturin_from_pypi.sh build ${MATURIN_ARGS}
      - name: Report memory use of the Linux dist build
        if: >-
          always() && inputs.prebuilt-run-id == '' && runner.os == 'Linux'
          && inputs.cargo-profile == 'dist'
        uses: ./.github/actions/dist-build-memory
        with:
          step: report
      - name: Upload the cargo build timing reports
        if: inputs.prebuilt-run-id == '' && matrix.platform.name == 'linux-x86_64'
        uses: actions/upload-artifact@v4
        with:
          name: cargo-timings-${{ matrix.platform.name }}
          path: target/cargo-timings/cargo-timing-*.html
          if-no-files-found: warn
          retention-days: 30
      - name: Download prebuilt wheels
        if: inputs.prebuilt-run-id != ''
        uses: actions/download-artifact@v4
        with:
          name: dist-wheel-${{ matrix.platform.name }}
          path: dist
          run-id: ${{ inputs.prebuilt-run-id }}
          github-token: ${{ github.token }}
      - name: Smoke test installed wheels and native console entry points
        shell: bash
        env:
          SMOKE_ARCH: ${{ matrix.platform.smoke-arch }}
        run: |
          python -m venv "${RUNNER_TEMP}/wheel-smoke"
          if [[ -x "${RUNNER_TEMP}/wheel-smoke/Scripts/python.exe" ]]; then
            smoke_python="${RUNNER_TEMP}/wheel-smoke/Scripts/python.exe"
          else
            smoke_python="${RUNNER_TEMP}/wheel-smoke/bin/python"
          fi
          # A cross-compiled wheel runs under Rosetta; the cached Python is universal, so force
          # the wheel's architecture or it starts as arm64 and rejects the wheel.
          run_as=()
          if [[ -n "${SMOKE_ARCH}" ]]; then
            run_as=(arch "-${SMOKE_ARCH}")
          fi
          "${run_as[@]}" "${smoke_python}" -m pip install --find-links dist dist/*.whl
          cd "${RUNNER_TEMP}"
          "${run_as[@]}" "${smoke_python}" -c "import sqlbuild._native"
          smoke_scripts="$(dirname "${smoke_python}")"
          "${run_as[@]}" "${smoke_scripts}/sqb" --version
          "${run_as[@]}" "${smoke_scripts}/sqb" --help
          "${run_as[@]}" "${smoke_scripts}/sqlbuild" --help
          "${run_as[@]}" "${smoke_scripts}/sqlbuild" --version
          # The sqlbuild forwarder returns sqb's own exit code; an unknown command is a usage error.
          code=0
          "${run_as[@]}" "${smoke_scripts}/sqlbuild" no-such-command > /dev/null 2>&1 || code=$?
          if [[ "${code}" != "2" ]]; then
            echo "sqlbuild no-such-command exited ${code}; sqb exits 2" >&2
            exit 1
          fi
          exe="${{ runner.os == 'Windows' && '.exe' || '' }}"
          {
            echo "### Sizes (${{ matrix.platform.name }})"
            echo
            echo "| File | Bytes |"
            echo "| --- | ---: |"
            for file in "${GITHUB_WORKSPACE}"/dist/*.whl "${smoke_scripts}/sqb${exe}" "${smoke_scripts}/sqlbuild${exe}"; do
              echo "| $(basename "${file}") | $(wc -c < "${file}" | tr -d ' ') |"
            done
          } >> "${GITHUB_STEP_SUMMARY}"
          if [[ "${{ inputs.prebuilt-run-id == '' }}" == "true" ]]; then
            cd "${GITHUB_WORKSPACE}"
            "${run_as[@]}" "${smoke_python}" -m tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.helpers \
              "${smoke_scripts}/sqb${{ runner.os == 'Windows' && '.exe' || '' }}"
          fi
      - name: Check wheel contents and run a native command
        if: matrix.platform.name == 'linux-x86_64'
        run: |
          wheel="$(ls dist/sqlbuild-[0-9]*.whl)"
          if unzip -l "${wheel}" | grep -E '(__pycache__/|\.pyc$|\.tmp$)'; then
            echo "Wheel contains generated or temporary script files"
            exit 1
          fi
          unzip -l "${wheel}" | grep 'sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md'
          smoke="${RUNNER_TEMP}/wheel-smoke/bin"
          project="${RUNNER_TEMP}/sqlbuild-scope-smoke"
          cd "${RUNNER_TEMP}"
          "${smoke}/python" -c \
            "import sqlbuild._native as native; assert native.semantic_help_catalogue()"
          mkdir -p "${project}/models"
          printf 'name = "scope_smoke"\nadapter = "duckdb"\n' > "${project}/sqlbuild_project.toml"
          printf 'MODEL();\nSELECT 1 AS id\n' > "${project}/models/orders.sql"
          "${smoke}/sqb" --project-dir "${project}" scope model:orders --json
          "${smoke}/sqlbuild" --project-dir "${project}" scope model:orders
          # The packaged sqb server must serve the embedded UI, not only the API.
          server_dir="${RUNNER_TEMP}/ui-server-smoke"
          mkdir -p "${server_dir}"
          (cd "${server_dir}" && env -u SQLBUILD_CONTROL_DATABASE_URL \
            "${smoke}/sqb" server --host 127.0.0.1 --port 8765 --no-scheduler) &
          server_pid=$!
          for _ in $(seq 1 60); do
            curl -sf -o /dev/null http://127.0.0.1:8765/api/openapi.json && break
            sleep 1
          done
          curl -sf http://127.0.0.1:8765/runs | grep -q '/_app/immutable/'
          curl -sfI http://127.0.0.1:8765/theme.js | grep -qi '^content-type: text/javascript'
          kill "${server_pid}"
      # A transient network error in the upload would otherwise cost the whole build, so it retries
      # once, replacing any artifact the failed attempt left behind.
      - id: upload
        uses: actions/upload-artifact@v4
        if: inputs.upload
        continue-on-error: true
        with:
          name: dist-wheel-${{ matrix.platform.name }}
          path: dist
          if-no-files-found: error
          retention-days: ${{ inputs.retention-days }}
      - name: Retry the distribution upload
        uses: actions/upload-artifact@v4
        if: inputs.upload && steps.upload.outcome == 'failure'
        with:
          name: dist-wheel-${{ matrix.platform.name }}
          path: dist
          if-no-files-found: error
          retention-days: ${{ inputs.retention-days }}
          overwrite: true

  sdist:
    name: Build sdist
    needs: source
    if: inputs.sdist
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v4
        if: inputs.prebuilt-run-id == ''
        with:
          ref: ${{ needs.source.outputs.sha }}
      - uses: dtolnay/rust-toolchain@master
        if: inputs.prebuilt-run-id == ''
        with:
          toolchain: stable
      - id: sdist
        uses: PyO3/maturin-action@v1
        if: inputs.prebuilt-run-id == ''
        continue-on-error: true
        with:
          command: sdist
          args: --out dist
      - id: sdist-retry
        name: Retry the source distribution
        uses: PyO3/maturin-action@v1
        if: inputs.prebuilt-run-id == '' && steps.sdist.outcome == 'failure'
        continue-on-error: true
        with:
          command: sdist
          args: --out dist
      - name: Build the source distribution with maturin from PyPI
        if: >-
          inputs.prebuilt-run-id == '' && steps.sdist.outcome == 'failure'
          && steps.sdist-retry.outcome == 'failure'
        run: scripts/maturin_from_pypi.sh sdist --out dist
      - id: sdist-cli
        uses: PyO3/maturin-action@v1
        if: inputs.prebuilt-run-id == ''
        continue-on-error: true
        with:
          command: sdist
          args: --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml
      - id: sdist-cli-retry
        name: Retry the sqb source distribution
        uses: PyO3/maturin-action@v1
        if: inputs.prebuilt-run-id == '' && steps.sdist-cli.outcome == 'failure'
        continue-on-error: true
        with:
          command: sdist
          args: --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml
      - name: Build the sqb source distribution with maturin from PyPI
        if: >-
          inputs.prebuilt-run-id == '' && steps.sdist-cli.outcome == 'failure'
          && steps.sdist-cli-retry.outcome == 'failure'
        run: scripts/maturin_from_pypi.sh sdist --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml
      - name: Download prebuilt source distributions
        if: inputs.prebuilt-run-id != ''
        uses: actions/download-artifact@v4
        with:
          name: dist-sdist
          path: dist
          run-id: ${{ inputs.prebuilt-run-id }}
          github-token: ${{ github.token }}
      - name: Inspect source distribution contents
        run: |
          for archive in dist/*.tar.gz; do
            tar -tzf "$archive" > "${RUNNER_TEMP}/archive-contents.txt"
            if grep -E '(__pycache__/|\.pyc$|\.tmp$)' "${RUNNER_TEMP}/archive-contents.txt"; then
              echo "Source distribution contains generated or temporary script files"
              exit 1
            fi
          done
      - uses: actions/upload-artifact@v4
        if: inputs.upload
        with:
          name: dist-sdist
          path: dist
          if-no-files-found: error
          retention-days: ${{ inputs.retention-days }}

  manifest:
    name: Record distribution source
    needs: [source, wheels, sdist]
    if: >-
      ${{ !cancelled() && inputs.upload && needs.wheels.result == 'success'
      && (needs.sdist.result == 'success' || needs.sdist.result == 'skipped') }}
    runs-on: ubuntu-24.04
    steps:
      - name: Write the manifest
        env:
          SHA: ${{ needs.source.outputs.sha }}
          TREE: ${{ needs.source.outputs.tree }}
        run: |
          jq -n --arg sha "${SHA}" --arg tree "${TREE}" --arg run_id "${GITHUB_RUN_ID}" \
            '{sha: $sha, tree: $tree, run_id: $run_id}' \
            > "${RUNNER_TEMP}/release-manifest.json"
          cat "${RUNNER_TEMP}/release-manifest.json"
      - uses: actions/upload-artifact@v4
        with:
          name: release-manifest
          path: ${{ runner.temp }}/release-manifest.json
          if-no-files-found: error
          retention-days: ${{ inputs.retention-days }}