ci: cache macOS dist dependencies and retry wheel uploads

.github/workflows/build-distributions.yml

@@ -39,6 +39,13 @@
         required: false
         type: string
         default: dist
+      dependency-cache-writer:
+        description: >-
+          Build only the platforms that use the dependency compile cache and save it; it is saved
+          only from scheduled runs and pushes on main
+        required: false
+        type: boolean
+        default: false
     outputs:
       sha:
         description: "Commit the distributions were built from"
@@ -93,29 +100,41 @@
     name: Build wheel (${{ matrix.platform.name }})
     needs: source
     runs-on: ${{ matrix.platform.runner }}
-    # Cold dist builds take up to about 2x the 17-31 minute release-profile jobs on Windows and macOS.
-    timeout-minutes: 120
+    # Cold dist builds take up to about 2x the 17-31 minute release-profile jobs on Windows and macOS,
+    # and a cold macOS arm64 dist build has taken an hour.
+    timeout-minutes: 150
     strategy:
       fail-fast: false
       matrix:
+        # dependency-cache marks the slowest platforms, whose dist builds restore compiled
+        # dependencies. Linux builds run in maturin's manylinux container, where the host cache
+        # does not apply.
         platform:
           - name: linux-x86_64
             runner: ubuntu-24.04
             target: x86_64
+            dependency-cache: none
           - name: linux-aarch64
             runner: ubuntu-24.04-arm
             target: aarch64
+            dependency-cache: none
           - name: macos-x86_64
             runner: macos-15
             target: x86_64
             python-architecture: x64
             smoke-arch: x86_64
+            dependency-cache: restore
           - name: macos-arm64
             runner: macos-15
             target: aarch64
+            dependency-cache: restore
           - name: windows-x86_64
             runner: windows-2025
             target: x86_64
+            dependency-cache: none
+        exclude:
+          - platform:
+              dependency-cache: ${{ inputs.dependency-cache-writer && 'none' || 'never' }}
     steps:
       - uses: actions/checkout@v4
         if: inputs.prebuilt-run-id == ''
@@ -129,28 +148,40 @@
         if: inputs.prebuilt-run-id == ''
         with:
           toolchain: stable
+      # Compiled dependencies of dist builds, keyed on the lockfile, toolchain, profile and target.
+      # Only Dist Dependency Cache saves them, from main, so release builds restore a cache built
+      # from main and never write one.
+      - uses: Swatinem/rust-cache@v2
+        if: >-
+          inputs.prebuilt-run-id == '' && inputs.cargo-profile == 'dist'
+          && matrix.platform.dependency-cache != 'none'
+        with:
+          shared-key: dist-${{ matrix.platform.name }}-${{ inputs.cargo-profile }}
+          save-if: >-
+            ${{ inputs.dependency-cache-writer && github.ref == 'refs/heads/main'
+            && (github.event_name == 'push' || github.event_name == 'schedule') }}
       - uses: actions/setup-node@v4
         if: inputs.prebuilt-run-id == ''
         with:
           node-version: "22"
           cache: npm
           cache-dependency-path: ui/package-lock.json
       - name: Build the web UI that sqb server embeds
         if: inputs.prebuilt-run-id == ''
         working-directory: ui
         run: |
           npm ci
           npm run build
       # A Linux dist build can need more memory than the hosted runner has; see the action.
       - name: Add swap and limit cargo jobs for the Linux dist build
         if: inputs.prebuilt-run-id == '' && runner.os == 'Linux' && inputs.cargo-profile == 'dist'
         uses: ./.github/actions/dist-build-memory
         with:
           step: start
       # maturin-action downloads maturin from GitHub releases, which can fail on its own. Each
       # build retries once; macOS and Windows then fall back to the same maturin from PyPI. Linux
       # builds need the action's manylinux container, so they have no fallback.
       - id: extension
         uses: PyO3/maturin-action@v1
         if: inputs.prebuilt-run-id == ''
         continue-on-error: true
@@ -236,14 +267,14 @@
             --profile ${{ inputs.cargo-profile }} --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml --features ui
           TARGET: ${{ matrix.platform.target }}
         run: scripts/maturin_from_pypi.sh build ${MATURIN_ARGS}
       - name: Report memory use of the Linux dist build
         if: >-
           always() && inputs.prebuilt-run-id == '' && runner.os == 'Linux'
           && inputs.cargo-profile == 'dist'
         uses: ./.github/actions/dist-build-memory
         with:
           step: report
       - name: Upload the cargo build timing reports
         if: inputs.prebuilt-run-id == '' && matrix.platform.name == 'linux-x86_64'
         uses: actions/upload-artifact@v4
         with:
@@ -259,54 +290,54 @@
           path: dist
           run-id: ${{ inputs.prebuilt-run-id }}
           github-token: ${{ github.token }}
       - name: Smoke test installed wheels and native console entry points
         shell: bash
         env:
           SMOKE_ARCH: ${{ matrix.platform.smoke-arch }}
         run: |
           python -m venv "${RUNNER_TEMP}/wheel-smoke"
           if [[ -x "${RUNNER_TEMP}/wheel-smoke/Scripts/python.exe" ]]; then
             smoke_python="${RUNNER_TEMP}/wheel-smoke/Scripts/python.exe"
           else
             smoke_python="${RUNNER_TEMP}/wheel-smoke/bin/python"
           fi
           # A cross-compiled wheel runs under Rosetta; the cached Python is universal, so force
           # the wheel's architecture or it starts as arm64 and rejects the wheel.
           run_as=()
           if [[ -n "${SMOKE_ARCH}" ]]; then
             run_as=(arch "-${SMOKE_ARCH}")
           fi
           "${run_as[@]}" "${smoke_python}" -m pip install --find-links dist dist/*.whl
           cd "${RUNNER_TEMP}"
           "${run_as[@]}" "${smoke_python}" -c "import sqlbuild._native"
           smoke_scripts="$(dirname "${smoke_python}")"
           "${run_as[@]}" "${smoke_scripts}/sqb" --version
           "${run_as[@]}" "${smoke_scripts}/sqb" --help
           "${run_as[@]}" "${smoke_scripts}/sqlbuild" --help
           "${run_as[@]}" "${smoke_scripts}/sqlbuild" --version
           # The sqlbuild forwarder returns sqb's own exit code; an unknown command is a usage error.
           code=0
           "${run_as[@]}" "${smoke_scripts}/sqlbuild" no-such-command > /dev/null 2>&1 || code=$?
           if [[ "${code}" != "2" ]]; then
             echo "sqlbuild no-such-command exited ${code}; sqb exits 2" >&2
             exit 1
           fi
           exe="${{ runner.os == 'Windows' && '.exe' || '' }}"
           {
             echo "### Sizes (${{ matrix.platform.name }})"
             echo
             echo "| File | Bytes |"
             echo "| --- | ---: |"
             for file in "${GITHUB_WORKSPACE}"/dist/*.whl "${smoke_scripts}/sqb${exe}" "${smoke_scripts}/sqlbuild${exe}"; do
               echo "| $(basename "${file}") | $(wc -c < "${file}" | tr -d ' ') |"
             done
           } >> "${GITHUB_STEP_SUMMARY}"
           if [[ "${{ inputs.prebuilt-run-id == '' }}" == "true" ]]; then
             cd "${GITHUB_WORKSPACE}"
             "${run_as[@]}" "${smoke_python}" -m tests.integration.src.sqlbuild.compiler.compiled_project.publication.wheel.helpers \
               "${smoke_scripts}/sqb${{ runner.os == 'Windows' && '.exe' || '' }}"
           fi
       - name: Check wheel contents and run a native command
         if: matrix.platform.name == 'linux-x86_64'
         run: |
           wheel="$(ls dist/sqlbuild-[0-9]*.whl)"
@@ -338,13 +369,26 @@
           curl -sf http://127.0.0.1:8765/runs | grep -q '/_app/immutable/'
           curl -sfI http://127.0.0.1:8765/theme.js | grep -qi '^content-type: text/javascript'
           kill "${server_pid}"
-      - uses: actions/upload-artifact@v4
+      # A transient network error in the upload would otherwise cost the whole build, so it retries
+      # once, replacing any artifact the failed attempt left behind.
+      - id: upload
+        uses: actions/upload-artifact@v4
         if: inputs.upload
+        continue-on-error: true
+        with:
+          name: dist-wheel-${{ matrix.platform.name }}
+          path: dist
+          if-no-files-found: error
+          retention-days: ${{ inputs.retention-days }}
+      - name: Retry the distribution upload
+        uses: actions/upload-artifact@v4
+        if: inputs.upload && steps.upload.outcome == 'failure'
         with:
           name: dist-wheel-${{ matrix.platform.name }}
           path: dist
           if-no-files-found: error
           retention-days: ${{ inputs.retention-days }}
+          overwrite: true
 
   sdist:
     name: Build sdist

.github/workflows/dist-dependency-cache.yml

@@ -0,0 +1,59 @@
+name: Dist Dependency Cache
+
+# Saves the compiled dependencies that the macOS wheel jobs of Build Distributions restore for
+# release builds. Only this workflow saves them, and only on main, so every release-please branch
+# and release tag can read them while no pull request or release run can write them. It builds the
+# macOS legs only when main has no cache for the current lockfile and toolchain. It has its own
+# concurrency group because a cold macOS dist build takes about an hour, which must not hold back
+# the Verify caches that Rust Build Cache refreshes.
+on:
+  push:
+    branches: [main]
+  # A new stable toolchain changes the key without a push to main.
+  schedule:
+    - cron: "30 4 * * *"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: dist-dependency-cache-${{ github.ref }}
+  cancel-in-progress: false
+
+jobs:
+  lookup:
+    name: Look up the macOS caches
+    runs-on: macos-15
+    timeout-minutes: 15
+    outputs:
+      missing: >-
+        ${{ steps.arm64.outputs.cache-hit != 'true' || steps.x86_64.outputs.cache-hit != 'true' }}
+    steps:
+      - uses: actions/checkout@v4
+      - uses: dtolnay/rust-toolchain@master
+        with:
+          toolchain: stable
+      # The keys must match the restore step of the Build Distributions wheel jobs.
+      - name: Look up the macOS arm64 cache
+        id: arm64
+        uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: dist-macos-arm64-dist
+          lookup-only: true
+          save-if: false
+      - name: Look up the macOS x86_64 cache
+        id: x86_64
+        uses: Swatinem/rust-cache@v2
+        with:
+          shared-key: dist-macos-x86_64-dist
+          lookup-only: true
+          save-if: false
+
+  distributions:
+    name: macOS dist dependencies
+    needs: lookup
+    if: needs.lookup.outputs.missing == 'true'
+    uses: ./.github/workflows/build-distributions.yml
+    with:
+      sdist: false
+      dependency-cache-writer: true

.github/workflows/rust-cache.yml

@@ -4,7 +4,8 @@ name: Rust Build Cache
 # by a pull request or merge queue run is visible only to that ref, while one saved on main is
 # visible to every pull request. Each job runs the same cargo builds as the Verify jobs sharing its
 # key, so the cached artifacts match theirs, and builds only when main has no cache for the current
-# lockfile and toolchain. Release distributions never use these caches.
+# lockfile and toolchain. Release distributions never use these caches; Dist Dependency Cache saves
+# theirs.
 on:
   push:
     branches: [main]