diff --git a/.github/workflows/build-distributions.yml b/.github/workflows/build-distributions.yml index 08b7f036c..a14f9e02d 100644 --- a/.github/workflows/build-distributions.yml +++ b/.github/workflows/build-distributions.yml @@ -39,6 +39,13 @@ on: required: false type: string default: dist + dependency-cache-writer: + description: >- + Build only the platforms that use the dependency compile cache and save it; it is saved + only from scheduled runs and pushes on main + required: false + type: boolean + default: false outputs: sha: description: "Commit the distributions were built from" @@ -93,29 +100,41 @@ jobs: name: Build wheel (${{ matrix.platform.name }}) needs: source runs-on: ${{ matrix.platform.runner }} - # Cold dist builds take up to about 2x the 17-31 minute release-profile jobs on Windows and macOS. - timeout-minutes: 120 + # Cold dist builds take up to about 2x the 17-31 minute release-profile jobs on Windows and macOS, + # and a cold macOS arm64 dist build has taken an hour. + timeout-minutes: 150 strategy: fail-fast: false matrix: + # dependency-cache marks the slowest platforms, whose dist builds restore compiled + # dependencies. Linux builds run in maturin's manylinux container, where the host cache + # does not apply. platform: - name: linux-x86_64 runner: ubuntu-24.04 target: x86_64 + dependency-cache: none - name: linux-aarch64 runner: ubuntu-24.04-arm target: aarch64 + dependency-cache: none - name: macos-x86_64 runner: macos-15 target: x86_64 python-architecture: x64 smoke-arch: x86_64 + dependency-cache: restore - name: macos-arm64 runner: macos-15 target: aarch64 + dependency-cache: restore - name: windows-x86_64 runner: windows-2025 target: x86_64 + dependency-cache: none + exclude: + - platform: + dependency-cache: ${{ inputs.dependency-cache-writer && 'none' || 'never' }} steps: - uses: actions/checkout@v4 if: inputs.prebuilt-run-id == '' @@ -129,6 +148,18 @@ jobs: if: inputs.prebuilt-run-id == '' with: toolchain: stable + # Compiled dependencies of dist builds, keyed on the lockfile, toolchain, profile and target. + # Only Dist Dependency Cache saves them, from main, so release builds restore a cache built + # from main and never write one. + - uses: Swatinem/rust-cache@v2 + if: >- + inputs.prebuilt-run-id == '' && inputs.cargo-profile == 'dist' + && matrix.platform.dependency-cache != 'none' + with: + shared-key: dist-${{ matrix.platform.name }}-${{ inputs.cargo-profile }} + save-if: >- + ${{ inputs.dependency-cache-writer && github.ref == 'refs/heads/main' + && (github.event_name == 'push' || github.event_name == 'schedule') }} - uses: actions/setup-node@v4 if: inputs.prebuilt-run-id == '' with: @@ -338,13 +369,26 @@ jobs: curl -sf http://127.0.0.1:8765/runs | grep -q '/_app/immutable/' curl -sfI http://127.0.0.1:8765/theme.js | grep -qi '^content-type: text/javascript' kill "${server_pid}" - - uses: actions/upload-artifact@v4 + # A transient network error in the upload would otherwise cost the whole build, so it retries + # once, replacing any artifact the failed attempt left behind. + - id: upload + uses: actions/upload-artifact@v4 if: inputs.upload + continue-on-error: true + with: + name: dist-wheel-${{ matrix.platform.name }} + path: dist + if-no-files-found: error + retention-days: ${{ inputs.retention-days }} + - name: Retry the distribution upload + uses: actions/upload-artifact@v4 + if: inputs.upload && steps.upload.outcome == 'failure' with: name: dist-wheel-${{ matrix.platform.name }} path: dist if-no-files-found: error retention-days: ${{ inputs.retention-days }} + overwrite: true sdist: name: Build sdist diff --git a/.github/workflows/dist-dependency-cache.yml b/.github/workflows/dist-dependency-cache.yml new file mode 100644 index 000000000..d46321fb0 --- /dev/null +++ b/.github/workflows/dist-dependency-cache.yml @@ -0,0 +1,59 @@ +name: Dist Dependency Cache + +# Saves the compiled dependencies that the macOS wheel jobs of Build Distributions restore for +# release builds. Only this workflow saves them, and only on main, so every release-please branch +# and release tag can read them while no pull request or release run can write them. It builds the +# macOS legs only when main has no cache for the current lockfile and toolchain. It has its own +# concurrency group because a cold macOS dist build takes about an hour, which must not hold back +# the Verify caches that Rust Build Cache refreshes. +on: + push: + branches: [main] + # A new stable toolchain changes the key without a push to main. + schedule: + - cron: "30 4 * * *" + +permissions: + contents: read + +concurrency: + group: dist-dependency-cache-${{ github.ref }} + cancel-in-progress: false + +jobs: + lookup: + name: Look up the macOS caches + runs-on: macos-15 + timeout-minutes: 15 + outputs: + missing: >- + ${{ steps.arm64.outputs.cache-hit != 'true' || steps.x86_64.outputs.cache-hit != 'true' }} + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + # The keys must match the restore step of the Build Distributions wheel jobs. + - name: Look up the macOS arm64 cache + id: arm64 + uses: Swatinem/rust-cache@v2 + with: + shared-key: dist-macos-arm64-dist + lookup-only: true + save-if: false + - name: Look up the macOS x86_64 cache + id: x86_64 + uses: Swatinem/rust-cache@v2 + with: + shared-key: dist-macos-x86_64-dist + lookup-only: true + save-if: false + + distributions: + name: macOS dist dependencies + needs: lookup + if: needs.lookup.outputs.missing == 'true' + uses: ./.github/workflows/build-distributions.yml + with: + sdist: false + dependency-cache-writer: true diff --git a/.github/workflows/rust-cache.yml b/.github/workflows/rust-cache.yml index 625671d7c..6cd39d49c 100644 --- a/.github/workflows/rust-cache.yml +++ b/.github/workflows/rust-cache.yml @@ -4,7 +4,8 @@ name: Rust Build Cache # by a pull request or merge queue run is visible only to that ref, while one saved on main is # visible to every pull request. Each job runs the same cargo builds as the Verify jobs sharing its # key, so the cached artifacts match theirs, and builds only when main has no cache for the current -# lockfile and toolchain. Release distributions never use these caches. +# lockfile and toolchain. Release distributions never use these caches; Dist Dependency Cache saves +# theirs. on: push: branches: [main]