build-wheel.yml

# SPDX-FileCopyrightText: Copyright (c) 2024-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# SPDX-License-Identifier: Apache-2.0

on:
  workflow_call:
    inputs:
      host-platform:
        required: true
        type: string
      cuda-version:
        required: true
        type: string
      cuda-channel:
        required: false
        type: string
        default: stable
      prev-cuda-version:
        required: true
        type: string
      python-versions:
        required: false
        type: string
        default: '["3.10", "3.11", "3.12", "3.13", "3.14", "3.14t", "3.15", "3.15t"]'
      single-cuda-major:
        description: "Build wheels for only the current CUDA major; skip the prior-major build and wheel merge"
        required: false
        type: boolean
        default: false
      build-pathfinder:
        required: false
        type: boolean
        default: true
      build-bindings:
        required: false
        type: boolean
        default: true
      build-core:
        required: false
        type: boolean
        default: true
      build-python:
        required: false
        type: boolean
        default: true
      test-bindings:
        required: false
        type: boolean
        default: true
      test-core:
        required: false
        type: boolean
        default: true
      test-python-version:
        description: "Build compiled test artifacts only for this Python version; empty means all"
        required: false
        type: string
        default: ""
      baseline-run-id:
        required: false
        type: string
        default: ""
      baseline-sha:
        required: false
        type: string
        default: ""

defaults:
  run:
    shell: bash --noprofile --norc -xeuo pipefail {0}

permissions:
  actions: read
  contents: read  # This is required for actions/checkout

jobs:
  build:
    strategy:
      fail-fast: false
      matrix:
        python-version: ${{ fromJSON(inputs.python-versions) }}
    name: py${{ matrix.python-version }}
    runs-on: ${{ (inputs.host-platform == 'linux-64' && 'linux-amd64-cpu8') ||
                 (inputs.host-platform == 'linux-aarch64' && 'linux-arm64-cpu8') ||
                 (inputs.host-platform == 'win-64' && 'windows-2022') ||
                 (inputs.host-platform == 'win-arm64' && 'windows-11-arm') }}
    env:
      BUILD_TEST_ARTIFACTS: ${{ inputs.test-python-version == '' || matrix.python-version == inputs.test-python-version }}
    steps:
      - name: Checkout ${{ github.event.repository.name }}
        uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0  # v7.0.0
        with:
          # Treeless clone: setuptools-scm needs the commit graph (`git describe`)
          # but not historical blobs.
          fetch-depth: 0
          filter: blob:none

      - name: Install latest rapidsai/sccache
        if: ${{ startsWith(inputs.host-platform, 'linux') && (inputs.build-bindings || inputs.build-core) }}
        run: |
          curl -fsSL "https://github.com/rapidsai/sccache/releases/latest/download/sccache-$(uname -m)-unknown-linux-musl.tar.gz" \
            | sudo tar -C /usr/local/bin -xvzf - --wildcards --strip-components=1 -x '*/sccache'
          echo "SCCACHE_PATH=/usr/local/bin/sccache" >> "$GITHUB_ENV"

      # xref: https://github.com/orgs/community/discussions/42856#discussioncomment-7678867
      - name: Adding addtional GHA cache-related env vars
        if: ${{ inputs.build-bindings || inputs.build-core }}
        uses: actions/github-script@v9
        with:
          script: |
            core.exportVariable('ACTIONS_CACHE_SERVICE_V2', 'on');
            core.exportVariable('ACTIONS_CACHE_URL', process.env['ACTIONS_CACHE_URL'] || '');
            core.exportVariable('ACTIONS_RESULTS_URL', process.env['ACTIONS_RESULTS_URL'] || '');
            core.exportVariable('ACTIONS_RUNTIME_URL', process.env['ACTIONS_RUNTIME_URL'] || '');
            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env['ACTIONS_RUNTIME_TOKEN'] || '');

      - name: Setup proxy cache
        uses: nv-gha-runners/setup-proxy-cache@main
        continue-on-error: true
        # Skip cache on GitHub-hosted Windows runners.
        if: ${{ !startsWith(inputs.host-platform, 'win') }}
        with:
          enable-apt: true

      - name: Set up Python
        id: setup-python1
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6.3.0
        with:
          # WAR: setup-python is not relocatable, and cibuildwheel hard-wires to 3.12...
          # see https://github.com/actions/setup-python/issues/871
          python-version: "3.12"
          architecture: ${{ ((inputs.host-platform == 'linux-aarch64' || inputs.host-platform == 'win-arm64') && 'arm64') || 'x64' }}

      - name: Set up MSVC
        if: ${{ startsWith(inputs.host-platform, 'win') && (inputs.build-bindings || inputs.build-core || inputs.test-bindings || inputs.test-core) }}
        uses: step-security/msvc-dev-cmd@22c98154b708dbd743e6f27a933cf6ceba3305c4  # v1.13.1
        with:
          arch: ${{ (inputs.host-platform == 'win-arm64' && 'arm64') || 'x64' }}

      - name: Verify Windows ARM64 runner
        if: ${{ inputs.host-platform == 'win-arm64' }}
        run: |
          python -c "import platform; machine = platform.machine().lower(); print(machine); assert machine in {'arm64', 'aarch64'}"

      - name: Set up yq
        # GitHub made an unprofessional decision to not provide it in their Windows VMs,
        # see https://github.com/actions/runner-images/issues/7443.
        if: ${{ startsWith(inputs.host-platform, 'win') && inputs.build-core && !inputs.single-cuda-major }}
        env:
          YQ_VERSION: v4.52.5
          YQ_ARCH: ${{ (inputs.host-platform == 'win-arm64' && 'arm64') || 'amd64' }}
          YQ_SHA256: ${{ (inputs.host-platform == 'win-arm64' && '236867affa7f18701d4c763cf16b6df962cf4f7e89a8570a5954cf94a38f41c7') || '47594981f3848a4b4447494adeca9555f908f7cf0a89c4da3fd0243a4631da1c' }}
          YQ_DIR: yq
        shell: pwsh -command ". '{0}'"
        run: |
          $yqUrl = "https://github.com/mikefarah/yq/releases/download/${env:YQ_VERSION}/yq_windows_${env:YQ_ARCH}.exe"
          mkdir -Force -ErrorAction SilentlyContinue "${env:YQ_DIR}" | Out-Null
          Invoke-WebRequest -UseBasicParsing -OutFile "${env:YQ_DIR}/yq.exe" -Uri "$yqUrl"
          $hash = (Get-FileHash -Algorithm SHA256 "${env:YQ_DIR}/yq.exe").Hash.ToLower()
          if ($hash -ne $env:YQ_SHA256) {
            Write-Error "SHA256 mismatch for yq: expected $env:YQ_SHA256, got $hash"
            exit 1
          }
          ls -l $env:YQ_DIR
          echo "$((Get-Location).Path)\\$env:YQ_DIR" >> $env:GITHUB_PATH
          $env:Path += ";$((Get-Location).Path)\\$env:YQ_DIR"
          yq --version

      - name: Set environment variables
        env:
          CUDA_VER: ${{ inputs.cuda-version }}
          HOST_PLATFORM: ${{ inputs.host-platform }}
          PY_VER: ${{ matrix.python-version }}
          SHA: ${{ github.sha }}
        run: ./ci/tools/env-vars build

      - name: Dump environment
        run: |
          env

      - name: Install twine
        run: |
          pip install twine

      # To keep the build workflow simple, all matrix jobs will build a wheel for later use within this workflow.
      - name: Build and check cuda.pathfinder wheel
        if: ${{ inputs.build-pathfinder }}
        run: |
          pushd cuda_pathfinder
          pip wheel -v --no-deps .
          popd

      - name: Download reusable cuda.pathfinder wheel
        if: ${{ !inputs.build-pathfinder }}
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
        with:
          name: cuda-pathfinder-wheel
          path: cuda_pathfinder
          github-token: ${{ github.token }}
          run-id: ${{ inputs.baseline-run-id }}

      - name: List the cuda.pathfinder artifacts directory
        run: |
          if [[ "${{ inputs.host-platform }}" == win* ]]; then
            export CHOWN=chown
          else
            export CHOWN="sudo chown"
          fi
          $CHOWN -R $(whoami) cuda_pathfinder/*.whl
          ls -lahR cuda_pathfinder

      # We only need/want a single pure python wheel, pick linux-64 index 0.
      # This is what we will use for testing & releasing.
      - name: Check cuda.pathfinder wheel
        if: ${{ inputs.build-pathfinder && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }}
        run: |
          twine check --strict cuda_pathfinder/*.whl

      - name: Constrain builds to the local cuda.pathfinder wheel
        if: ${{ inputs.build-bindings }}
        run: |
          pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl)
          test "${#pathfinder_wheels[@]}" -eq 1
          test -f "${pathfinder_wheels[0]}"
          mkdir -p wheel-constraints
          if [[ "${{ inputs.host-platform }}" == win* ]]; then
            pathfinder_uri="file:///$(cygpath -am "${pathfinder_wheels[0]}")"
          else
            pathfinder_uri="file:///host$(realpath "${pathfinder_wheels[0]}")"
          fi
          printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}" | tee wheel-constraints/cuda-bindings.txt

      - name: Upload cuda.pathfinder build artifacts
        if: ${{ strategy.job-index == 0 && inputs.host-platform == 'linux-64' }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: cuda-pathfinder-wheel
          path: cuda_pathfinder/*.whl
          if-no-files-found: error

      - name: Set up mini CTK
        if: ${{ inputs.build-bindings || inputs.build-core || inputs.test-bindings || inputs.test-core }}
        uses: ./.github/actions/fetch_ctk
        continue-on-error: false
        with:
          host-platform: ${{ inputs.host-platform }}
          cuda-version: ${{ inputs.cuda-version }}
          cuda-channel: ${{ inputs.cuda-channel }}

      - name: Build cuda.bindings wheel
        if: ${{ inputs.build-bindings }}
        uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c  # v4.1.1
        with:
          package-dir: ./cuda_bindings/
          output-dir: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
        env:
          CIBW_BUILD: ${{ env.CIBW_BUILD }}
          CIBW_ARCHS_WINDOWS: ${{ (inputs.host-platform == 'win-arm64' && 'ARM64') || 'AMD64' }}
          CIBW_BEFORE_BUILD_LINUX: 'python -m pip install --upgrade "pip>=25.3"'
          CIBW_BEFORE_BUILD_WINDOWS: 'python -m pip install --upgrade "pip>=25.3" delvewheel'
          # TODO: remove cpython-prerelease once 3.15 is officially supported
          # Allow CPython pre-release builds (currently 3.15 / 3.15t). This is a
          # no-op for stable Python versions because CIBW_BUILD still filters
          # the target version.
          CIBW_ENABLE: cpython-prerelease
          # CIBW mounts the host filesystem under /host
          CIBW_ENVIRONMENT_LINUX: >
            CUDA_PATH=/host/${{ env.CUDA_PATH }}
            CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
            PIP_BUILD_CONSTRAINT=/host/${{ github.workspace }}/wheel-constraints/cuda-bindings.txt
            PIP_CONSTRAINT=/host/${{ github.workspace }}/wheel-constraints/cuda-bindings.txt
            CC="/host/${{ env.SCCACHE_PATH }} cc"
            CXX="/host/${{ env.SCCACHE_PATH }} c++"
            SCCACHE_GHA_ENABLED=true
            ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
            ACTIONS_RUNTIME_URL=${{ env.ACTIONS_RUNTIME_URL }}
            ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
            ACTIONS_CACHE_URL=${{ env.ACTIONS_CACHE_URL }}
            ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}
            SCCACHE_DIR=/host/${{ env.SCCACHE_DIR }}
            SCCACHE_CACHE_SIZE=${{ env.SCCACHE_CACHE_SIZE }}
          CIBW_ENVIRONMENT_WINDOWS: >
            CUDA_PATH="$(cygpath -w ${{ env.CUDA_PATH }})"
            CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
            PIP_BUILD_CONSTRAINT="$(cygpath -w ./wheel-constraints/cuda-bindings.txt)"
            PIP_CONSTRAINT="$(cygpath -w ./wheel-constraints/cuda-bindings.txt)"
          # check cache stats before leaving cibuildwheel
          CIBW_BEFORE_TEST_LINUX: >
            "/host/${{ env.SCCACHE_PATH }}" --show-adv-stats &&
            "/host/${{ env.SCCACHE_PATH }}" --show-stats --stats-format=json > /host/${{ github.workspace }}/sccache_bindings.json
          # force the test stage to be run (so that before-test is not skipped)
          # TODO: we might want to think twice on adding this, it does a lot of
          # things before reaching this command.
          CIBW_TEST_COMMAND: >
            echo "ok!"

      - name: Report sccache stats (cuda.bindings)
        if: ${{ inputs.build-bindings && !startsWith(inputs.host-platform, 'win') }}
        uses: ./.github/actions/sccache-summary
        with:
          json-file: sccache_bindings.json
          label: "cuda.bindings"
          build-step: "Build cuda.bindings wheel"

      - name: Download reusable cuda.bindings wheel
        if: ${{ !inputs.build-bindings }}
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
        with:
          name: ${{ env.CUDA_BINDINGS_ARTIFACT_BASENAME }}-${{ inputs.baseline-sha }}
          path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
          github-token: ${{ github.token }}
          run-id: ${{ inputs.baseline-run-id }}

      - name: List the cuda.bindings artifacts directory
        run: |
          if [[ "${{ inputs.host-platform }}" == win* ]]; then
            export CHOWN=chown
          else
            export CHOWN="sudo chown"
          fi
          $CHOWN -R $(whoami) ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}
          ls -lahR ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}

      - name: Check cuda.bindings wheel
        if: ${{ inputs.build-bindings }}
        run: |
          twine check --strict ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl

      - name: Constrain cuda.core to the local cuda.bindings wheel
        if: ${{ inputs.build-core }}
        run: |
          pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl)
          bindings_wheels=("${CUDA_BINDINGS_ARTIFACTS_DIR}"/cuda_bindings-"${BUILD_CUDA_MAJOR}".*.whl)
          test "${#pathfinder_wheels[@]}" -eq 1
          test "${#bindings_wheels[@]}" -eq 1
          test -f "${pathfinder_wheels[0]}"
          test -f "${bindings_wheels[0]}"
          mkdir -p wheel-constraints
          if [[ "${{ inputs.host-platform }}" == win* ]]; then
            pathfinder_uri="file:///$(cygpath -am "${pathfinder_wheels[0]}")"
            bindings_uri="file:///$(cygpath -am "${bindings_wheels[0]}")"
          else
            pathfinder_uri="file:///host$(realpath "${pathfinder_wheels[0]}")"
            bindings_uri="file:///host$(realpath "${bindings_wheels[0]}")"
          fi
          {
            printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}"
            printf 'cuda-bindings @ %s\n' "${bindings_uri}"
          } | tee wheel-constraints/cuda-core.txt

      - name: Upload cuda.bindings build artifacts
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}
          path: ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl
          if-no-files-found: error

      - name: Build cuda.core wheel
        if: ${{ inputs.build-core }}
        uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c  # v4.1.1
        with:
          package-dir: ./cuda_core/
          output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
        env:
          CIBW_BUILD: ${{ env.CIBW_BUILD }}
          CIBW_ARCHS_WINDOWS: ${{ (inputs.host-platform == 'win-arm64' && 'ARM64') || 'AMD64' }}
          CIBW_BEFORE_BUILD_LINUX: 'python -m pip install --upgrade "pip>=25.3"'
          CIBW_BEFORE_BUILD_WINDOWS: 'python -m pip install --upgrade "pip>=25.3" delvewheel'
          # TODO: remove cpython-prerelease once 3.15 is officially supported
          # Allow CPython pre-release builds (currently 3.15 / 3.15t). This is a
          # no-op for stable Python versions because CIBW_BUILD still filters
          # the target version.
          CIBW_ENABLE: cpython-prerelease
          # CIBW mounts the host filesystem under /host
          CIBW_ENVIRONMENT_LINUX: >
            CUDA_PATH=/host/${{ env.CUDA_PATH }}
            CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
            CUDA_CORE_BUILD_MAJOR=${{ env.BUILD_CUDA_MAJOR }}
            PIP_BUILD_CONSTRAINT=/host/${{ github.workspace }}/wheel-constraints/cuda-core.txt
            PIP_CONSTRAINT=/host/${{ github.workspace }}/wheel-constraints/cuda-core.txt
            CC="/host/${{ env.SCCACHE_PATH }} cc"
            CXX="/host/${{ env.SCCACHE_PATH }} c++"
            SCCACHE_GHA_ENABLED=true
            ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
            ACTIONS_RUNTIME_URL=${{ env.ACTIONS_RUNTIME_URL }}
            ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
            ACTIONS_CACHE_URL=${{ env.ACTIONS_CACHE_URL }}
            ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}
            SCCACHE_DIR=/host/${{ env.SCCACHE_DIR }}
            SCCACHE_CACHE_SIZE=${{ env.SCCACHE_CACHE_SIZE }}
          CIBW_ENVIRONMENT_WINDOWS: >
            CUDA_PATH="$(cygpath -w ${{ env.CUDA_PATH }})"
            CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
            CUDA_CORE_BUILD_MAJOR=${{ env.BUILD_CUDA_MAJOR }}
            PIP_BUILD_CONSTRAINT="$(cygpath -w ./wheel-constraints/cuda-core.txt)"
            PIP_CONSTRAINT="$(cygpath -w ./wheel-constraints/cuda-core.txt)"
          # check cache stats before leaving cibuildwheel
          CIBW_BEFORE_TEST_LINUX: >
            "/host${{ env.SCCACHE_PATH }}" --show-adv-stats &&
            "/host${{ env.SCCACHE_PATH }}" --show-stats --stats-format=json > /host/${{ github.workspace }}/sccache_core.json
          # force the test stage to be run (so that before-test is not skipped)
          # TODO: we might want to think twice on adding this, it does a lot of
          # things before reaching this command.
          CIBW_TEST_COMMAND: >
            echo "ok!"

      - name: Report sccache stats (cuda.core)
        if: ${{ inputs.build-core && !startsWith(inputs.host-platform, 'win') }}
        uses: ./.github/actions/sccache-summary
        with:
          json-file: sccache_core.json
          label: "cuda.core"
          build-step: "Build cuda.core wheel"

      - name: List the cuda.core artifacts directory and rename
        if: ${{ inputs.build-core }}
        run: |
          if [[ "${{ inputs.host-platform }}" == win* ]]; then
            export CHOWN=chown
          else
            export CHOWN="sudo chown"
          fi
          $CHOWN -R $(whoami) ${{ env.CUDA_CORE_ARTIFACTS_DIR }}

          # Rename wheel to include CUDA version suffix
          mkdir -p "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_CUDA_MAJOR}"
          for wheel in ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl; do
            if [[ -f "${wheel}" ]]; then
              base_name=$(basename "${wheel}" .whl)
              new_name="${base_name}.cu${BUILD_CUDA_MAJOR}.whl"
              mv "${wheel}" "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_CUDA_MAJOR}/${new_name}"
              echo "Renamed wheel to: ${new_name}"
            fi
          done

          ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }}

      - name: Finalize single-major cuda.core wheel
        if: ${{ inputs.single-cuda-major && inputs.build-core }}
        run: |
          for wheel in "${{ env.CUDA_CORE_ARTIFACTS_DIR }}"/cu"${BUILD_CUDA_MAJOR}"/*.cu"${BUILD_CUDA_MAJOR}".whl; do
            base_name=$(basename "${wheel}" ".cu${BUILD_CUDA_MAJOR}.whl")
            mv "${wheel}" "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/${base_name}.whl"
          done
          ls -lahR "${{ env.CUDA_CORE_ARTIFACTS_DIR }}"

      - name: Download reusable cuda.core wheel
        if: ${{ !inputs.build-core }}
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
        with:
          name: ${{ env.CUDA_CORE_ARTIFACT_BASENAME }}-${{ inputs.baseline-sha }}
          path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
          github-token: ${{ github.token }}
          run-id: ${{ inputs.baseline-run-id }}

      # We only need/want a single pure python wheel, pick linux-64 index 0.
      - name: Build and check cuda-python wheel
        if: ${{ inputs.build-python && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }}
        run: |
          pushd cuda_python
          pip wheel -v --no-deps .
          twine check --strict *.whl
          popd

      - name: Download reusable cuda-python wheel
        if: ${{ !inputs.build-python && strategy.job-index == 0 && inputs.host-platform == 'linux-64' }}
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c  # v8.0.1
        with:
          name: cuda-python-wheel
          path: cuda_python
          github-token: ${{ github.token }}
          run-id: ${{ inputs.baseline-run-id }}

      - name: List the cuda-python artifacts directory
        if: ${{ strategy.job-index == 0 && inputs.host-platform == 'linux-64' }}
        run: |
          if [[ "${{ inputs.host-platform }}" == win* ]]; then
            export CHOWN=chown
          else
            export CHOWN="sudo chown"
          fi
          $CHOWN -R $(whoami) cuda_python/*.whl
          ls -lahR cuda_python

      - name: Upload cuda-python build artifacts
        if: ${{ strategy.job-index == 0 && inputs.host-platform == 'linux-64' }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: cuda-python-wheel
          path: cuda_python/*.whl
          if-no-files-found: error

      - name: Set up Python
        id: setup-python2
        if: ${{ env.BUILD_TEST_ARTIFACTS == 'true' && (inputs.test-bindings || inputs.test-core) }}
        uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1  # v6.3.0
        with:
          python-version: ${{ matrix.python-version }}
          architecture: ${{ ((inputs.host-platform == 'linux-aarch64' || inputs.host-platform == 'win-arm64') && 'arm64') || 'x64' }}
          # TODO: remove allow-prereleases once 3.15 is officially supported
          allow-prereleases: ${{ startsWith(matrix.python-version, '3.15') }}

      - name: Enable Scientific Python Nightly Wheels for Python 3.15
        if: ${{ env.BUILD_TEST_ARTIFACTS == 'true' && (inputs.test-bindings || inputs.test-core) && startsWith(matrix.python-version, '3.15') }}
        run: |
          echo "PIP_EXTRA_INDEX_URL=https://pypi.anaconda.org/scientific-python-nightly-wheels/simple" >> "$GITHUB_ENV"
          echo "PIP_ONLY_BINARY=numpy" >> "$GITHUB_ENV"

      - name: verify free-threaded build
        if: ${{ env.BUILD_TEST_ARTIFACTS == 'true' && (inputs.test-bindings || inputs.test-core) && endsWith(matrix.python-version, 't') }}
        run: python -c 'import sys; assert not sys._is_gil_enabled()'

      - name: Set up Python include paths
        if: ${{ env.BUILD_TEST_ARTIFACTS == 'true' && (inputs.test-bindings || inputs.test-core) }}
        run: |
          if [[ "${{ inputs.host-platform }}" == linux* ]]; then
            echo "CPLUS_INCLUDE_PATH=${Python3_ROOT_DIR}/include/python${{ matrix.python-version }}" >> $GITHUB_ENV
          elif [[ "${{ inputs.host-platform }}" == win* ]]; then
            echo "CL=/I\"${Python3_ROOT_DIR}\include\python${{ matrix.python-version }}\"" >> $GITHUB_ENV
          fi
          # For caching
          echo "PY_EXT_SUFFIX=$(python -c "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")" >> $GITHUB_ENV

      - name: Install cuda.pathfinder (required for next step)
        if: ${{ env.BUILD_TEST_ARTIFACTS == 'true' && (inputs.test-bindings || inputs.test-core) }}
        run: |
          pip install cuda_pathfinder/*.whl

      - name: Hide GNU link.exe so Meson finds MSVC link.exe
        if: ${{ env.BUILD_TEST_ARTIFACTS == 'true' && startsWith(inputs.host-platform, 'win') && (inputs.test-bindings || inputs.test-core) }}
        run: |
          if [ -f "/c/Program Files/Git/usr/bin/link.exe" ]; then
            mv "/c/Program Files/Git/usr/bin/link.exe" "/c/Program Files/Git/usr/bin/link.exe.bak"
          fi

      - name: Build cuda.bindings Cython tests
        if: ${{ env.BUILD_TEST_ARTIFACTS == 'true' && inputs.test-bindings }}
        run: |
          pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl --group ./cuda_bindings/pyproject.toml:test
          pushd ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }}
          bash build_tests.sh
          popd

      - name: Upload cuda.bindings Cython tests
        if: ${{ env.BUILD_TEST_ARTIFACTS == 'true' && inputs.test-bindings }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: ${{ env.CUDA_BINDINGS_ARTIFACT_NAME }}-tests
          path: ${{ env.CUDA_BINDINGS_CYTHON_TESTS_DIR }}/test_*${{ env.PY_EXT_SUFFIX }}
          if-no-files-found: error

      - name: Build cuda.core Cython tests
        if: ${{ env.BUILD_TEST_ARTIFACTS == 'true' && inputs.test-core }}
        run: |
          pip install ${{ env.CUDA_BINDINGS_ARTIFACTS_DIR }}/*.whl
          if ${{ inputs.build-core && !inputs.single-cuda-major }}; then
            core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_CUDA_MAJOR}" -maxdepth 1 -type f -name '*.whl' -print -quit)
          else
            core_wheel=$(find "${{ env.CUDA_CORE_ARTIFACTS_DIR }}" -maxdepth 1 -type f -name '*.whl' -print -quit)
          fi
          if [[ -z "${core_wheel}" ]]; then
            echo "No cuda.core wheel found" >&2
            exit 1
          fi
          pip install "${core_wheel}" --group ./cuda_core/pyproject.toml:test
          pushd ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }}
          bash build_tests.sh
          popd

      - name: Upload cuda.core Cython tests
        if: ${{ env.BUILD_TEST_ARTIFACTS == 'true' && inputs.test-core }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-tests
          path: ${{ env.CUDA_CORE_CYTHON_TESTS_DIR }}/test_*${{ env.PY_EXT_SUFFIX }}
          if-no-files-found: error

      # Note: This overwrites CUDA_PATH etc
      - name: Set up mini CTK
        if: ${{ !inputs.single-cuda-major && (inputs.build-core || inputs.test-core) }}
        uses: ./.github/actions/fetch_ctk
        continue-on-error: false
        with:
          host-platform: ${{ inputs.host-platform }}
          cuda-version: ${{ inputs.prev-cuda-version }}
          cuda-path: "./cuda_toolkit_prev"

      - name: Build cuda.core test binaries
        if: ${{ env.BUILD_TEST_ARTIFACTS == 'true' && inputs.test-core }}
        run: |
          nvcc --version
          python "${{ env.CUDA_CORE_TEST_BINARIES_DIR }}/build_test_binaries.py"

      - name: Upload cuda.core test binaries
        if: ${{ env.BUILD_TEST_ARTIFACTS == 'true' && inputs.test-core }}
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}-test-binaries
          path: |
            ${{ env.CUDA_CORE_TEST_BINARIES_DIR }}/*.o
            ${{ env.CUDA_CORE_TEST_BINARIES_DIR }}/*.a
            ${{ env.CUDA_CORE_TEST_BINARIES_DIR }}/*.lib
          if-no-files-found: error

      - name: Download cuda.bindings build artifacts from the prior branch
        if: ${{ !inputs.single-cuda-major && inputs.build-core }}
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          if ! (command -v gh 2>&1 >/dev/null); then
            # See https://github.com/cli/cli/blob/trunk/docs/install_linux.md#debian-ubuntu-linux-raspberry-pi-os-apt.
            # gh is needed for artifact fetching.
            mkdir -p -m 755 /etc/apt/keyrings \
                  && out=$(mktemp) && wget -nv -O$out https://cli.github.com/packages/githubcli-archive-keyring.gpg \
                  && cat $out | tee /etc/apt/keyrings/githubcli-archive-keyring.gpg > /dev/null \
            && chmod go+r /etc/apt/keyrings/githubcli-archive-keyring.gpg \
            && echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list > /dev/null \
            && apt update \
            && apt install gh -y
          fi

          OLD_BRANCH=$(yq '.backport_branch' ci/versions.yml)
          OLD_BASENAME="cuda-bindings-python${PYTHON_VERSION_FORMATTED}-cuda*-${{ inputs.host-platform }}*"
          LATEST_PRIOR_RUN_ID=$(./ci/tools/lookup-run-id --branch "${OLD_BRANCH}" NVIDIA/cuda-python "CI")
          PREV_BINDINGS_DIR="cuda_bindings/dist-prev"

          gh run download $LATEST_PRIOR_RUN_ID -p ${OLD_BASENAME} -R NVIDIA/cuda-python
          rm -rf ${OLD_BASENAME}-tests  # exclude cython test artifacts
          ls -al $OLD_BASENAME
          mkdir -p "${PREV_BINDINGS_DIR}"
          mv $OLD_BASENAME/*.whl "${PREV_BINDINGS_DIR}"
          rmdir $OLD_BASENAME

      - name: Constrain previous cuda.core to the downloaded cuda.bindings wheel
        if: ${{ !inputs.single-cuda-major && inputs.build-core }}
        run: |
          pathfinder_wheels=(cuda_pathfinder/cuda_pathfinder-*.whl)
          bindings_wheels=(cuda_bindings/dist-prev/cuda_bindings-"${BUILD_PREV_CUDA_MAJOR}".*.whl)
          test "${#pathfinder_wheels[@]}" -eq 1
          test "${#bindings_wheels[@]}" -eq 1
          test -f "${pathfinder_wheels[0]}"
          test -f "${bindings_wheels[0]}"
          mkdir -p wheel-constraints
          if [[ "${{ inputs.host-platform }}" == win* ]]; then
            pathfinder_uri="file:///$(cygpath -am "${pathfinder_wheels[0]}")"
            bindings_uri="file:///$(cygpath -am "${bindings_wheels[0]}")"
          else
            pathfinder_uri="file:///host$(realpath "${pathfinder_wheels[0]}")"
            bindings_uri="file:///host$(realpath "${bindings_wheels[0]}")"
          fi
          {
            printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}"
            printf 'cuda-bindings @ %s\n' "${bindings_uri}"
          } | tee wheel-constraints/cuda-core-prev.txt

      - name: Build cuda.core wheel
        if: ${{ !inputs.single-cuda-major && inputs.build-core }}
        uses: pypa/cibuildwheel@4726cd35bb13f7bde50cf2761f2499ac7b3aa32c  # v4.1.1
        with:
          package-dir: ./cuda_core/
          output-dir: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
        env:
          CIBW_BUILD: ${{ env.CIBW_BUILD }}
          CIBW_ARCHS_WINDOWS: ${{ (inputs.host-platform == 'win-arm64' && 'ARM64') || 'AMD64' }}
          CIBW_BEFORE_BUILD_LINUX: 'python -m pip install --upgrade "pip>=25.3"'
          CIBW_BEFORE_BUILD_WINDOWS: 'python -m pip install --upgrade "pip>=25.3" delvewheel'
          # TODO: remove cpython-prerelease once 3.15 is officially supported
          # Allow CPython pre-release builds (currently 3.15 / 3.15t). This is a
          # no-op for stable Python versions because CIBW_BUILD still filters
          # the target version.
          CIBW_ENABLE: cpython-prerelease
          # CIBW mounts the host filesystem under /host
          CIBW_ENVIRONMENT_LINUX: >
            CUDA_PATH=/host/${{ env.CUDA_PATH }}
            CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
            CUDA_CORE_BUILD_MAJOR=${{ env.BUILD_PREV_CUDA_MAJOR }}
            PIP_BUILD_CONSTRAINT=/host/${{ github.workspace }}/wheel-constraints/cuda-core-prev.txt
            PIP_CONSTRAINT=/host/${{ github.workspace }}/wheel-constraints/cuda-core-prev.txt
            CC="/host/${{ env.SCCACHE_PATH }} cc"
            CXX="/host/${{ env.SCCACHE_PATH }} c++"
            SCCACHE_GHA_ENABLED=true
            ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
            ACTIONS_RUNTIME_URL=${{ env.ACTIONS_RUNTIME_URL }}
            ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
            ACTIONS_CACHE_URL=${{ env.ACTIONS_CACHE_URL }}
            ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}
            SCCACHE_DIR=/host/${{ env.SCCACHE_DIR }}
            SCCACHE_CACHE_SIZE=${{ env.SCCACHE_CACHE_SIZE }}
          CIBW_ENVIRONMENT_WINDOWS: >
            CUDA_PATH="$(cygpath -w ${{ env.CUDA_PATH }})"
            CUDA_PYTHON_PARALLEL_LEVEL=${{ env.CUDA_PYTHON_PARALLEL_LEVEL }}
            CUDA_CORE_BUILD_MAJOR=${{ env.BUILD_PREV_CUDA_MAJOR }}
            PIP_BUILD_CONSTRAINT="$(cygpath -w ./wheel-constraints/cuda-core-prev.txt)"
            PIP_CONSTRAINT="$(cygpath -w ./wheel-constraints/cuda-core-prev.txt)"
          # check cache stats before leaving cibuildwheel
          CIBW_BEFORE_TEST_LINUX: >
            "/host${{ env.SCCACHE_PATH }}" --show-adv-stats &&
            "/host${{ env.SCCACHE_PATH }}" --show-stats --stats-format=json > /host/${{ github.workspace }}/sccache_core_prev.json
          # force the test stage to be run (so that before-test is not skipped)
          # TODO: we might want to think twice on adding this, it does a lot of
          # things before reaching this command.
          CIBW_TEST_COMMAND: >
            echo "ok!"

      - name: Report sccache stats (cuda.core prev)
        if: ${{ !inputs.single-cuda-major && inputs.build-core && !startsWith(inputs.host-platform, 'win') }}
        uses: ./.github/actions/sccache-summary
        with:
          json-file: sccache_core_prev.json
          label: "cuda.core (prev CTK)"
          build-step: "Build cuda.core wheel"

      - name: List the cuda.core artifacts directory and rename
        if: ${{ !inputs.single-cuda-major && inputs.build-core }}
        run: |
          if [[ "${{ inputs.host-platform }}" == win* ]]; then
            export CHOWN=chown
          else
            export CHOWN="sudo chown"
          fi
          $CHOWN -R $(whoami) ${{ env.CUDA_CORE_ARTIFACTS_DIR }}
          ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }}

          # Rename wheel to include CUDA version suffix
          mkdir -p "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_PREV_CUDA_MAJOR}"
          for wheel in ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl; do
            if [[ -f "${wheel}" ]]; then
              base_name=$(basename "${wheel}" .whl)
              new_name="${base_name}.cu${BUILD_PREV_CUDA_MAJOR}.whl"
              mv "${wheel}" "${{ env.CUDA_CORE_ARTIFACTS_DIR }}/cu${BUILD_PREV_CUDA_MAJOR}/${new_name}"
              echo "Renamed wheel to: ${new_name}"
            fi
          done

          ls -lahR ${{ env.CUDA_CORE_ARTIFACTS_DIR }}

      - name: Merge cuda.core wheels
        if: ${{ !inputs.single-cuda-major && inputs.build-core }}
        run: |
          pip install wheel
          python ci/tools/merge_cuda_core_wheels.py \
            "${{ env.CUDA_CORE_ARTIFACTS_DIR }}"/cu"${BUILD_CUDA_MAJOR}"/cuda_core*.whl \
            "${{ env.CUDA_CORE_ARTIFACTS_DIR }}"/cu"${BUILD_PREV_CUDA_MAJOR}"/cuda_core*.whl \
            --output-dir "${{ env.CUDA_CORE_ARTIFACTS_DIR }}"

      - name: Check cuda.core wheel
        if: ${{ inputs.build-core }}
        run: |
          twine check --strict ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl

      - name: Upload cuda.core build artifacts
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: ${{ env.CUDA_CORE_ARTIFACT_NAME }}
          path: ${{ env.CUDA_CORE_ARTIFACTS_DIR }}/*.whl
          if-no-files-found: error