diff --git a/.github/actions/dist-build-memory/action.yml b/.github/actions/dist-build-memory/action.yml new file mode 100644 index 000000000..67f4f518b --- /dev/null +++ b/.github/actions/dist-build-memory/action.yml @@ -0,0 +1,48 @@ +name: Memory for a Linux dist build +description: >- + A Linux build with the dist cargo profile compiles its largest single-unit crates at the same + time and can use more memory than the hosted runner has; the runner is then lost without a build + error. `start` adds host swap (which a manylinux container shares), limits cargo to three crates + at a time for the rest of the job, and samples used memory; `report` stops sampling and prints + the peak. +inputs: + step: + description: "`start` before the dist build, `report` after it (run it with always())" + required: true +runs: + using: composite + steps: + - name: Add swap and limit cargo jobs + if: inputs.step == 'start' + shell: bash + run: | + free -m + avail="$(df -BG --output=avail /mnt 2>/dev/null | tail -n 1 | tr -dc '0-9')" + swap_dir=/mnt + if [[ -z "${avail}" || "${avail}" -lt 24 ]]; then + swap_dir="" + fi + swapfile="${swap_dir}/sqlbuild-dist.swap" + sudo fallocate -l 16G "${swapfile}" + sudo chmod 600 "${swapfile}" + sudo mkswap "${swapfile}" > /dev/null + sudo swapon "${swapfile}" + free -m + # maturin-action passes CARGO_* variables into its manylinux container. + echo "CARGO_BUILD_JOBS=3" >> "${GITHUB_ENV}" + ( while :; do + free -m | awk '/^Mem:/ { ram = $3 } /^Swap:/ { swap = $3 } END { print ram, swap }' + sleep 5 + done ) > "${RUNNER_TEMP}/dist-memory.log" 2>&1 & + echo $! > "${RUNNER_TEMP}/dist-memory.pid" + - name: Report memory use + if: inputs.step == 'report' + shell: bash + run: | + # Stop sampling, so later steps such as timing measurements run without it. + kill "$(cat "${RUNNER_TEMP}/dist-memory.pid")" 2>/dev/null || true + free -m + grep -E '^pswp(in|out) ' /proc/vmstat + awk '{ if ($1 > ram) ram = $1; if ($2 > swap) swap = $2; if ($1 + $2 > both) both = $1 + $2 } + END { printf "Peak used memory: RAM %d MiB, swap %d MiB, together %d MiB\n", ram, swap, both }' \ + "${RUNNER_TEMP}/dist-memory.log" | tee -a "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/build-distributions.yml b/.github/workflows/build-distributions.yml index e8ae0b969..08b7f036c 100644 --- a/.github/workflows/build-distributions.yml +++ b/.github/workflows/build-distributions.yml @@ -141,29 +141,12 @@ jobs: run: | npm ci npm run build - # In a Linux dist build the largest single-unit crates compile at the same time and can use - # more memory than the hosted runner has; the runner is then lost without a build error. - # Swap on the host, which the manylinux container shares, absorbs that peak, the builds below - # compile at most three crates at a time, and the sampled memory use is reported afterwards. - - name: Add swap for the Linux dist build + # A Linux dist build can need more memory than the hosted runner has; see the action. + - name: Add swap and limit cargo jobs for the Linux dist build if: inputs.prebuilt-run-id == '' && runner.os == 'Linux' && inputs.cargo-profile == 'dist' - run: | - free -m - avail="$(df -BG --output=avail /mnt 2>/dev/null | tail -n 1 | tr -dc '0-9')" - swap_dir=/mnt - if [[ -z "${avail}" || "${avail}" -lt 24 ]]; then - swap_dir="" - fi - swapfile="${swap_dir}/sqlbuild-dist.swap" - sudo fallocate -l 16G "${swapfile}" - sudo chmod 600 "${swapfile}" - sudo mkswap "${swapfile}" > /dev/null - sudo swapon "${swapfile}" - free -m - ( while :; do - free -m | awk '/^Mem:/ { ram = $3 } /^Swap:/ { swap = $3 } END { print ram, swap }' - sleep 5 - done ) > "${RUNNER_TEMP}/dist-memory.log" 2>&1 & + uses: ./.github/actions/dist-build-memory + with: + step: start # maturin-action downloads maturin from GitHub releases, which can fail on its own. Each # build retries once; macOS and Windows then fall back to the same maturin from PyPI. Linux # builds need the action's manylinux container, so they have no fallback. @@ -177,7 +160,6 @@ jobs: args: >- --profile ${{ inputs.cargo-profile }} --out dist --features extension-module ${{ matrix.platform.name == 'linux-x86_64' && '--timings' || '' }} - ${{ runner.os == 'Linux' && inputs.cargo-profile == 'dist' && '--jobs 3' || '' }} target: ${{ matrix.platform.target }} manylinux: auto - id: extension-retry @@ -191,7 +173,6 @@ jobs: args: >- --profile ${{ inputs.cargo-profile }} --out dist --features extension-module ${{ matrix.platform.name == 'linux-x86_64' && '--timings' || '' }} - ${{ runner.os == 'Linux' && inputs.cargo-profile == 'dist' && '--jobs 3' || '' }} target: ${{ matrix.platform.target }} manylinux: auto - name: Build the extension with maturin from PyPI @@ -222,7 +203,6 @@ jobs: args: >- --profile ${{ inputs.cargo-profile }} --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml --features ui ${{ matrix.platform.name == 'linux-x86_64' && '--timings' || '' }} - ${{ runner.os == 'Linux' && inputs.cargo-profile == 'dist' && '--jobs 3' || '' }} target: ${{ matrix.platform.target }} manylinux: auto - id: cli-retry @@ -240,7 +220,6 @@ jobs: args: >- --profile ${{ inputs.cargo-profile }} --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml --features ui ${{ matrix.platform.name == 'linux-x86_64' && '--timings' || '' }} - ${{ runner.os == 'Linux' && inputs.cargo-profile == 'dist' && '--jobs 3' || '' }} target: ${{ matrix.platform.target }} manylinux: auto - name: Build sqb with maturin from PyPI @@ -261,12 +240,9 @@ jobs: if: >- always() && inputs.prebuilt-run-id == '' && runner.os == 'Linux' && inputs.cargo-profile == 'dist' - run: | - free -m - grep -E '^pswp(in|out) ' /proc/vmstat - awk '{ if ($1 > ram) ram = $1; if ($2 > swap) swap = $2; if ($1 + $2 > both) both = $1 + $2 } - END { printf "Peak used memory: RAM %d MiB, swap %d MiB, together %d MiB\n", ram, swap, both }' \ - "${RUNNER_TEMP}/dist-memory.log" | tee -a "${GITHUB_STEP_SUMMARY}" + uses: ./.github/actions/dist-build-memory + with: + step: report - name: Upload the cargo build timing reports if: inputs.prebuilt-run-id == '' && matrix.platform.name == 'linux-x86_64' uses: actions/upload-artifact@v4 diff --git a/.github/workflows/release-performance.yml b/.github/workflows/release-performance.yml index cfd805f9d..855c7caa5 100644 --- a/.github/workflows/release-performance.yml +++ b/.github/workflows/release-performance.yml @@ -18,6 +18,7 @@ on: pull_request: paths: - .github/workflows/release-performance.yml + - .github/actions/dist-build-memory/** - scripts/compare_release_performance.py - scripts/release_performance/** - scripts/compile_performance_ratio/_helpers/edit.py @@ -67,12 +68,22 @@ jobs: - name: Install dependencies run: uv sync --locked --all-groups --all-extras # The candidate uses the released wheels' cargo profile, so both sides of the comparison match. + # That build can need more memory than the hosted runner has; see the action. + - name: Add swap and limit cargo jobs for the candidate build + uses: ./.github/actions/dist-build-memory + with: + step: start - name: Build candidate wheel env: MATURIN_PEP517_ARGS: --profile dist run: | uv build --wheel --out-dir "${RUNNER_TEMP}/candidate" uv build --wheel crates/sqlbuild-cli --out-dir "${RUNNER_TEMP}/candidate" + - name: Report memory use of the candidate build + if: always() + uses: ./.github/actions/dist-build-memory + with: + step: report # A release tag created while the candidate built names the baseline's benchmark generator. - name: Fetch release tags run: git fetch --tags --force --quiet origin