From 48193c377ebd5e3eb97c2a9d9c55dd758a139ca4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:16:15 +0000 Subject: [PATCH 1/2] ci(release): share the dist build memory guard with Release Performance Move the Linux dist build's swap, cargo job limit and memory report into a composite action, and use it for the Release Performance candidate wheel, which builds with the same dist profile and lost its runner the same way. The job limit is now CARGO_BUILD_JOBS=3, which maturin-action passes into its manylinux container, instead of a --jobs argument on each build. --- .github/actions/dist-build-memory/action.yml | 48 ++++++++++++++++++++ .github/workflows/build-distributions.yml | 40 ++++------------ .github/workflows/release-performance.yml | 10 ++++ 3 files changed, 66 insertions(+), 32 deletions(-) create mode 100644 .github/actions/dist-build-memory/action.yml diff --git a/.github/actions/dist-build-memory/action.yml b/.github/actions/dist-build-memory/action.yml new file mode 100644 index 000000000..67f4f518b --- /dev/null +++ b/.github/actions/dist-build-memory/action.yml @@ -0,0 +1,48 @@ +name: Memory for a Linux dist build +description: >- + A Linux build with the dist cargo profile compiles its largest single-unit crates at the same + time and can use more memory than the hosted runner has; the runner is then lost without a build + error. `start` adds host swap (which a manylinux container shares), limits cargo to three crates + at a time for the rest of the job, and samples used memory; `report` stops sampling and prints + the peak. +inputs: + step: + description: "`start` before the dist build, `report` after it (run it with always())" + required: true +runs: + using: composite + steps: + - name: Add swap and limit cargo jobs + if: inputs.step == 'start' + shell: bash + run: | + free -m + avail="$(df -BG --output=avail /mnt 2>/dev/null | tail -n 1 | tr -dc '0-9')" + swap_dir=/mnt + if [[ -z "${avail}" || "${avail}" -lt 24 ]]; then + swap_dir="" + fi + swapfile="${swap_dir}/sqlbuild-dist.swap" + sudo fallocate -l 16G "${swapfile}" + sudo chmod 600 "${swapfile}" + sudo mkswap "${swapfile}" > /dev/null + sudo swapon "${swapfile}" + free -m + # maturin-action passes CARGO_* variables into its manylinux container. + echo "CARGO_BUILD_JOBS=3" >> "${GITHUB_ENV}" + ( while :; do + free -m | awk '/^Mem:/ { ram = $3 } /^Swap:/ { swap = $3 } END { print ram, swap }' + sleep 5 + done ) > "${RUNNER_TEMP}/dist-memory.log" 2>&1 & + echo $! > "${RUNNER_TEMP}/dist-memory.pid" + - name: Report memory use + if: inputs.step == 'report' + shell: bash + run: | + # Stop sampling, so later steps such as timing measurements run without it. + kill "$(cat "${RUNNER_TEMP}/dist-memory.pid")" 2>/dev/null || true + free -m + grep -E '^pswp(in|out) ' /proc/vmstat + awk '{ if ($1 > ram) ram = $1; if ($2 > swap) swap = $2; if ($1 + $2 > both) both = $1 + $2 } + END { printf "Peak used memory: RAM %d MiB, swap %d MiB, together %d MiB\n", ram, swap, both }' \ + "${RUNNER_TEMP}/dist-memory.log" | tee -a "${GITHUB_STEP_SUMMARY}" diff --git a/.github/workflows/build-distributions.yml b/.github/workflows/build-distributions.yml index e8ae0b969..08b7f036c 100644 --- a/.github/workflows/build-distributions.yml +++ b/.github/workflows/build-distributions.yml @@ -141,29 +141,12 @@ jobs: run: | npm ci npm run build - # In a Linux dist build the largest single-unit crates compile at the same time and can use - # more memory than the hosted runner has; the runner is then lost without a build error. - # Swap on the host, which the manylinux container shares, absorbs that peak, the builds below - # compile at most three crates at a time, and the sampled memory use is reported afterwards. - - name: Add swap for the Linux dist build + # A Linux dist build can need more memory than the hosted runner has; see the action. + - name: Add swap and limit cargo jobs for the Linux dist build if: inputs.prebuilt-run-id == '' && runner.os == 'Linux' && inputs.cargo-profile == 'dist' - run: | - free -m - avail="$(df -BG --output=avail /mnt 2>/dev/null | tail -n 1 | tr -dc '0-9')" - swap_dir=/mnt - if [[ -z "${avail}" || "${avail}" -lt 24 ]]; then - swap_dir="" - fi - swapfile="${swap_dir}/sqlbuild-dist.swap" - sudo fallocate -l 16G "${swapfile}" - sudo chmod 600 "${swapfile}" - sudo mkswap "${swapfile}" > /dev/null - sudo swapon "${swapfile}" - free -m - ( while :; do - free -m | awk '/^Mem:/ { ram = $3 } /^Swap:/ { swap = $3 } END { print ram, swap }' - sleep 5 - done ) > "${RUNNER_TEMP}/dist-memory.log" 2>&1 & + uses: ./.github/actions/dist-build-memory + with: + step: start # maturin-action downloads maturin from GitHub releases, which can fail on its own. Each # build retries once; macOS and Windows then fall back to the same maturin from PyPI. Linux # builds need the action's manylinux container, so they have no fallback. @@ -177,7 +160,6 @@ jobs: args: >- --profile ${{ inputs.cargo-profile }} --out dist --features extension-module ${{ matrix.platform.name == 'linux-x86_64' && '--timings' || '' }} - ${{ runner.os == 'Linux' && inputs.cargo-profile == 'dist' && '--jobs 3' || '' }} target: ${{ matrix.platform.target }} manylinux: auto - id: extension-retry @@ -191,7 +173,6 @@ jobs: args: >- --profile ${{ inputs.cargo-profile }} --out dist --features extension-module ${{ matrix.platform.name == 'linux-x86_64' && '--timings' || '' }} - ${{ runner.os == 'Linux' && inputs.cargo-profile == 'dist' && '--jobs 3' || '' }} target: ${{ matrix.platform.target }} manylinux: auto - name: Build the extension with maturin from PyPI @@ -222,7 +203,6 @@ jobs: args: >- --profile ${{ inputs.cargo-profile }} --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml --features ui ${{ matrix.platform.name == 'linux-x86_64' && '--timings' || '' }} - ${{ runner.os == 'Linux' && inputs.cargo-profile == 'dist' && '--jobs 3' || '' }} target: ${{ matrix.platform.target }} manylinux: auto - id: cli-retry @@ -240,7 +220,6 @@ jobs: args: >- --profile ${{ inputs.cargo-profile }} --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml --features ui ${{ matrix.platform.name == 'linux-x86_64' && '--timings' || '' }} - ${{ runner.os == 'Linux' && inputs.cargo-profile == 'dist' && '--jobs 3' || '' }} target: ${{ matrix.platform.target }} manylinux: auto - name: Build sqb with maturin from PyPI @@ -261,12 +240,9 @@ jobs: if: >- always() && inputs.prebuilt-run-id == '' && runner.os == 'Linux' && inputs.cargo-profile == 'dist' - run: | - free -m - grep -E '^pswp(in|out) ' /proc/vmstat - awk '{ if ($1 > ram) ram = $1; if ($2 > swap) swap = $2; if ($1 + $2 > both) both = $1 + $2 } - END { printf "Peak used memory: RAM %d MiB, swap %d MiB, together %d MiB\n", ram, swap, both }' \ - "${RUNNER_TEMP}/dist-memory.log" | tee -a "${GITHUB_STEP_SUMMARY}" + uses: ./.github/actions/dist-build-memory + with: + step: report - name: Upload the cargo build timing reports if: inputs.prebuilt-run-id == '' && matrix.platform.name == 'linux-x86_64' uses: actions/upload-artifact@v4 diff --git a/.github/workflows/release-performance.yml b/.github/workflows/release-performance.yml index cfd805f9d..8656d1388 100644 --- a/.github/workflows/release-performance.yml +++ b/.github/workflows/release-performance.yml @@ -67,12 +67,22 @@ jobs: - name: Install dependencies run: uv sync --locked --all-groups --all-extras # The candidate uses the released wheels' cargo profile, so both sides of the comparison match. + # That build can need more memory than the hosted runner has; see the action. + - name: Add swap and limit cargo jobs for the candidate build + uses: ./.github/actions/dist-build-memory + with: + step: start - name: Build candidate wheel env: MATURIN_PEP517_ARGS: --profile dist run: | uv build --wheel --out-dir "${RUNNER_TEMP}/candidate" uv build --wheel crates/sqlbuild-cli --out-dir "${RUNNER_TEMP}/candidate" + - name: Report memory use of the candidate build + if: always() + uses: ./.github/actions/dist-build-memory + with: + step: report # A release tag created while the candidate built names the baseline's benchmark generator. - name: Fetch release tags run: git fetch --tags --force --quiet origin From b352b1835e1bcf6c6d0d7fe618a05205d066d89d Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 10 Nov 2026 21:06:53 +0000 Subject: [PATCH 2/2] ci(release): run Release Performance when the dist build memory action changes --- .github/workflows/release-performance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-performance.yml b/.github/workflows/release-performance.yml index 8656d1388..855c7caa5 100644 --- a/.github/workflows/release-performance.yml +++ b/.github/workflows/release-performance.yml @@ -18,6 +18,7 @@ on: pull_request: paths: - .github/workflows/release-performance.yml + - .github/actions/dist-build-memory/** - scripts/compare_release_performance.py - scripts/release_performance/** - scripts/compile_performance_ratio/_helpers/edit.py