diff --git a/.github/workflows/build-distributions.yml b/.github/workflows/build-distributions.yml new file mode 100644 index 0000000000..b27cad102c --- /dev/null +++ b/.github/workflows/build-distributions.yml @@ -0,0 +1,259 @@ +name: Build Distributions + +# The single owner of the wheel platform matrix, the maturin settings and the distribution smoke +# tests. Verify builds the wheels for pull requests that touch the native build, Release +# Distributions builds and keeps every distribution for a release pull request, and Publish either +# smoke tests and reuses those distributions or builds them here from the release tag. +on: + workflow_call: + inputs: + ref: + description: "Commit, tag or branch to build; empty builds the triggering commit" + required: false + type: string + default: "" + sdist: + description: "Also build the source distributions" + required: false + type: boolean + default: true + upload: + description: "Upload the distributions and a manifest recording their source commit and tree" + required: false + type: boolean + default: false + prebuilt-run-id: + description: >- + Release Distributions run whose distributions are smoke tested and uploaded instead of + building; its manifest must record the tree of ref. The caller grants actions: read. + required: false + type: string + default: "" + retention-days: + description: "Retention of uploaded artifacts in days; 0 uses the repository default" + required: false + type: number + default: 0 + outputs: + sha: + description: "Commit the distributions were built from" + value: ${{ jobs.source.outputs.sha }} + tree: + description: "Tree of the commit the distributions were built from" + value: ${{ jobs.source.outputs.tree }} + +jobs: + source: + name: Resolve source + runs-on: ubuntu-24.04 + outputs: + sha: ${{ steps.source.outputs.sha }} + tree: ${{ steps.source.outputs.tree }} + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ inputs.ref }} + - name: Pin the source commit + id: source + run: | + sha="$(git rev-parse HEAD)" + tree="$(git rev-parse 'HEAD^{tree}')" + echo "sha=${sha}" >> "${GITHUB_OUTPUT}" + echo "tree=${tree}" >> "${GITHUB_OUTPUT}" + echo "Distributions for commit ${sha} with tree ${tree}" >> "${GITHUB_STEP_SUMMARY}" + - name: Download the prebuilt manifest + if: inputs.prebuilt-run-id != '' + uses: actions/download-artifact@v4 + with: + name: release-manifest + path: ${{ runner.temp }}/prebuilt + run-id: ${{ inputs.prebuilt-run-id }} + github-token: ${{ github.token }} + - name: Require prebuilt distributions from this exact tree + if: inputs.prebuilt-run-id != '' + env: + MANIFEST: ${{ runner.temp }}/prebuilt/release-manifest.json + PREBUILT_RUN_ID: ${{ inputs.prebuilt-run-id }} + TREE: ${{ steps.source.outputs.tree }} + run: | + recorded="$(jq -r '.tree // empty' "${MANIFEST}")" + if [[ "${recorded}" != "${TREE}" ]]; then + echo "Run ${PREBUILT_RUN_ID} built tree ${recorded:-}, not ${TREE}" >&2 + exit 1 + fi + echo "Reusing distributions of run ${PREBUILT_RUN_ID}, built from tree ${TREE}" \ + >> "${GITHUB_STEP_SUMMARY}" + + wheels: + name: Build wheel (${{ matrix.platform.name }}) + needs: source + runs-on: ${{ matrix.platform.runner }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + platform: + - name: linux-x86_64 + runner: ubuntu-24.04 + target: x86_64 + - name: linux-aarch64 + runner: ubuntu-24.04-arm + target: aarch64 + - name: macos-x86_64 + runner: macos-15-intel + target: x86_64 + - name: macos-arm64 + runner: macos-15 + target: aarch64 + - name: windows-x86_64 + runner: windows-2025 + target: x86_64 + steps: + - uses: actions/checkout@v4 + if: inputs.prebuilt-run-id == '' + with: + ref: ${{ needs.source.outputs.sha }} + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - uses: dtolnay/rust-toolchain@master + if: inputs.prebuilt-run-id == '' + with: + toolchain: stable + - uses: PyO3/maturin-action@v1 + if: inputs.prebuilt-run-id == '' + with: + command: build + args: --release --out dist --features extension-module + target: ${{ matrix.platform.target }} + manylinux: auto + - uses: PyO3/maturin-action@v1 + if: inputs.prebuilt-run-id == '' + with: + command: build + args: --release --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml + target: ${{ matrix.platform.target }} + manylinux: auto + - name: Download prebuilt wheels + if: inputs.prebuilt-run-id != '' + uses: actions/download-artifact@v4 + with: + name: dist-wheel-${{ matrix.platform.name }} + path: dist + run-id: ${{ inputs.prebuilt-run-id }} + github-token: ${{ github.token }} + - name: Smoke test installed wheels and native console entry points + shell: bash + run: | + python -m venv "${RUNNER_TEMP}/wheel-smoke" + if [[ -x "${RUNNER_TEMP}/wheel-smoke/Scripts/python.exe" ]]; then + smoke_python="${RUNNER_TEMP}/wheel-smoke/Scripts/python.exe" + else + smoke_python="${RUNNER_TEMP}/wheel-smoke/bin/python" + fi + "${smoke_python}" -m pip install --find-links dist dist/*.whl + cd "${RUNNER_TEMP}" + "${smoke_python}" -c "import sqlbuild._native" + smoke_scripts="$(dirname "${smoke_python}")" + "${smoke_scripts}/sqb" --version + "${smoke_scripts}/sqb" --help + "${smoke_scripts}/sqlbuild" --help + - name: Check wheel contents and run a native command + if: matrix.platform.name == 'linux-x86_64' + run: | + wheel="$(ls dist/sqlbuild-[0-9]*.whl)" + if unzip -l "${wheel}" | grep -E '(__pycache__/|\.pyc$|\.tmp$)'; then + echo "Wheel contains generated or temporary script files" + exit 1 + fi + unzip -l "${wheel}" | grep 'sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md' + smoke="${RUNNER_TEMP}/wheel-smoke/bin" + project="${RUNNER_TEMP}/sqlbuild-scope-smoke" + cd "${RUNNER_TEMP}" + "${smoke}/python" -c \ + "import sqlbuild._native as native; assert native.semantic_help_catalogue()" + mkdir -p "${project}/models" + printf 'name = "scope_smoke"\nadapter = "duckdb"\n' > "${project}/sqlbuild_project.toml" + printf 'MODEL();\nSELECT 1 AS id\n' > "${project}/models/orders.sql" + "${smoke}/sqb" --project-dir "${project}" scope model:orders --json + "${smoke}/sqlbuild" --project-dir "${project}" scope model:orders + - uses: actions/upload-artifact@v4 + if: inputs.upload + with: + name: dist-wheel-${{ matrix.platform.name }} + path: dist + if-no-files-found: error + retention-days: ${{ inputs.retention-days }} + + sdist: + name: Build sdist + needs: source + if: inputs.sdist + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + if: inputs.prebuilt-run-id == '' + with: + ref: ${{ needs.source.outputs.sha }} + - uses: dtolnay/rust-toolchain@master + if: inputs.prebuilt-run-id == '' + with: + toolchain: stable + - uses: PyO3/maturin-action@v1 + if: inputs.prebuilt-run-id == '' + with: + command: sdist + args: --out dist + - uses: PyO3/maturin-action@v1 + if: inputs.prebuilt-run-id == '' + with: + command: sdist + args: --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml + - name: Download prebuilt source distributions + if: inputs.prebuilt-run-id != '' + uses: actions/download-artifact@v4 + with: + name: dist-sdist + path: dist + run-id: ${{ inputs.prebuilt-run-id }} + github-token: ${{ github.token }} + - name: Inspect source distribution contents + run: | + for archive in dist/*.tar.gz; do + tar -tzf "$archive" > "${RUNNER_TEMP}/archive-contents.txt" + if grep -E '(__pycache__/|\.pyc$|\.tmp$)' "${RUNNER_TEMP}/archive-contents.txt"; then + echo "Source distribution contains generated or temporary script files" + exit 1 + fi + done + - uses: actions/upload-artifact@v4 + if: inputs.upload + with: + name: dist-sdist + path: dist + if-no-files-found: error + retention-days: ${{ inputs.retention-days }} + + manifest: + name: Record distribution source + needs: [source, wheels, sdist] + if: >- + ${{ !cancelled() && inputs.upload && needs.wheels.result == 'success' + && (needs.sdist.result == 'success' || needs.sdist.result == 'skipped') }} + runs-on: ubuntu-24.04 + steps: + - name: Write the manifest + env: + SHA: ${{ needs.source.outputs.sha }} + TREE: ${{ needs.source.outputs.tree }} + run: | + jq -n --arg sha "${SHA}" --arg tree "${TREE}" --arg run_id "${GITHUB_RUN_ID}" \ + '{sha: $sha, tree: $tree, run_id: $run_id}' \ + > "${RUNNER_TEMP}/release-manifest.json" + cat "${RUNNER_TEMP}/release-manifest.json" + - uses: actions/upload-artifact@v4 + with: + name: release-manifest + path: ${{ runner.temp }}/release-manifest.json + if-no-files-found: error + retention-days: ${{ inputs.retention-days }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 35ebae4c4b..07a45ddac5 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -15,133 +15,92 @@ jobs: release-tag: name: Resolve release tag runs-on: ubuntu-24.04 + permissions: + actions: read + contents: read outputs: tag: ${{ steps.release-tag.outputs.tag }} + tree: ${{ steps.prebuilt.outputs.tree }} + prebuilt-run-id: ${{ steps.prebuilt.outputs.run_id }} steps: - name: Resolve release tag id: release-tag + env: + TAG: ${{ inputs.tag }} run: | - tag="${{ inputs.tag }}" - - case "$tag" in + case "${TAG}" in v0*) ;; *) - echo "Refusing to publish tag '$tag'. PyPI publishing is limited to v0* tags." + echo "Refusing to publish tag '${TAG}'. PyPI publishing is limited to v0* tags." exit 1 ;; esac - echo "tag=$tag" >> "$GITHUB_OUTPUT" + echo "tag=${TAG}" >> "$GITHUB_OUTPUT" - wheels: - # Keep the platform matrix and maturin settings in sync with verify.yml. - name: Build wheel (${{ matrix.platform.name }}) - needs: release-tag - runs-on: ${{ matrix.platform.runner }} - strategy: - fail-fast: false - matrix: - platform: - - name: linux-x86_64 - runner: ubuntu-24.04 - target: x86_64 - - name: linux-aarch64 - runner: ubuntu-24.04-arm - target: aarch64 - - name: macos-x86_64 - runner: macos-15-intel - target: x86_64 - - name: macos-arm64 - runner: macos-15 - target: aarch64 - - name: windows-x86_64 - runner: windows-2025 - target: x86_64 - steps: - uses: actions/checkout@v4 with: - ref: ${{ needs.release-tag.outputs.tag }} - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - - uses: PyO3/maturin-action@v1 - with: - command: build - args: --release --out dist --features extension-module - target: ${{ matrix.platform.target }} - manylinux: auto - - uses: PyO3/maturin-action@v1 - with: - command: build - args: --release --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml - target: ${{ matrix.platform.target }} - manylinux: auto - - name: Smoke test installed wheel and console entry points - if: matrix.platform.name == 'linux-x86_64' + ref: ${{ steps.release-tag.outputs.tag }} + + # Release Distributions builds every distribution on the release pull request. Its run is + # reused only when it succeeded and its commit has exactly the release tag's tree, so the + # reused distributions come from the same sources and the same reviewed build workflow. + - name: Find distributions built from the release tree + id: prebuilt + env: + GH_TOKEN: ${{ github.token }} run: | - if unzip -l dist/sqlbuild-[0-9]*.whl | grep -E '(__pycache__/|\.pyc$|\.tmp$)'; then - echo "Wheel contains generated or temporary script files" - exit 1 + tree="$(git rev-parse 'HEAD^{tree}')" + echo "tree=${tree}" >> "${GITHUB_OUTPUT}" + runs="$(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/workflows/release-distributions.yml/runs?event=workflow_dispatch&status=success&per_page=50" \ + --jq '.workflow_runs[] + | select(.conclusion == "success" and .event == "workflow_dispatch") + | "\(.id) \(.head_sha)"')" || runs="" + run_id="" + while read -r candidate head_sha; do + [[ -n "${candidate}" ]] || continue + candidate_tree="$(gh api "repos/${GITHUB_REPOSITORY}/git/commits/${head_sha}" \ + --jq .tree.sha 2>/dev/null)" || continue + [[ "${candidate_tree}" == "${tree}" ]] || continue + artifacts="$(gh api \ + "repos/${GITHUB_REPOSITORY}/actions/runs/${candidate}/artifacts?per_page=100")" \ + || continue + if ! jq -e '([.artifacts[] | select(.expired)] | length == 0) + and ([.artifacts[].name] | index("release-manifest") and index("dist-sdist"))' \ + <<< "${artifacts}" > /dev/null; then + echo "Run ${candidate} built tree ${tree}, but its artifacts are incomplete or expired" + continue + fi + run_id="${candidate}" + break + done <<< "${runs}" + echo "run_id=${run_id}" >> "${GITHUB_OUTPUT}" + if [[ -n "${run_id}" ]]; then + echo "Publishing distributions prebuilt by run ${run_id} from tree ${tree}" \ + | tee -a "${GITHUB_STEP_SUMMARY}" + else + echo "No prebuilt distributions for tree ${tree}; building from the tag" \ + | tee -a "${GITHUB_STEP_SUMMARY}" fi - unzip -l dist/sqlbuild-[0-9]*.whl | grep 'sqlbuild/.agents/skills/sqlbuild/references/docs/CONTENTS.md' - python -m venv /tmp/sqlbuild-wheel-smoke - /tmp/sqlbuild-wheel-smoke/bin/python -m pip install --find-links dist dist/sqlbuild-[0-9]*.whl - /tmp/sqlbuild-wheel-smoke/bin/python -c "import sqlbuild._native as native; assert native.semantic_help_catalogue()" - /tmp/sqlbuild-wheel-smoke/bin/sqb --version - /tmp/sqlbuild-wheel-smoke/bin/sqb --help - /tmp/sqlbuild-wheel-smoke/bin/sqlbuild --help - mkdir -p /tmp/sqlbuild-scope-smoke/models - printf 'name = "scope_smoke"\nadapter = "duckdb"\n' > /tmp/sqlbuild-scope-smoke/sqlbuild_project.toml - printf 'MODEL();\nSELECT 1 AS id\n' > /tmp/sqlbuild-scope-smoke/models/orders.sql - /tmp/sqlbuild-wheel-smoke/bin/sqb --project-dir /tmp/sqlbuild-scope-smoke scope model:orders --json - /tmp/sqlbuild-wheel-smoke/bin/sqlbuild --project-dir /tmp/sqlbuild-scope-smoke scope model:orders - - uses: actions/upload-artifact@v4 - with: - name: wheel-${{ matrix.platform.name }} - path: dist - sdist: - name: Build sdist + distributions: + name: Distributions needs: release-tag - runs-on: ubuntu-24.04 - steps: - - uses: actions/checkout@v4 - with: - ref: ${{ needs.release-tag.outputs.tag }} - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - - uses: PyO3/maturin-action@v1 - with: - command: sdist - args: --out dist - - uses: PyO3/maturin-action@v1 - with: - command: sdist - args: --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml - - name: Inspect source distribution contents - run: | - for archive in dist/*.tar.gz; do - tar -tzf "$archive" > archive-contents.txt - if grep -E '(__pycache__/|\.pyc$|\.tmp$)' archive-contents.txt; then - echo "Source distribution contains generated or temporary script files" - exit 1 - fi - done - - uses: actions/upload-artifact@v4 - with: - name: sdist - path: dist + permissions: + actions: read + contents: read + uses: ./.github/workflows/build-distributions.yml + with: + ref: ${{ needs.release-tag.outputs.tag }} + upload: true + prebuilt-run-id: ${{ needs.release-tag.outputs.prebuilt-run-id }} pypi: name: Publish to PyPI needs: - release-tag - - wheels - - sdist + - distributions runs-on: ubuntu-24.04 environment: pypi permissions: @@ -150,8 +109,38 @@ jobs: steps: - uses: actions/download-artifact@v4 with: + pattern: dist-* path: dist merge-multiple: true + - name: Check distributions match the release tag + env: + TAG: ${{ needs.release-tag.outputs.tag }} + TREE: ${{ needs.release-tag.outputs.tree }} + BUILT_TREE: ${{ needs.distributions.outputs.tree }} + run: | + if [[ -z "${TREE}" || "${BUILT_TREE}" != "${TREE}" ]]; then + echo "Distributions were built from tree ${BUILT_TREE:-}, not ${TREE:-}" >&2 + exit 1 + fi + version="${TAG#v}" + shopt -s nullglob + files=(dist/*) + if (( ${#files[@]} == 0 )); then + echo "No distributions were downloaded" >&2 + exit 1 + fi + for file in "${files[@]}"; do + name="$(basename "${file}")" + case "${name}" in + sqlbuild-"${version}"-*.whl|sqlbuild-"${version}".tar.gz) ;; + sqlbuild_cli-"${version}"-*.whl|sqlbuild_cli-"${version}".tar.gz) ;; + *) + echo "Unexpected distribution ${name} for ${TAG}" >&2 + exit 1 + ;; + esac + done + printf '%s\n' "${files[@]}" - name: Publish distributions uses: pypa/gh-action-pypi-publish@release/v1 with: diff --git a/.github/workflows/release-distributions.yml b/.github/workflows/release-distributions.yml new file mode 100644 index 0000000000..ba0b94800d --- /dev/null +++ b/.github/workflows/release-distributions.yml @@ -0,0 +1,30 @@ +name: Release Distributions +run-name: >- + ${{ inputs.request_id && format('{0} [{1}]', github.workflow, inputs.request_id) + || github.workflow }} + +# Release Please dispatches this for each release pull request and reports its result through the +# required Verify status. Publish uploads these distributions once the release is tagged instead of +# rebuilding them, but only from a successful run of this workflow whose commit has exactly the +# release tag's tree; otherwise it builds from the tag. +on: + workflow_dispatch: + inputs: + request_id: + description: "Release Please attempt that dispatched this run; leave empty for manual runs" + required: false + type: string + +permissions: + contents: read + +jobs: + distributions: + name: Distributions + uses: ./.github/workflows/build-distributions.yml + with: + # Pin the dispatched commit so every job builds the tree this run is matched on. + ref: ${{ github.sha }} + upload: true + # Comfortably longer than a release pull request stays open before it is merged and published. + retention-days: 30 diff --git a/.github/workflows/release-performance.yml b/.github/workflows/release-performance.yml index a02c75e8d9..ba90a40606 100644 --- a/.github/workflows/release-performance.yml +++ b/.github/workflows/release-performance.yml @@ -1,21 +1,20 @@ name: Release Performance run-name: >- - ${{ inputs.request_id && format('{0} [{1}]', github.workflow, inputs.request_id) - || github.event.pull_request.title || github.event.head_commit.message || github.workflow }} + ${{ github.event.pull_request.title || github.event.head_commit.message || github.workflow }} -# Release Please dispatches this workflow for each release pull request and reports its result -# through the required Verify status. Pull requests that change the comparison itself run it too. +# Every push to main compares main with the latest release on the same runner. A regression fails +# the run and opens or updates one issue labelled performance-regression; it does not gate the +# release pull request. Pull requests that change the comparison itself run it too. on: + push: + branches: + - main workflow_dispatch: inputs: baseline: description: "Published sqlbuild version to compare against (default: the release before the candidate)" required: false type: string - request_id: - description: "Release Please attempt that dispatched this run; leave empty for manual runs" - required: false - type: string pull_request: paths: - .github/workflows/release-performance.yml @@ -33,8 +32,12 @@ on: permissions: contents: read +# Every main commit gets its own comparison, so a later push never cancels the run that would +# report an earlier regression. Pull request runs supersede older runs of the same pull request. concurrency: - group: release-performance-${{ github.ref }} + group: >- + release-performance-${{ github.event_name }}-${{ + github.event_name == 'push' && github.sha || github.ref }} cancel-in-progress: ${{ github.event_name == 'pull_request' }} jobs: @@ -42,6 +45,8 @@ jobs: name: Compare with the previous release runs-on: ubuntu-24.04 timeout-minutes: 90 + outputs: + regressed: ${{ steps.compare.outputs.regressed }} steps: - uses: actions/checkout@v4 with: @@ -65,32 +70,108 @@ jobs: run: | uv build --wheel --out-dir "${RUNNER_TEMP}/candidate" uv build --wheel crates/sqlbuild-cli --out-dir "${RUNNER_TEMP}/candidate" + # A release tag created while the candidate built names the baseline's benchmark generator. + - name: Fetch release tags + run: git fetch --tags --force --quiet origin # Temporary: one-model edit compiles use MAX_EDIT_COMPILE_TIME_RATIO (1.35x) because an edited # model recompiles uncached; tighten it back to 1.15x once native rendering is the default. - name: Compare candidate and previous release on this runner + id: compare env: BASELINE: ${{ inputs.baseline }} - REQUEST_ID: ${{ inputs.request_id }} + EVENT_NAME: ${{ github.event_name }} run: | - if [[ -n "${REQUEST_ID}" && -n "${BASELINE}" ]]; then - echo "Release Please runs always compare with the previous release; drop baseline" >&2 - exit 1 + # A main commit that is not itself a release still carries the latest release's version, + # so it is compared with that release rather than the one before it. A release commit + # bumps the version and is compared with the release before it. + if [[ "${EVENT_NAME}" == "push" && -z "${BASELINE}" ]]; then + project_version() { + python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["project"]["version"])' + } + version="$(project_version < pyproject.toml)" + previous="$(git show HEAD^:pyproject.toml | project_version)" || previous="" + if [[ "${version}" == "${previous}" ]]; then + BASELINE="${version}" + fi fi wheels=("${RUNNER_TEMP}"/candidate/sqlbuild-*.whl) + evidence="${RUNNER_TEMP}/release-performance" + mkdir -p "${evidence}" args=( --candidate "${wheels[0]}" - --work-dir "${RUNNER_TEMP}/release-performance" - --output "${RUNNER_TEMP}/release-performance.json" + --work-dir "${RUNNER_TEMP}/release-performance-work" + --output "${evidence}/release-performance.json" ) if [[ -n "${BASELINE}" ]]; then args+=(--baseline "${BASELINE}") fi - uv run python -m scripts.compare_release_performance "${args[@]}" + # The comparison's own exit status, not tee's, decides the outcome. + set +e + uv run python -m scripts.compare_release_performance "${args[@]}" \ + | tee "${evidence}/release-performance.md" + status="${PIPESTATUS[0]}" + set -e + regressed=false + if [[ "${status}" != "0" && -f "${evidence}/release-performance.json" ]] && \ + jq -e 'any(.verdicts[]; .regressed)' "${evidence}/release-performance.json" > /dev/null; then + regressed=true + fi + echo "status=${status}" >> "${GITHUB_OUTPUT}" + echo "regressed=${regressed}" >> "${GITHUB_OUTPUT}" - name: Upload comparison evidence if: always() uses: actions/upload-artifact@v4 with: name: release-performance-${{ github.run_id }}-${{ github.run_attempt }} - path: ${{ runner.temp }}/release-performance.json + path: ${{ runner.temp }}/release-performance if-no-files-found: warn retention-days: 90 + - name: Fail on a regression or comparison error + if: steps.compare.outputs.status != '0' + env: + STATUS: ${{ steps.compare.outputs.status }} + run: | + echo "The release performance comparison failed with exit status ${STATUS:-unknown}" >&2 + exit 1 + + report: + name: Report regression + needs: compare + if: ${{ always() && github.event_name == 'push' && needs.compare.outputs.regressed == 'true' }} + runs-on: ubuntu-24.04 + # Overlapping main runs report one at a time so they update a single regression issue. + concurrency: + group: release-performance-report + cancel-in-progress: false + permissions: + issues: write + steps: + - uses: actions/download-artifact@v4 + with: + name: release-performance-${{ github.run_id }}-${{ github.run_attempt }} + path: evidence + - name: Open or update the performance regression issue + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + LABEL: performance-regression + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + { + echo "Release performance regressed at ${GITHUB_SHA} on main." + echo + echo "Comparison run: ${RUN_URL}" + echo + cat evidence/release-performance.md + } > report.md + gh label create "${LABEL}" --color B60205 \ + --description "Main is slower or larger than the latest release" 2>/dev/null || true + issue="$(gh issue list --label "${LABEL}" --state open --limit 1 \ + --json number --jq '.[0].number // empty')" + if [[ -n "${issue}" ]]; then + gh issue comment "${issue}" --body-file report.md + echo "Updated issue #${issue}" >> "${GITHUB_STEP_SUMMARY}" + else + gh issue create --title "Release performance regression on main" \ + --label "${LABEL}" --body-file report.md + fi diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 2023792121..5c5e60d109 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -92,8 +92,11 @@ jobs: if: steps.release.outputs.prs_created == 'true' env: RELEASE_BRANCH: ${{ steps.release_branch.outputs.name }} + # Release Please only bumps workspace versions, so updating the workspace entries of + # Cargo.lock is enough and needs no compile; metadata --locked proves the lock is complete. run: | - cargo check --workspace + cargo update --workspace + cargo metadata --locked --format-version 1 > /dev/null uv lock --python 3.12 uv sync --locked --all-groups --all-extras --python 3.12 if git diff --quiet -- Cargo.lock uv.lock; then @@ -142,21 +145,24 @@ jobs: } # Verify is the required status, so it only succeeds after every trusted check passes, - # including the same-runner comparison with the previous release. Each dispatched run - # carries this attempt's request id in its run name and is matched on it exactly. - workflows=(pull-request-metadata.yml version-guard.yml release-performance.yml) - contexts=("Pull Request Metadata" "Block automatic 1.0 releases" "Release performance") + # including building the distributions that Publish reuses once the release is tagged. + # Verify's test suites skip release pull requests, whose product code already passed + # them on main; release performance runs on main after each merge and does not gate. + # Each dispatched run carries this attempt's request id in its run name and is matched + # on it exactly. + workflows=(pull-request-metadata.yml version-guard.yml release-distributions.yml) + contexts=("Pull Request Metadata" "Block automatic 1.0 releases" "Release distributions") for context in "${contexts[@]}"; do post_status pending "${context}" "Trusted release check is running" done - post_status pending Verify "Validating trusted release metadata and performance" + post_status pending Verify "Validating trusted release metadata and distributions" dispatched_after="$(date -u -d '-2 minutes' +%Y-%m-%dT%H:%M:%SZ)" gh workflow run pull-request-metadata.yml --ref "${RELEASE_BRANCH}" \ -f "pr_number=${RELEASE_PR}" -f "request_id=${REQUEST_ID}" gh workflow run version-guard.yml --ref "${RELEASE_BRANCH}" \ -f "request_id=${REQUEST_ID}" - gh workflow run release-performance.yml --ref "${RELEASE_BRANCH}" \ + gh workflow run release-distributions.yml --ref "${RELEASE_BRANCH}" \ -f "request_id=${REQUEST_ID}" for index in "${!workflows[@]}"; do workflow="${workflows[${index}]}" @@ -185,6 +191,7 @@ jobs: fi done + # Keep this allowlist in sync with the release metadata check in verify.yml. unexpected_files="$(gh pr view "${RELEASE_PR}" --repo "${GITHUB_REPOSITORY}" \ --json files --jq '[.files[].path | select(. != ".release-please-manifest.json" and . != "CHANGELOG.md" and . != "pyproject.toml" and . != "uv.lock" and . != "Cargo.lock" and . != "Cargo.toml")] | join("\n")')" if [[ -n "${unexpected_files}" ]]; then diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index 71b65946ea..132e236851 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -37,6 +37,8 @@ jobs: EVENT_NAME: ${{ github.event_name }} BASE_SHA: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha }} HEAD_SHA: ${{ github.event.pull_request.head.sha || github.event.merge_group.head_sha }} + HEAD_REF: ${{ github.event.pull_request.head.ref }} + HEAD_REPOSITORY: ${{ github.event.pull_request.head.repo.full_name }} run: | write_scope() { { @@ -68,6 +70,51 @@ jobs: exit 0 fi + # Succeeds when every change is a release metadata change: the manifest, the changelog, + # or version lines of the package and lock files, so no dependency or build setting + # moves. Keep the file list in sync with the release checks in release-please.yml. + release_metadata_only() { + local path diff offending changed_any=false + local metadata_paths="${RUNNER_TEMP}/release-metadata-paths" + local version='[0-9]+\.[0-9]+\.[0-9]+' + local version_line="^[-+](version = \"${version}\"" + version_line+="| \"[a-z0-9_-]+==${version}\", # x-release-please-version)\$" + git diff --name-only --no-renames -z "${BASE_SHA}...${HEAD_SHA}" > "${metadata_paths}" \ + || return 1 + while IFS= read -r -d '' path; do + changed_any=true + case "${path}" in + .release-please-manifest.json|CHANGELOG.md) ;; + Cargo.toml|Cargo.lock|pyproject.toml|uv.lock) + diff="$(git diff -U0 --text --no-color --no-renames "${BASE_SHA}...${HEAD_SHA}" -- "${path}")" \ + || return 1 + offending="$(awk '/^@@/ { hunk = 1; next } hunk && /^[-+]/' <<< "${diff}" \ + | grep -Ev "${version_line}" || true)" + if [[ -n "${offending}" ]]; then + echo "${path} changes more than version lines:" >&2 + echo "${offending}" >&2 + return 1 + fi + ;; + *) return 1 ;; + esac + done < "${metadata_paths}" + [[ "${changed_any}" == "true" ]] + } + + # Release Please pull requests change only release metadata on top of product code that + # already passed every suite before it reached main, so the suites are skipped. Their + # required Verify status is posted by the trusted release checks in release-please.yml + # after the metadata, version and distribution checks pass. + if [[ "${EVENT_NAME}" == "pull_request" \ + && "${HEAD_REPOSITORY}" == "${GITHUB_REPOSITORY}" \ + && "${HEAD_REF}" == release-please--branches--* ]] && release_metadata_only; then + echo "Release Please pull request changes only release metadata; skipping the suites" \ + | tee -a "${GITHUB_STEP_SUMMARY}" + write_scope false false false false false false false false + exit 0 + fi + duckdb=false postgres=false performance=false @@ -91,7 +138,7 @@ jobs: *) docs_only=false ;; esac case "${path}" in - crates/*|Cargo.toml|Cargo.lock|pyproject.toml|.github/workflows/verify.yml) + crates/*|Cargo.toml|Cargo.lock|pyproject.toml|.github/workflows/verify.yml|.github/workflows/build-distributions.yml) wheels=true ;; esac @@ -806,67 +853,12 @@ jobs: -vv --color=yes -n auto --dist loadfile wheels: - # Keep the platform matrix and maturin settings in sync with publish.yml. - name: Build wheel (${{ matrix.platform.name }}) + name: Wheels needs: [changes] if: needs.changes.outputs.wheels == 'true' - runs-on: ${{ matrix.platform.runner }} - timeout-minutes: 45 - strategy: - fail-fast: false - matrix: - platform: - - name: linux-x86_64 - runner: ubuntu-24.04 - target: x86_64 - - name: linux-aarch64 - runner: ubuntu-24.04-arm - target: aarch64 - - name: macos-x86_64 - runner: macos-15-intel - target: x86_64 - - name: macos-arm64 - runner: macos-15 - target: aarch64 - - name: windows-x86_64 - runner: windows-2025 - target: x86_64 - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - uses: dtolnay/rust-toolchain@master - with: - toolchain: stable - - uses: PyO3/maturin-action@v1 - with: - command: build - args: --release --out dist --features extension-module - target: ${{ matrix.platform.target }} - manylinux: auto - - uses: PyO3/maturin-action@v1 - with: - command: build - args: --release --out dist --manifest-path crates/sqlbuild-cli/Cargo.toml - target: ${{ matrix.platform.target }} - manylinux: auto - - name: Smoke test installed wheels and native console entry points - shell: bash - run: | - python -m venv "${RUNNER_TEMP}/wheel-smoke" - if [[ -x "${RUNNER_TEMP}/wheel-smoke/Scripts/python.exe" ]]; then - smoke_python="${RUNNER_TEMP}/wheel-smoke/Scripts/python.exe" - else - smoke_python="${RUNNER_TEMP}/wheel-smoke/bin/python" - fi - "${smoke_python}" -m pip install --find-links dist dist/*.whl - cd "${RUNNER_TEMP}" - "${smoke_python}" -c "import sqlbuild._native" - smoke_scripts="$(dirname "${smoke_python}")" - "${smoke_scripts}/sqb" --version - "${smoke_scripts}/sqb" --help - "${smoke_scripts}/sqlbuild" --help + uses: ./.github/workflows/build-distributions.yml + with: + sdist: false verify: name: Verify diff --git a/AGENTS.md b/AGENTS.md index 3831a4f3ef..96562afb4d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -50,6 +50,18 @@ - Before attributing a regression to a cause, prove it with an on/off experiment (a temporary kill switch or a revert) on the same machine. Profiles identify suspects; experiments convict. Remove experiment switches before committing. +- Profile native code with a native sampling profiler and call graphs (for example `perf` with frame + pointers). Python-level samplers misattribute time inside the extension. +- Before building an optimisation, confirm from the profile or code that the target path actually + runs on the benchmark workload under its dialect and configuration. +- Estimate the ceiling first from a per-second CPU timeline. Removing CPU from a phase that already + saturates every core saves little wall time; single-core phases bound what parallelism can save. + Do not build changes whose ceiling is small. +- Check output identity (compiled files and compile JSON excluding timings) before timing a change, + and force the path under test: for example, edit one model so per-model caches are consulted + instead of whole-project reuse. +- Measure each change against the current integrated baseline; independent savings do not simply + add up. Report only measured results or profile-derived ceilings, never projected savings. - Measure the modes users run: cold without cache, cold writing the cache, and warm. A change that affects analysis, validation, or caching must report all three. - Performance fixes must not reduce coverage. Show that diagnostics are unchanged or a strict, diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md index 5055d2f863..9268171389 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md @@ -134,11 +134,11 @@ command that needs a feature newer than a compared version declares `minimum_ver as skipped, with the reason, in the job summary; any other failure of either version fails the check. -Release Please dispatches `.github/workflows/release-performance.yml` with the metadata and version -checks and posts the required `Verify` status only after it passes, so a regressed release pull -request cannot auto-merge. Each dispatch carries a request id in its run name, and Release Please -waits only on the runs it started. After a transient failure, use **Re-run failed jobs** on the -Release Please run: it re-dispatches all three checks and repaints `Verify` on the current head. +`.github/workflows/release-performance.yml` runs on every push to `main` and does not gate release +pull requests. A commit that bumps the version is compared with the release before it; any other +commit is compared with the release whose version it still carries. A newer push cancels the +comparison of an older commit. A regression fails the run and opens an issue labelled +`performance-regression`, or comments on the open one, with the commit and the comparison summary. Run the comparison manually with an optional `baseline` input, or locally: ```bash