diff --git a/.github/workflows/pull-request-metadata.yml b/.github/workflows/pull-request-metadata.yml index 736a91f369..09d068efee 100644 --- a/.github/workflows/pull-request-metadata.yml +++ b/.github/workflows/pull-request-metadata.yml @@ -1,4 +1,7 @@ name: Pull Request Metadata +run-name: >- + ${{ inputs.request_id && format('{0} [{1}]', github.workflow, inputs.request_id) + || github.event.pull_request.title || github.event.head_commit.message || github.workflow }} on: pull_request: @@ -9,6 +12,10 @@ on: description: Pull request number to validate required: true type: number + request_id: + description: "Release Please attempt that dispatched this run; leave empty for manual runs" + required: false + type: string permissions: contents: read diff --git a/.github/workflows/release-performance.yml b/.github/workflows/release-performance.yml new file mode 100644 index 0000000000..22492427f9 --- /dev/null +++ b/.github/workflows/release-performance.yml @@ -0,0 +1,84 @@ +name: Release Performance +run-name: >- + ${{ inputs.request_id && format('{0} [{1}]', github.workflow, inputs.request_id) + || github.event.pull_request.title || github.event.head_commit.message || github.workflow }} + +# Release Please dispatches this workflow for each release pull request and reports its result +# through the required Verify status. Pull requests that change the comparison itself run it too. +on: + workflow_dispatch: + inputs: + baseline: + description: "Published sqlbuild version to compare against (default: the release before the candidate)" + required: false + type: string + request_id: + description: "Release Please attempt that dispatched this run; leave empty for manual runs" + required: false + type: string + pull_request: + paths: + - .github/workflows/release-performance.yml + - scripts/compare_release_performance.py + - scripts/release_performance/** + +permissions: + contents: read + +concurrency: + group: release-performance-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + compare: + name: Compare with the previous release + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + with: + # Release tags identify the previous release before its wheels reach PyPI. + fetch-depth: 0 + - uses: astral-sh/setup-uv@v5 + with: + version: "0.12.3" + enable-cache: true + cache-dependency-glob: | + uv.lock + Cargo.lock + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - name: Install Python + run: uv python install 3.12 + - name: Install dependencies + run: uv sync --locked --all-groups --all-extras + - name: Build candidate wheel + run: uv build --wheel --out-dir "${RUNNER_TEMP}/candidate" + - name: Compare candidate and previous release on this runner + env: + BASELINE: ${{ inputs.baseline }} + REQUEST_ID: ${{ inputs.request_id }} + run: | + if [[ -n "${REQUEST_ID}" && -n "${BASELINE}" ]]; then + echo "Release Please runs always compare with the previous release; drop baseline" >&2 + exit 1 + fi + wheels=("${RUNNER_TEMP}"/candidate/sqlbuild-*.whl) + args=( + --candidate "${wheels[0]}" + --work-dir "${RUNNER_TEMP}/release-performance" + --output "${RUNNER_TEMP}/release-performance.json" + ) + if [[ -n "${BASELINE}" ]]; then + args+=(--baseline "${BASELINE}") + fi + uv run python -m scripts.compare_release_performance "${args[@]}" + - name: Upload comparison evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: release-performance-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/release-performance.json + if-no-files-found: warn + retention-days: 90 diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 6e3c5a8890..e4b2a4b414 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -19,6 +19,10 @@ concurrency: jobs: release-please: runs-on: ubuntu-24.04 + outputs: + prs_created: ${{ steps.release.outputs.prs_created }} + release_branch: ${{ steps.release_branch.outputs.name }} + release_pr: ${{ steps.release_branch.outputs.pr }} steps: - name: Create delivery token id: delivery-token @@ -56,11 +60,13 @@ jobs: RELEASE_PRS: ${{ steps.release.outputs.prs }} run: | release_branch="$(jq -r '.[0].headBranchName // empty' <<< "${RELEASE_PRS}")" - if [[ -z "${release_branch}" ]]; then - echo "Release Please reported a created PR without a head branch" >&2 + release_pr="$(jq -r '.[0].number // empty' <<< "${RELEASE_PRS}")" + if [[ -z "${release_branch}" || -z "${release_pr}" ]]; then + echo "Release Please reported a created PR without a head branch or number" >&2 exit 1 fi echo "name=${release_branch}" >> "${GITHUB_OUTPUT}" + echo "pr=${release_pr}" >> "${GITHUB_OUTPUT}" - uses: actions/checkout@v4 if: steps.release.outputs.prs_created == 'true' @@ -99,84 +105,114 @@ jobs: git commit -m "chore: refresh release lockfiles" git push origin "HEAD:${RELEASE_BRANCH}" + # A separate job so that "Re-run failed jobs" re-dispatches and re-evaluates every trusted check + # and repaints their statuses and Verify on the current head of the release branch. + release-checks: + name: Release pull request checks + needs: release-please + if: needs.release-please.outputs.prs_created == 'true' + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.release-please.outputs.release_branch }} + - name: Run release pull request checks - if: steps.release.outputs.prs_created == 'true' env: GH_TOKEN: ${{ github.token }} - RELEASE_BRANCH: ${{ steps.release_branch.outputs.name }} - RELEASE_PRS: ${{ steps.release.outputs.prs }} + RELEASE_BRANCH: ${{ needs.release-please.outputs.release_branch }} + RELEASE_PR: ${{ needs.release-please.outputs.release_pr }} + REQUEST_ID: ${{ github.run_id }}-${{ github.run_attempt }} run: | - release_pr="$(jq -r '.[0].number // empty' <<< "${RELEASE_PRS}")" - if [[ -z "${release_pr}" ]]; then - echo "Release Please reported a created PR without a number" >&2 - exit 1 - fi head_sha="$(git rev-parse HEAD)" + recovery="rerun the failed Release Please job to re-check" + post_status() { + gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ + -f "state=$1" -f "context=$2" -f "description=$3" ${4:+-f "target_url=$4"} >/dev/null + } + fail_verify() { + post_status failure Verify "$1; ${recovery}" "${2:-}" + { + echo "### Release pull request checks failed" + echo + echo "$1. After fixing or confirming a transient cause, use **Re-run failed jobs**" + echo "on this workflow run: it re-dispatches every trusted check and repaints Verify." + } >> "${GITHUB_STEP_SUMMARY}" + exit 1 + } - workflows=(pull-request-metadata.yml version-guard.yml) - contexts=("Pull Request Metadata" "Block automatic 1.0 releases") + # Verify is the required status, so it only succeeds after every trusted check passes, + # including the same-runner comparison with the previous release. Each dispatched run + # carries this attempt's request id in its run name and is matched on it exactly. + workflows=(pull-request-metadata.yml version-guard.yml release-performance.yml) + contexts=("Pull Request Metadata" "Block automatic 1.0 releases" "Release performance") for context in "${contexts[@]}"; do - gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ - -f state=pending -f "context=${context}" \ - -f description="Trusted release check is running" >/dev/null + post_status pending "${context}" "Trusted release check is running" done - gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ - -f state=pending -f context=Verify \ - -f description="Validating trusted release metadata" >/dev/null + post_status pending Verify "Validating trusted release metadata and performance" + dispatched_after="$(date -u -d '-2 minutes' +%Y-%m-%dT%H:%M:%SZ)" gh workflow run pull-request-metadata.yml --ref "${RELEASE_BRANCH}" \ - -f "pr_number=${release_pr}" - gh workflow run version-guard.yml --ref "${RELEASE_BRANCH}" + -f "pr_number=${RELEASE_PR}" -f "request_id=${REQUEST_ID}" + gh workflow run version-guard.yml --ref "${RELEASE_BRANCH}" \ + -f "request_id=${REQUEST_ID}" + gh workflow run release-performance.yml --ref "${RELEASE_BRANCH}" \ + -f "request_id=${REQUEST_ID}" for index in "${!workflows[@]}"; do workflow="${workflows[${index}]}" context="${contexts[${index}]}" run_id="" for _ in {1..30}; do run_id="$(gh run list --workflow "${workflow}" --branch "${RELEASE_BRANCH}" \ - --event workflow_dispatch --limit 20 --json databaseId,headSha \ - --jq ".[] | select(.headSha == \"${head_sha}\") | .databaseId" | head -n 1)" + --event workflow_dispatch --limit 50 \ + --json databaseId,headSha,displayTitle,createdAt \ + --jq ".[] | select(.headSha == \"${head_sha}\" + and (.displayTitle | endswith(\" [${REQUEST_ID}]\")) + and .createdAt >= \"${dispatched_after}\") | .databaseId" | head -n 1)" [[ -n "${run_id}" ]] && break sleep 2 done if [[ -z "${run_id}" ]]; then - gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ - -f state=failure -f "context=${context}" \ - -f description="Trusted release check did not start" >/dev/null - exit 1 + post_status failure "${context}" "Trusted release check did not start" + fail_verify "${context} did not start" fi run_url="$(gh run view "${run_id}" --json url --jq .url)" - if gh run watch "${run_id}" --exit-status; then - gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ - -f state=success -f "context=${context}" -f target_url="${run_url}" \ - -f description="Trusted release check passed" >/dev/null + if gh run watch "${run_id}" --exit-status --interval 30; then + post_status success "${context}" "Trusted release check passed" "${run_url}" else - gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ - -f state=failure -f "context=${context}" -f target_url="${run_url}" \ - -f description="Trusted release check failed" >/dev/null - exit 1 + post_status failure "${context}" "Trusted release check failed" "${run_url}" + fail_verify "${context} failed" "${run_url}" fi done - unexpected_files="$(gh pr view "${release_pr}" --repo "${GITHUB_REPOSITORY}" \ + unexpected_files="$(gh pr view "${RELEASE_PR}" --repo "${GITHUB_REPOSITORY}" \ --json files --jq '[.files[].path | select(. != ".release-please-manifest.json" and . != "CHANGELOG.md" and . != "pyproject.toml" and . != "uv.lock" and . != "Cargo.lock" and . != "crates/sqlbuild-rules-native/Cargo.toml")] | join("\n")')" if [[ -n "${unexpected_files}" ]]; then echo "Release pull request contains non-metadata changes:" >&2 echo "${unexpected_files}" >&2 - gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ - -f state=failure -f context=Verify \ - -f description="Release pull request contains non-metadata changes" >/dev/null + post_status failure Verify "Release pull request contains non-metadata changes" exit 1 fi - gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ - -f state=success -f context=Verify \ - -f description="Product changes passed verification before release metadata update" >/dev/null + post_status success Verify \ + "Product changes passed verification before release metadata update" + + # The checks can wait longer than an installation token lives, so mint one just before use. + - name: Create delivery token + id: delivery-token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.CHIO_DELIVERY_APP_ID }} + private-key: ${{ secrets.CHIO_DELIVERY_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ github.event.repository.name }} + permission-actions: write + permission-contents: write + permission-issues: write + permission-pull-requests: write - name: Enable release pull request auto-merge - if: steps.release.outputs.prs_created == 'true' env: # The delivery app merges, so the resulting push to main starts the tag and publish runs. GH_TOKEN: ${{ steps.delivery-token.outputs.token }} - RELEASE_PRS: ${{ steps.release.outputs.prs }} - run: | - release_pr="$(jq -r '.[0].number // empty' <<< "${RELEASE_PRS}")" - gh pr merge "${release_pr}" --repo "${GITHUB_REPOSITORY}" --auto --squash + RELEASE_PR: ${{ needs.release-please.outputs.release_pr }} + run: gh pr merge "${RELEASE_PR}" --repo "${GITHUB_REPOSITORY}" --auto --squash diff --git a/.github/workflows/version-guard.yml b/.github/workflows/version-guard.yml index 44c6345efe..064bf19e53 100644 --- a/.github/workflows/version-guard.yml +++ b/.github/workflows/version-guard.yml @@ -1,4 +1,7 @@ name: Version Guard +run-name: >- + ${{ inputs.request_id && format('{0} [{1}]', github.workflow, inputs.request_id) + || github.event.pull_request.title || github.event.head_commit.message || github.workflow }} on: pull_request: @@ -8,6 +11,11 @@ on: branches: - main workflow_dispatch: + inputs: + request_id: + description: "Release Please attempt that dispatched this run; leave empty for manual runs" + required: false + type: string permissions: contents: read diff --git a/Makefile b/Makefile index e0dcabcf55..5e1dfc37ec 100644 --- a/Makefile +++ b/Makefile @@ -121,6 +121,7 @@ E2E_DUCKDB_CLI_DATA_PATHS := \ E2E_DUCKDB_CLI_PATHS := \ tests/e2e/scripts/cli_preview \ tests/e2e/scripts/compile_performance_ratio \ + tests/e2e/scripts/release_performance \ tests/e2e/src/sqlbuild/cli/commands/main/adapters \ tests/e2e/src/sqlbuild/cli/commands/main/audit \ tests/e2e/src/sqlbuild/cli/commands/main/bigquery \ diff --git a/scripts/compare_release_performance.py b/scripts/compare_release_performance.py new file mode 100644 index 0000000000..e6b3ec1daa --- /dev/null +++ b/scripts/compare_release_performance.py @@ -0,0 +1,15 @@ +"""Direct wrapper for comparing a release candidate with the previous published release.""" + +from __future__ import annotations + +from scripts.release_performance.main.compare import compare_release_performance + + +def main(argv: list[str] | None = None) -> int: + """Compare candidate and baseline command performance on the same machine.""" + + return compare_release_performance(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/compile_performance_ratio/_helpers/report.py b/scripts/compile_performance_ratio/_helpers/report.py index 586f218f68..3df30ab101 100644 --- a/scripts/compile_performance_ratio/_helpers/report.py +++ b/scripts/compile_performance_ratio/_helpers/report.py @@ -15,7 +15,7 @@ def comparison_markdown(*, comparison: CompileComparison, runs: int, max_ratio: lines: list[str] = [ f"### {comparison.kind} {comparison.models} models: head vs base (same runner)", "", - f"Runner CPU: {_cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.", + f"Runner CPU: {cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.", "", "| Metric | Base | Head | Ratio |", "|---|---:|---:|---:|", @@ -41,7 +41,9 @@ def append_summary(*, path: Path | None, markdown: str) -> None: summary.write(markdown + "\n") -def _cpu_model() -> str: +def cpu_model() -> str: + """Return the runner CPU model name.""" + cpuinfo: Path = Path("/proc/cpuinfo") if cpuinfo.exists(): for line in cpuinfo.read_text(encoding="utf-8").splitlines(): diff --git a/scripts/release_performance/__init__.py b/scripts/release_performance/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/release_performance/_helpers/__init__.py b/scripts/release_performance/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/release_performance/_helpers/benchmark.py b/scripts/release_performance/_helpers/benchmark.py new file mode 100644 index 0000000000..7dfd8cbb53 --- /dev/null +++ b/scripts/release_performance/_helpers/benchmark.py @@ -0,0 +1,193 @@ +"""Prepare identical benchmark projects per version and time interleaved command runs.""" + +from __future__ import annotations + +import json +import os +import shutil +import signal +import subprocess +import sys +from pathlib import Path +from typing import cast + +from scripts.release_performance.constants import ( + BUILD_PROJECT, + BUILD_WARMUPS, + COMMAND_TIMEOUT_SECONDS, + COMPILE_COMMAND, + EXCLUDED_ENVIRONMENT_KEYS, + EXCLUDED_ENVIRONMENT_PREFIX, + INSPECTION_PROJECT, + INSPECTION_WARMUPS, + STDERR_TAIL_CHARACTERS, + TIME_FORMAT, +) +from scripts.release_performance.exceptions import ReleasePerformanceError +from scripts.release_performance.models import ( + BenchmarkCommand, + CommandComparison, + CommandSample, + InstalledVersion, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + write_build_benchmark_project, + write_inspection_benchmark_project, +) + +_MIB: int = 1024 * 1024 +_TEMPLATE_SUFFIX: str = "-template" + + +def write_pristine_projects(*, root: Path, inspection_models: int, build_models: int) -> Path: + """Generate the inspection and build benchmarks once so every version reads the same files.""" + + pristine: Path = root / "pristine" + write_inspection_benchmark_project( + project_dir=pristine / INSPECTION_PROJECT, model_count=inspection_models + ) + write_build_benchmark_project(project_dir=pristine / BUILD_PROJECT, model_count=build_models) + return pristine + + +def prepare_version_projects( + *, version: InstalledVersion, pristine: Path, root: Path +) -> dict[str, Path]: + """Copy the pristine projects for one version and warm its caches with untimed runs.""" + + projects: dict[str, Path] = { + INSPECTION_PROJECT: root / version.label / INSPECTION_PROJECT, + BUILD_PROJECT: root / version.label / BUILD_PROJECT, + } + logs: Path = root / version.label / "logs" + logs.mkdir(parents=True, exist_ok=True) + for name, warmups in ((INSPECTION_PROJECT, INSPECTION_WARMUPS), (BUILD_PROJECT, BUILD_WARMUPS)): + _ = shutil.copytree(pristine / name, projects[name]) + for index, sqb_args in enumerate(warmups): + output: Path = _run_sqb( + version=version, + project_dir=projects[name], + sqb_args=sqb_args, + log_prefix=logs / f"warmup-{name}-{index}", + )[1] + if sqb_args[0] == COMPILE_COMMAND: + _assert_clean_compile(version=version, output=output) + _ = shutil.copytree( + projects[BUILD_PROJECT], projects[BUILD_PROJECT].with_name(BUILD_PROJECT + _TEMPLATE_SUFFIX) + ) + return projects + + +def compare_command( + *, + command: BenchmarkCommand, + versions: tuple[InstalledVersion, InstalledVersion], + projects: dict[str, dict[str, Path]], + root: Path, + runs: int, +) -> CommandComparison: + """Time baseline and candidate alternately runs times each and keep every sample.""" + + samples: dict[str, list[CommandSample]] = {version.label: [] for version in versions} + for run in range(runs): + for version in versions: + project_dir: Path = projects[version.label][command.project] + if command.project == BUILD_PROJECT: + _reset_build_project(project_dir=project_dir) + sample: CommandSample = _run_sqb( + version=version, + project_dir=project_dir, + sqb_args=command.sqb_args, + log_prefix=root / version.label / "logs" / "measured", + )[0] + samples[version.label].append(sample) + print( + f"{command.name} run {run + 1}/{runs} {version.label} {version.version}: " + f"wall {sample.wall_seconds:.2f}s, CPU {sample.cpu_seconds:.2f}s, " + f"peak RSS {sample.peak_rss_bytes / _MIB:.0f} MiB", + file=sys.stderr, + flush=True, + ) + baseline, candidate = versions + return CommandComparison( + name=command.name, + baseline=tuple(samples[baseline.label]), + candidate=tuple(samples[candidate.label]), + ) + + +def _reset_build_project(*, project_dir: Path) -> None: + shutil.rmtree(project_dir, ignore_errors=True) + _ = shutil.copytree(project_dir.with_name(BUILD_PROJECT + _TEMPLATE_SUFFIX), project_dir) + + +def _assert_clean_compile(*, version: InstalledVersion, output: Path) -> None: + payload: dict[str, object] = cast(dict[str, object], json.loads(output.read_bytes())) + diagnostics: list[object] = cast(list[object], payload.get("diagnostics", [])) + if diagnostics: + raise ReleasePerformanceError( + f"The benchmark is not valid for {version.label} {version.version}: compile reported " + f"{len(diagnostics)} diagnostics, first {json.dumps(diagnostics[0])[:500]}" + ) + + +def _run_sqb( + *, + version: InstalledVersion, + project_dir: Path, + sqb_args: tuple[str, ...], + log_prefix: Path, +) -> tuple[CommandSample, Path]: + measurement: Path = log_prefix.with_suffix(".time") + output: Path = log_prefix.with_suffix(".out") + errors: Path = log_prefix.with_suffix(".err") + command: list[str] = [ + "/usr/bin/time", + "--quiet", + "--output", + str(measurement), + "--format", + TIME_FORMAT, + str(version.sqb), + "--project-dir", + str(project_dir), + "--no-color", + *sqb_args, + ] + with output.open("wb") as stdout, errors.open("wb") as stderr: + with subprocess.Popen( + command, stdout=stdout, stderr=stderr, env=_environment(), start_new_session=True + ) as process: + try: + returncode: int = process.wait(timeout=COMMAND_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + _ = process.wait() + raise ReleasePerformanceError( + f"{version.label} {version.version} `sqb {' '.join(sqb_args)}` exceeded " + f"{COMMAND_TIMEOUT_SECONDS:.0f}s" + ) from None + if returncode != 0: + tail: str = errors.read_text(encoding="utf-8", errors="replace")[-STDERR_TAIL_CHARACTERS:] + raise ReleasePerformanceError( + f"{version.label} {version.version} `sqb {' '.join(sqb_args)}` exited with " + f"{returncode}. If the command needs a feature newer than this version, declare " + f"minimum_version for it in scripts/release_performance/constants.py.\n{tail}" + ) + wall, rss_kib, user, system = measurement.read_text(encoding="utf-8").split()[-4:] + return ( + CommandSample( + wall_seconds=float(wall), + cpu_seconds=float(user) + float(system), + peak_rss_bytes=int(rss_kib) * 1024, + ), + output, + ) + + +def _environment() -> dict[str, str]: + return { + key: value + for key, value in os.environ.items() + if key not in EXCLUDED_ENVIRONMENT_KEYS and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX) + } diff --git a/scripts/release_performance/_helpers/report.py b/scripts/release_performance/_helpers/report.py new file mode 100644 index 0000000000..43eed207c9 --- /dev/null +++ b/scripts/release_performance/_helpers/report.py @@ -0,0 +1,94 @@ +"""Render a release performance comparison for logs and the CI job summary.""" + +from __future__ import annotations + +from pathlib import Path + +from scripts.release_performance._helpers.verdict import metric_verdicts, regression_messages +from scripts.release_performance.constants import ( + MAX_CPU_RATIO, + MAX_RSS_RATIO, + MAX_WALL_RATIO, + MIN_CPU_REGRESSION_SECONDS, + MIN_RSS_REGRESSION_BYTES, + MIN_WALL_REGRESSION_SECONDS, +) +from scripts.release_performance.models import MetricVerdict, ReleaseComparison + +_MIB: int = 1024 * 1024 + + +def comparison_markdown(*, comparison: ReleaseComparison) -> str: + """Summarize per-command medians, ratios and the overall verdict as Markdown.""" + + lines: list[str] = [ + f"### Release performance: {comparison.candidate_version} (candidate) vs " + f"{comparison.baseline_version} (baseline)", + "", + f"Same runner: {comparison.runner.cpu_model}, {comparison.runner.cpu_count} CPUs; " + f"load average {_load(comparison.runner.load_average_before)} before and " + f"{_load(comparison.runner.load_average_after)} after. " + f"{comparison.runs} interleaved runs per version; medians shown.", + "", + f"Limits: wall {_percent(MAX_WALL_RATIO)} (ignored under " + f"{MIN_WALL_REGRESSION_SECONDS:g} s), CPU {_percent(MAX_CPU_RATIO)} (ignored under " + f"{MIN_CPU_REGRESSION_SECONDS:g} s), peak RSS {_percent(MAX_RSS_RATIO)} (ignored under " + f"{MIN_RSS_REGRESSION_BYTES // _MIB} MiB).", + "", + "| Command | Wall (s) | Wall ratio | CPU (s) | CPU ratio | Peak RSS (MiB) | RSS ratio " + "| Result |", + "|---|---:|---:|---:|---:|---:|---:|---|", + ] + for command in comparison.commands: + wall, cpu, rss = metric_verdicts(comparison=command) + regressed: bool = wall.regressed or cpu.regressed or rss.regressed + lines.append( + f"| `{command.name}` " + f"| {wall.baseline:.2f} → {wall.candidate:.2f} | {_ratio(wall)} " + f"| {cpu.baseline:.2f} → {cpu.candidate:.2f} | {_ratio(cpu)} " + f"| {rss.baseline / _MIB:.0f} → {rss.candidate / _MIB:.0f} | {_ratio(rss)} " + f"| {'❌ regressed' if regressed else '✅ ok'} |" + ) + for skipped in comparison.skipped: + lines.append(f"| `{skipped.name}` | – | – | – | – | – | – | ⚠️ skipped: {skipped.reason} |") + lines.append("") + failures: tuple[str, ...] = regression_messages(commands=comparison.commands) + if failures: + lines.append("**Result: failed.** " + "; ".join(failures) + ".") + else: + lines.append("**Result: passed.** No command exceeded the release limits.") + if comparison.skipped: + lines.append("") + lines.append( + f"**{len(comparison.skipped)} command(s) skipped** because a compared version " + "predates them; they were not measured." + ) + return "\n".join(lines) + "\n" + + +def error_markdown(*, message: str) -> str: + """Report a comparison that could not complete.""" + + return f"### Release performance\n\n**Result: error.** {message}\n" + + +def append_summary(*, path: Path | None, markdown: str) -> None: + """Append to the CI job summary when one is configured.""" + + if path is None: + return + with path.open("a", encoding="utf-8") as summary: + _ = summary.write(markdown + "\n") + + +def _ratio(verdict: MetricVerdict) -> str: + text: str = f"{verdict.ratio:.3f}x" + return f"**{text}**" if verdict.regressed else text + + +def _percent(ratio: float) -> str: + return f"+{round((ratio - 1) * 100)}%" + + +def _load(values: tuple[float, float, float]) -> str: + return "/".join(f"{value:.2f}" for value in values) diff --git a/scripts/release_performance/_helpers/verdict.py b/scripts/release_performance/_helpers/verdict.py new file mode 100644 index 0000000000..d688ac2451 --- /dev/null +++ b/scripts/release_performance/_helpers/verdict.py @@ -0,0 +1,122 @@ +"""Decide whether the candidate regressed against the baseline beyond the release limits.""" + +from __future__ import annotations + +import statistics + +from scripts.release_performance.constants import ( + MAX_CPU_RATIO, + MAX_RSS_RATIO, + MAX_WALL_RATIO, + MIN_CPU_REGRESSION_SECONDS, + MIN_RSS_REGRESSION_BYTES, + MIN_WALL_REGRESSION_SECONDS, +) +from scripts.release_performance.models import ( + BenchmarkCommand, + CommandComparison, + CommandSample, + MetricVerdict, +) + + +def metric_verdicts(*, comparison: CommandComparison) -> tuple[MetricVerdict, ...]: + """Compare median wall time, CPU time and peak RSS of one command.""" + + baseline: CommandSample = median_sample(samples=comparison.baseline) + candidate: CommandSample = median_sample(samples=comparison.candidate) + return ( + _verdict( + command=comparison.name, + metric="wall", + baseline=baseline.wall_seconds, + candidate=candidate.wall_seconds, + max_ratio=MAX_WALL_RATIO, + floor=MIN_WALL_REGRESSION_SECONDS, + ), + _verdict( + command=comparison.name, + metric="CPU", + baseline=baseline.cpu_seconds, + candidate=candidate.cpu_seconds, + max_ratio=MAX_CPU_RATIO, + floor=MIN_CPU_REGRESSION_SECONDS, + ), + _verdict( + command=comparison.name, + metric="peak RSS", + baseline=float(baseline.peak_rss_bytes), + candidate=float(candidate.peak_rss_bytes), + max_ratio=MAX_RSS_RATIO, + floor=float(MIN_RSS_REGRESSION_BYTES), + ), + ) + + +def regression_messages(*, commands: tuple[CommandComparison, ...]) -> tuple[str, ...]: + """Describe every metric that exceeds both its ratio limit and its absolute floor.""" + + return tuple( + f"{verdict.command}: {verdict.metric} {verdict.ratio:.3f}x exceeds {verdict.max_ratio:.2f}x" + for verdict in all_verdicts(commands=commands) + if verdict.regressed + ) + + +def all_verdicts(*, commands: tuple[CommandComparison, ...]) -> tuple[MetricVerdict, ...]: + """Return the metric verdicts of every measured command in order.""" + + verdicts: list[MetricVerdict] = [] + for comparison in commands: + verdicts.extend(metric_verdicts(comparison=comparison)) + return tuple(verdicts) + + +def median_sample(*, samples: tuple[CommandSample, ...]) -> CommandSample: + """Return the per-metric median of samples.""" + + return CommandSample( + wall_seconds=statistics.median(sample.wall_seconds for sample in samples), + cpu_seconds=statistics.median(sample.cpu_seconds for sample in samples), + peak_rss_bytes=int(statistics.median(sample.peak_rss_bytes for sample in samples)), + ) + + +def skip_reason( + *, command: BenchmarkCommand, baseline_version: str, candidate_version: str +) -> str | None: + """Explain why a command cannot run on a compared version, or return None when it can.""" + + if command.minimum_version is None: + return None + minimum: tuple[int, ...] = version_key(version=command.minimum_version) + for label, version in (("baseline", baseline_version), ("candidate", candidate_version)): + if version_key(version=version) < minimum: + return f"requires sqlbuild {command.minimum_version} or later; {label} is {version}" + return None + + +def version_key(*, version: str) -> tuple[int, ...]: + """Order plain X.Y.Z release versions numerically.""" + + return tuple(int(part) for part in version.split(".")) + + +def _verdict( + *, + command: str, + metric: str, + baseline: float, + candidate: float, + max_ratio: float, + floor: float, +) -> MetricVerdict: + return MetricVerdict( + command=command, + metric=metric, + baseline=baseline, + candidate=candidate, + max_ratio=max_ratio, + floor=floor, + regressed=candidate > baseline * max_ratio and candidate - baseline > floor, + ) diff --git a/scripts/release_performance/_helpers/versions.py b/scripts/release_performance/_helpers/versions.py new file mode 100644 index 0000000000..616405439b --- /dev/null +++ b/scripts/release_performance/_helpers/versions.py @@ -0,0 +1,302 @@ +"""Resolve and install the compared sqlbuild versions, each into its own virtual environment.""" + +from __future__ import annotations + +import hashlib +import json +import platform +import re +import shutil +import subprocess +import sys +import time +import urllib.request +from pathlib import Path +from typing import cast + +from scripts.release_performance._helpers.verdict import version_key +from scripts.release_performance.constants import ( + BASELINE_PUBLICATION_POLL_SECONDS, + PACKAGE_NAME, + PYPI_REQUEST_TIMEOUT_SECONDS, + PYPI_SIMPLE_JSON, + PYPI_SIMPLE_URL, + RELEASE_TAG_PATTERN, + STDERR_TAIL_CHARACTERS, + UNIVERSAL_WHEEL_MACHINE, + WHEEL_MACHINE_ALIASES, + WHEEL_PLATFORM_FAMILIES, +) +from scripts.release_performance.exceptions import ReleasePerformanceError + +_RELEASE_VERSION: re.Pattern[str] = re.compile(r"\d+\.\d+\.\d+") +_DISTRIBUTION_VERSION: re.Pattern[str] = re.compile( + rf"^{PACKAGE_NAME}-(\d+\.\d+\.\d+)(?:-.+\.whl|\.tar\.gz)$" +) + + +def release_versions(*, index: dict[str, object]) -> tuple[str, ...]: + """Return the X.Y.Z versions in a PEP 691 index that have a file that is not yanked.""" + + return _index_versions(index=index, include_yanked=False) + + +def choose_baseline( + *, candidate: str, index: dict[str, object], tags: tuple[str, ...] +) -> str | None: + """Return the highest installable or still-publishing release below the candidate.""" + + listed: frozenset[str] = frozenset(_index_versions(index=index, include_yanked=True)) + publishing: tuple[str, ...] = tuple(tag for tag in tags if tag not in listed) + return previous_version( + candidate=candidate, versions=(*release_versions(index=index), *publishing) + ) + + +def previous_version(*, candidate: str, versions: tuple[str, ...]) -> str | None: + """Return the highest version strictly below candidate.""" + + earlier: list[str] = [ + version + for version in versions + if version_key(version=version) < version_key(version=candidate) + ] + return max(earlier, key=lambda version: version_key(version=version)) if earlier else None + + +def compatible_wheels( + *, index: dict[str, object], version: str, machine: str, system: str +) -> tuple[dict[str, object], ...]: + """Return the non-yanked wheels of version whose platform tag matches this runner.""" + + files: list[dict[str, object]] = cast(list[dict[str, object]], index.get("files", [])) + prefix: str = f"{PACKAGE_NAME}-{version}-" + return tuple( + entry + for entry in files + if str(entry.get("filename", "")).startswith(prefix) + and str(entry.get("filename", "")).endswith(".whl") + and not entry.get("yanked") + and _platform_matches( + platform_tag=str(entry["filename"]).removesuffix(".whl").rsplit("-", 1)[-1], + machine=machine.lower(), + system=system, + ) + ) + + +def fetch_simple_index() -> dict[str, object]: + """Read the PyPI simple index uncompressed, which reflects new uploads before the CDN does.""" + + request: urllib.request.Request = urllib.request.Request( + PYPI_SIMPLE_URL, + headers={ + "Accept": PYPI_SIMPLE_JSON, + "Accept-Encoding": "identity", + "Cache-Control": "no-cache", + }, + ) + with urllib.request.urlopen(request, timeout=PYPI_REQUEST_TIMEOUT_SECONDS) as response: + return cast(dict[str, object], json.loads(response.read())) + + +def tagged_versions(*, repo_dir: Path) -> tuple[str, ...]: + """Return the X.Y.Z versions of the release tags in repo_dir, or none outside a clone.""" + + completed: subprocess.CompletedProcess[str] = subprocess.run( + ["git", "-C", str(repo_dir), "tag", "--list", RELEASE_TAG_PATTERN], + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + return () + return tuple( + tag.removeprefix("v") + for tag in completed.stdout.split() + if _RELEASE_VERSION.fullmatch(tag.removeprefix("v")) + ) + + +def resolve_baseline(*, candidate: str, repo_dir: Path) -> str: + """Return the highest published or not-yet-published tagged release below the candidate.""" + + baseline: str | None = choose_baseline( + candidate=candidate, + index=fetch_simple_index(), + tags=tagged_versions(repo_dir=repo_dir), + ) + if baseline is None: + raise ReleasePerformanceError(f"No published sqlbuild release precedes {candidate}") + return baseline + + +def install_published(*, version: str, venv_dir: Path, python: str, wait_seconds: float) -> Path: + """Install a published version, falling back to its wheel URL while the CDN lags.""" + + _create_venv(venv_dir=venv_dir, python=python) + deadline: float = time.monotonic() + wait_seconds + while True: + attempt: subprocess.CompletedProcess[str] = _uv_pip_install( + venv_dir=venv_dir, requirements=(f"{PACKAGE_NAME}=={version}",) + ) + if attempt.returncode == 0: + return _sqb(venv_dir=venv_dir) + print( + f"Index install of {PACKAGE_NAME}=={version} failed; trying the wheel URL: " + f"{_tail(attempt.stderr)}", + file=sys.stderr, + ) + if _install_from_wheel_urls(version=version, venv_dir=venv_dir): + return _sqb(venv_dir=venv_dir) + if time.monotonic() >= deadline: + raise ReleasePerformanceError( + f"{PACKAGE_NAME} {version} is not installable from PyPI; if the release is still " + "publishing, rerun once its wheels are available" + ) + print( + f"{PACKAGE_NAME} {version} is not on PyPI yet; retrying in " + f"{BASELINE_PUBLICATION_POLL_SECONDS:.0f}s", + file=sys.stderr, + ) + time.sleep(BASELINE_PUBLICATION_POLL_SECONDS) + + +def install_wheel(*, wheel: Path, venv_dir: Path, python: str) -> Path: + """Install a locally built wheel into a fresh virtual environment.""" + + _create_venv(venv_dir=venv_dir, python=python) + completed: subprocess.CompletedProcess[str] = _uv_pip_install( + venv_dir=venv_dir, requirements=(str(wheel.resolve()),) + ) + if completed.returncode != 0: + raise ReleasePerformanceError(f"Installing {wheel} failed: {_tail(completed.stderr)}") + return _sqb(venv_dir=venv_dir) + + +def installed_version(*, sqb: Path) -> str: + """Return the version an installed sqb reports.""" + + completed: subprocess.CompletedProcess[str] = subprocess.run( + [str(sqb), "--version"], capture_output=True, text=True, check=False + ) + match: re.Match[str] | None = _RELEASE_VERSION.search(completed.stdout) + if completed.returncode != 0 or match is None: + raise ReleasePerformanceError(f"{sqb} --version failed: {_tail(completed.stderr)}") + return match.group(0) + + +def existing_sqb(*, venv_dir: Path) -> Path: + """Return the sqb entry point of an existing virtual environment.""" + + return _sqb(venv_dir=venv_dir) + + +def _index_versions(*, index: dict[str, object], include_yanked: bool) -> tuple[str, ...]: + versions: set[str] = set() + files: list[dict[str, object]] = cast(list[dict[str, object]], index.get("files", [])) + for entry in files: + match: re.Match[str] | None = _DISTRIBUTION_VERSION.match(str(entry.get("filename", ""))) + if match is not None and (include_yanked or not entry.get("yanked")): + versions.add(match.group(1)) + return tuple(sorted(versions, key=lambda version: version_key(version=version))) + + +def _install_from_wheel_urls(*, version: str, venv_dir: Path) -> bool: + try: + index: dict[str, object] = fetch_simple_index() + except OSError as error: + print(f"Reading {PYPI_SIMPLE_URL} failed: {error}", file=sys.stderr) + return False + wheels: tuple[dict[str, object], ...] = compatible_wheels( + index=index, version=version, machine=platform.machine(), system=sys.platform + ) + if not wheels: + return False + download_dir: Path = venv_dir.parent / f"{venv_dir.name}-wheels" + download_dir.mkdir(parents=True, exist_ok=True) + for entry in wheels: + _download_wheel(entry=entry, download_dir=download_dir) + completed: subprocess.CompletedProcess[str] = _uv_pip_install( + venv_dir=venv_dir, + requirements=("--find-links", str(download_dir), f"{PACKAGE_NAME}=={version}"), + ) + if completed.returncode != 0: + print(f"Wheel URL install failed: {_tail(completed.stderr)}", file=sys.stderr) + return completed.returncode == 0 + + +def _download_wheel(*, entry: dict[str, object], download_dir: Path) -> None: + request: urllib.request.Request = urllib.request.Request( + str(entry["url"]), headers={"Accept-Encoding": "identity"} + ) + with urllib.request.urlopen(request, timeout=PYPI_REQUEST_TIMEOUT_SECONDS) as response: + content: bytes = response.read() + hashes: dict[str, str] = cast(dict[str, str], entry.get("hashes", {})) + expected: str | None = hashes.get("sha256") + if expected is not None and hashlib.sha256(content).hexdigest() != expected: + raise ReleasePerformanceError(f"Downloaded {entry['filename']} does not match its sha256") + _ = (download_dir / str(entry["filename"])).write_bytes(content) + + +def _platform_matches(*, platform_tag: str, machine: str, system: str) -> bool: + family: str | None = WHEEL_PLATFORM_FAMILIES.get(system) + machines: tuple[str, ...] = ( + machine, + WHEEL_MACHINE_ALIASES.get(machine, machine), + UNIVERSAL_WHEEL_MACHINE, + ) + if family is None or family not in platform_tag: + return False + return any(name in platform_tag for name in machines) + + +def _create_venv(*, venv_dir: Path, python: str) -> None: + shutil.rmtree(venv_dir, ignore_errors=True) + completed: subprocess.CompletedProcess[str] = subprocess.run( + [_uv(), "venv", "--quiet", "--python", python, str(venv_dir)], + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + raise ReleasePerformanceError( + f"Creating a Python {python} environment failed: {_tail(completed.stderr)}" + ) + + +def _uv_pip_install( + *, venv_dir: Path, requirements: tuple[str, ...] +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + _uv(), + "pip", + "install", + "--quiet", + "--python", + str(venv_dir / "bin" / "python"), + *requirements, + ], + capture_output=True, + text=True, + check=False, + ) + + +def _sqb(*, venv_dir: Path) -> Path: + sqb: Path = venv_dir / "bin" / "sqb" + if not sqb.exists(): + raise ReleasePerformanceError(f"{venv_dir} has no sqb entry point") + return sqb + + +def _uv() -> str: + uv: str | None = shutil.which("uv") + if uv is None: + raise ReleasePerformanceError("uv is required to create the compared environments") + return uv + + +def _tail(text: str) -> str: + return text.strip()[-STDERR_TAIL_CHARACTERS:] diff --git a/scripts/release_performance/_helpers/workflow.py b/scripts/release_performance/_helpers/workflow.py new file mode 100644 index 0000000000..5aefcaca53 --- /dev/null +++ b/scripts/release_performance/_helpers/workflow.py @@ -0,0 +1,172 @@ +"""Install both versions, time the benchmark commands and report the release verdict.""" + +from __future__ import annotations + +import dataclasses +import json +import os +import sys +import tempfile +from pathlib import Path + +from scripts.compile_performance_ratio._helpers.report import cpu_model +from scripts.release_performance._helpers.benchmark import ( + compare_command, + prepare_version_projects, + write_pristine_projects, +) +from scripts.release_performance._helpers.report import ( + append_summary, + comparison_markdown, + error_markdown, +) +from scripts.release_performance._helpers.verdict import ( + all_verdicts, + regression_messages, + skip_reason, +) +from scripts.release_performance._helpers.versions import ( + existing_sqb, + install_published, + install_wheel, + installed_version, + resolve_baseline, +) +from scripts.release_performance.constants import ( + BASELINE_LABEL, + BASELINE_PUBLICATION_WAIT_SECONDS, + BENCHMARK_COMMANDS, + CANDIDATE_LABEL, + REPO_ROOT, +) +from scripts.release_performance.exceptions import ReleasePerformanceError +from scripts.release_performance.models import ( + CommandComparison, + ComparisonOptions, + InstalledVersion, + ReleaseComparison, + RunnerContext, + SkippedCommand, +) + + +def run_release_comparison(*, options: ComparisonOptions) -> int: + """Compare the candidate with the baseline release and enforce the release limits.""" + + summary_value: str | None = os.environ.get("GITHUB_STEP_SUMMARY") + summary: Path | None = Path(summary_value) if summary_value else None + try: + if options.work_dir is not None: + options.work_dir.mkdir(parents=True, exist_ok=True) + comparison: ReleaseComparison = _compare(options=options, root=options.work_dir) + else: + with tempfile.TemporaryDirectory(prefix="sqlbuild-release-performance-") as root: + comparison = _compare(options=options, root=Path(root)) + except (ReleasePerformanceError, OSError) as error: + print(f"Release performance comparison failed: {error}", file=sys.stderr) + _ = append_summary(path=summary, markdown=error_markdown(message=str(error))) + return 1 + markdown: str = comparison_markdown(comparison=comparison) + print(markdown) + _ = append_summary(path=summary, markdown=markdown) + if options.output is not None: + _write_evidence(path=options.output, comparison=comparison) + failures: tuple[str, ...] = regression_messages(commands=comparison.commands) + if failures: + print("Release performance regression: " + "; ".join(failures), file=sys.stderr) + return 1 + print("Release performance within the limits of the previous release", file=sys.stderr) + return 0 + + +def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: + print(f"Installing candidate {options.candidate}", file=sys.stderr) + candidate: InstalledVersion = _install( + label=CANDIDATE_LABEL, + spec=options.candidate, + root=root, + python=options.python, + wait_seconds=0.0, + ) + baseline_spec: str = options.baseline or resolve_baseline( + candidate=candidate.version, repo_dir=REPO_ROOT + ) + print(f"Installing baseline {baseline_spec}", file=sys.stderr) + baseline: InstalledVersion = _install( + label=BASELINE_LABEL, + spec=baseline_spec, + root=root, + python=options.python, + wait_seconds=BASELINE_PUBLICATION_WAIT_SECONDS, + ) + print( + f"Comparing candidate {candidate.version} with baseline {baseline.version}", + file=sys.stderr, + ) + load_before: tuple[float, float, float] = os.getloadavg() + print("Generating benchmark projects", file=sys.stderr) + pristine: Path = write_pristine_projects( + root=root, inspection_models=options.inspection_models, build_models=options.build_models + ) + versions: tuple[InstalledVersion, InstalledVersion] = (baseline, candidate) + projects: dict[str, dict[str, Path]] = {} + for version in versions: + print(f"Warming caches for {version.label} {version.version}", file=sys.stderr) + projects[version.label] = prepare_version_projects( + version=version, pristine=pristine, root=root + ) + measured: list[CommandComparison] = [] + skipped: list[SkippedCommand] = [] + for command in BENCHMARK_COMMANDS: + reason: str | None = skip_reason( + command=command, baseline_version=baseline.version, candidate_version=candidate.version + ) + if reason is not None: + print(f"Skipping {command.name}: {reason}", file=sys.stderr) + skipped.append(SkippedCommand(name=command.name, reason=reason)) + continue + measured.append( + compare_command( + command=command, versions=versions, projects=projects, root=root, runs=options.runs + ) + ) + return ReleaseComparison( + baseline_version=baseline.version, + candidate_version=candidate.version, + runs=options.runs, + runner=RunnerContext( + cpu_model=cpu_model(), + cpu_count=os.cpu_count() or 0, + load_average_before=load_before, + load_average_after=os.getloadavg(), + ), + commands=tuple(measured), + skipped=tuple(skipped), + ) + + +def _install( + *, label: str, spec: str, root: Path, python: str, wait_seconds: float +) -> InstalledVersion: + path: Path = Path(spec) + venv_dir: Path = root / "venvs" / label + if path.is_dir(): + sqb: Path = existing_sqb(venv_dir=path) + elif spec.endswith(".whl"): + sqb = install_wheel(wheel=path, venv_dir=venv_dir, python=python) + else: + sqb = install_published( + version=spec, venv_dir=venv_dir, python=python, wait_seconds=wait_seconds + ) + return InstalledVersion(label=label, version=installed_version(sqb=sqb), sqb=sqb) + + +def _write_evidence(*, path: Path, comparison: ReleaseComparison) -> None: + evidence: dict[str, object] = { + **dataclasses.asdict(comparison), + "verdicts": [ + dataclasses.asdict(verdict) for verdict in all_verdicts(commands=comparison.commands) + ], + } + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(json.dumps(evidence, indent=2, default=str) + "\n", encoding="utf-8") diff --git a/scripts/release_performance/constants.py b/scripts/release_performance/constants.py new file mode 100644 index 0000000000..4880c25fb7 --- /dev/null +++ b/scripts/release_performance/constants.py @@ -0,0 +1,106 @@ +"""Stable constants for comparing a release candidate with the previous published release.""" + +from pathlib import Path + +from scripts.release_performance.models import BenchmarkCommand +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + SHARED_DIAMOND_HUB, + SHARED_DIAMOND_ROLLUP, +) + +MAX_WALL_RATIO: float = 1.25 +MAX_CPU_RATIO: float = 1.25 +MAX_RSS_RATIO: float = 1.25 +MIN_WALL_REGRESSION_SECONDS: float = 0.5 +MIN_CPU_REGRESSION_SECONDS: float = 0.5 +MIN_RSS_REGRESSION_BYTES: int = 32 * 1024 * 1024 + +DEFAULT_RUNS: int = 3 +DEFAULT_PYTHON: str = "3.12" +DEFAULT_INSPECTION_MODELS: int = 3_000 +DEFAULT_BUILD_MODELS: int = 1_000 +COMMAND_TIMEOUT_SECONDS: float = 900.0 + +PACKAGE_NAME: str = "sqlbuild" +PYPI_SIMPLE_URL: str = "https://pypi.org/simple/sqlbuild/" +PYPI_SIMPLE_JSON: str = "application/vnd.pypi.simple.v1+json" +PYPI_REQUEST_TIMEOUT_SECONDS: float = 60.0 +BASELINE_PUBLICATION_WAIT_SECONDS: float = 1_800.0 +BASELINE_PUBLICATION_POLL_SECONDS: float = 30.0 +RELEASE_TAG_PATTERN: str = "v*" +WHEEL_PLATFORM_FAMILIES: dict[str, str] = {"linux": "linux", "darwin": "macosx", "win32": "win"} +WHEEL_MACHINE_ALIASES: dict[str, str] = {"amd64": "x86_64", "arm64": "aarch64"} +UNIVERSAL_WHEEL_MACHINE: str = "universal2" +REPO_ROOT: Path = Path(__file__).resolve().parents[2] + +BASELINE_LABEL: str = "baseline" +CANDIDATE_LABEL: str = "candidate" +INSPECTION_PROJECT: str = "inspection" +BUILD_PROJECT: str = "build" +EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset({"VIRTUAL_ENV", "GITHUB_STEP_SUMMARY"}) +EXCLUDED_ENVIRONMENT_PREFIX: str = "DBT_" +TIME_FORMAT: str = "%e %M %U %S" +COMPILE_COMMAND: str = "compile" +STDERR_TAIL_CHARACTERS: int = 2_000 + +BENCHMARK_COMMANDS: tuple[BenchmarkCommand, ...] = ( + BenchmarkCommand( + name="compile (no cache)", + project=INSPECTION_PROJECT, + sqb_args=("compile", "--json", "--no-cache"), + ), + BenchmarkCommand( + name="compile (warm cache)", + project=INSPECTION_PROJECT, + sqb_args=("compile", "--json"), + ), + BenchmarkCommand( + name="plan --json", + project=INSPECTION_PROJECT, + sqb_args=("plan", "--json"), + ), + BenchmarkCommand( + name="lineage hub downstream", + project=INSPECTION_PROJECT, + sqb_args=("lineage", SHARED_DIAMOND_HUB, "--direction", "downstream"), + ), + BenchmarkCommand( + name="lineage rollup upstream", + project=INSPECTION_PROJECT, + sqb_args=("lineage", SHARED_DIAMOND_ROLLUP, "--direction", "upstream"), + ), + BenchmarkCommand( + name="lineage column trace", + project=INSPECTION_PROJECT, + sqb_args=("lineage", f"{SHARED_DIAMOND_ROLLUP}.amount", "--direction", "upstream"), + ), + BenchmarkCommand( + name="dag --json", + project=INSPECTION_PROJECT, + sqb_args=("dag", "--json"), + ), + BenchmarkCommand( + name="scope --json", + project=INSPECTION_PROJECT, + sqb_args=( + "scope", + f"model:{SHARED_DIAMOND_ROLLUP}", + "--json", + "--globals", + "all", + "--include-nearby", + ), + ), + BenchmarkCommand( + name="build (empty warehouse)", + project=BUILD_PROJECT, + sqb_args=("build",), + ), +) + +INSPECTION_WARMUPS: tuple[tuple[str, ...], ...] = ( + (COMPILE_COMMAND, "--json"), + ("lineage", SHARED_DIAMOND_HUB, "--depth", "1"), + ("scope", f"model:{SHARED_DIAMOND_ROLLUP}", "--json"), +) +BUILD_WARMUPS: tuple[tuple[str, ...], ...] = ((COMPILE_COMMAND, "--json"),) diff --git a/scripts/release_performance/exceptions.py b/scripts/release_performance/exceptions.py new file mode 100644 index 0000000000..b0386132d3 --- /dev/null +++ b/scripts/release_performance/exceptions.py @@ -0,0 +1,5 @@ +"""Release performance comparison errors.""" + + +class ReleasePerformanceError(RuntimeError): + """Raised when a compared version cannot be installed or cannot run the benchmark.""" diff --git a/scripts/release_performance/main/__init__.py b/scripts/release_performance/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/release_performance/main/compare.py b/scripts/release_performance/main/compare.py new file mode 100644 index 0000000000..73e65a04e2 --- /dev/null +++ b/scripts/release_performance/main/compare.py @@ -0,0 +1,56 @@ +"""Fail when a release candidate is materially slower or larger than the previous release.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from scripts.release_performance._helpers.workflow import run_release_comparison +from scripts.release_performance.constants import ( + DEFAULT_BUILD_MODELS, + DEFAULT_INSPECTION_MODELS, + DEFAULT_PYTHON, + DEFAULT_RUNS, +) +from scripts.release_performance.models import ComparisonOptions + + +def compare_release_performance(argv: list[str] | None = None) -> int: + """Compare the candidate with the baseline release and enforce the release limits.""" + + return run_release_comparison(options=_parse_args(argv)) + + +def _parse_args(argv: list[str] | None) -> ComparisonOptions: + parser: argparse.ArgumentParser = argparse.ArgumentParser( + description=( + "Time sqb commands for a release candidate and the previous published release on " + "this machine, and fail when the candidate is materially slower or larger. Each " + "version is a PyPI version, a wheel path, or an existing virtual environment." + ) + ) + parser.add_argument("--candidate", required=True) + parser.add_argument( + "--baseline", + default=None, + help="Defaults to the highest published or release-tagged version below the candidate.", + ) + parser.add_argument("--runs", type=int, default=DEFAULT_RUNS) + parser.add_argument("--python", default=DEFAULT_PYTHON) + parser.add_argument("--inspection-models", type=int, default=DEFAULT_INSPECTION_MODELS) + parser.add_argument("--build-models", type=int, default=DEFAULT_BUILD_MODELS) + parser.add_argument("--work-dir", type=Path, default=None) + parser.add_argument("--output", type=Path, default=None) + args: argparse.Namespace = parser.parse_args(argv) + if args.runs < 1: + parser.error("--runs must be at least 1") + return ComparisonOptions( + candidate=args.candidate, + baseline=args.baseline, + runs=args.runs, + python=args.python, + inspection_models=args.inspection_models, + build_models=args.build_models, + work_dir=args.work_dir, + output=args.output, + ) diff --git a/scripts/release_performance/models.py b/scripts/release_performance/models.py new file mode 100644 index 0000000000..2f71f3734a --- /dev/null +++ b/scripts/release_performance/models.py @@ -0,0 +1,86 @@ +"""Typed records for comparing a release candidate with the previous published release.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class BenchmarkCommand: + name: str + project: str + sqb_args: tuple[str, ...] + minimum_version: str | None = None + + +@dataclass(frozen=True) +class InstalledVersion: + label: str + version: str + sqb: Path + + +@dataclass(frozen=True) +class CommandSample: + wall_seconds: float + cpu_seconds: float + peak_rss_bytes: int + + +@dataclass(frozen=True) +class CommandComparison: + name: str + baseline: tuple[CommandSample, ...] + candidate: tuple[CommandSample, ...] + + +@dataclass(frozen=True) +class SkippedCommand: + name: str + reason: str + + +@dataclass(frozen=True) +class MetricVerdict: + command: str + metric: str + baseline: float + candidate: float + max_ratio: float + floor: float + regressed: bool + + @property + def ratio(self) -> float: + return self.candidate / self.baseline if self.baseline else float("inf") + + +@dataclass(frozen=True) +class RunnerContext: + cpu_model: str + cpu_count: int + load_average_before: tuple[float, float, float] + load_average_after: tuple[float, float, float] + + +@dataclass(frozen=True) +class ReleaseComparison: + baseline_version: str + candidate_version: str + runs: int + runner: RunnerContext + commands: tuple[CommandComparison, ...] + skipped: tuple[SkippedCommand, ...] + + +@dataclass(frozen=True) +class ComparisonOptions: + candidate: str + baseline: str | None + runs: int + python: str + inspection_models: int + build_models: int + work_dir: Path | None + output: Path | None diff --git a/tests/e2e/scripts/release_performance/__init__.py b/tests/e2e/scripts/release_performance/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/scripts/release_performance/_test_types.py b/tests/e2e/scripts/release_performance/_test_types.py new file mode 100644 index 0000000000..5c697e59ae --- /dev/null +++ b/tests/e2e/scripts/release_performance/_test_types.py @@ -0,0 +1,14 @@ +"""Test types for the release performance comparison.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ReleasePerformanceTestCase: + description: str + inspection_models: int + build_models: int + expected_return_code: int + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/scripts/release_performance/test_compare_release_performance.py b/tests/e2e/scripts/release_performance/test_compare_release_performance.py new file mode 100644 index 0000000000..1395cc1af0 --- /dev/null +++ b/tests/e2e/scripts/release_performance/test_compare_release_performance.py @@ -0,0 +1,85 @@ +"""End-to-end tests for the release performance comparison against the previous release.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from scripts.release_performance.constants import BENCHMARK_COMMANDS +from tests.e2e.scripts.release_performance._test_types import ReleasePerformanceTestCase + +_REPO_ROOT: Path = Path(__file__).resolve().parents[4] + + +@pytest.mark.parametrize( + "test_case", + ( + ReleasePerformanceTestCase( + description="the same build as candidate and baseline passes every command", + inspection_models=300, + build_models=100, + expected_return_code=0, + expected_fragments=( + "| `plan --json` |", + "| `build (empty warehouse)` |", + "| `lineage column trace` |", + "**Result: passed.**", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_two_installations_when_comparing_release_performance_then_reports_each_command( + test_case: ReleasePerformanceTestCase, tmp_path: Path +) -> None: + summary: Path = tmp_path / "summary.md" + evidence: Path = tmp_path / "evidence.json" + environment: dict[str, str] = dict(os.environ) + environment["GITHUB_STEP_SUMMARY"] = str(summary) + installation: str = str(Path(sys.executable).parent.parent) + + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + sys.executable, + "-m", + "scripts.compare_release_performance", + "--candidate", + installation, + "--baseline", + installation, + "--runs", + "1", + "--inspection-models", + str(test_case.inspection_models), + "--build-models", + str(test_case.build_models), + "--work-dir", + str(tmp_path / "work"), + "--output", + str(evidence), + ], + cwd=_REPO_ROOT, + capture_output=True, + text=True, + env=environment, + check=False, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_return_code, output + markdown: str = summary.read_text(encoding="utf-8") + for fragment in test_case.expected_fragments: + assert fragment in markdown + recorded: dict[str, object] = json.loads(evidence.read_text(encoding="utf-8")) + assert [command["name"] for command in recorded["commands"]] == [ + command.name for command in BENCHMARK_COMMANDS + ] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md index 180d6f4be7..7df260b12a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md @@ -84,3 +84,40 @@ empty warehouse once per module, with wall and peak-RSS limits. Its existing-sta built project, change the queries of a few models (and rename one table in the second case so rename discovery matches it against stored fingerprints), and bound the next `sqb plan --json`. All cases carry `models_3000` identifiers so they run only in the 3,000-model fresh-process job. + +## Release comparison with the previous release + +Fixed ceilings miss gradual creep and slowdowns that stay under a loose limit, so every release +pull request also compares the candidate with the previous published release on the same runner. +`scripts/compare_release_performance.py` installs the candidate wheel and the baseline from PyPI +into separate Python 3.12 environments, generates the inspection and build benchmarks above once, +and gives each version its own copy with compile, lineage and scope caches warmed by that version. +It then runs each command alternately for baseline and candidate, three times each, and compares +the medians of wall time, CPU time (user+sys) and peak RSS: + +- warm and uncached `sqb compile --json`, `sqb plan --json`, `sqb dag --json` and + `sqb scope --json` on the 3,000-model inspection benchmark; +- `sqb lineage` downstream from the hub, upstream from the rollup, and the upstream column trace of + `shared_orders_rollup.amount`; +- `sqb build` of the 1,000-model build benchmark from an empty warehouse, restored before each run. + +A command fails when its candidate median exceeds the baseline by more than 25% and by more than +0.5 s (wall and CPU) or 32 MiB (peak RSS); the limits live in +`scripts/release_performance/constants.py`. The baseline defaults to the highest version below +the candidate that is installable from PyPI, or release-tagged but not yet on PyPI; yanked releases +never count. A freshly tagged baseline is awaited until its wheels +are on PyPI, installing from the uncompressed simple index's wheel URL when the CDN still lags. A +command that needs a feature newer than a compared version declares `minimum_version` and appears +as skipped, with the reason, in the job summary; any other failure of either version fails the +check. + +Release Please dispatches `.github/workflows/release-performance.yml` with the metadata and version +checks and posts the required `Verify` status only after it passes, so a regressed release pull +request cannot auto-merge. Each dispatch carries a request id in its run name, and Release Please +waits only on the runs it started. After a transient failure, use **Re-run failed jobs** on the +Release Please run: it re-dispatches all three checks and repaints `Verify` on the current head. +Run the comparison manually with an optional `baseline` input, or locally: + +```bash +uv run python -m scripts.compare_release_performance --candidate 0.121.0 --baseline 0.119.1 +``` diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index b40d029121..510052ba73 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -735,21 +735,28 @@ def run_fresh_process_inspection_command( ) -def prepare_inspection_benchmark_project(*, project_dir: Path) -> None: - """Write and compile the shared-dependency benchmark, then warm lineage and scope caches.""" +def write_inspection_benchmark_project(*, project_dir: Path, model_count: int = 3_000) -> None: + """Write the semantic benchmark scaled to model_count models plus the shared lattice.""" + scale: float = model_count / 3_000 write_semantic_compile_project( project_dir=project_dir, - model_count=3_000, - source_count=713, - seed_count=141, - function_count=71, - macro_count=37, - test_count=2_945, - audit_count=5_056, + model_count=model_count, + source_count=max(1, round(713 * scale)), + seed_count=max(1, round(141 * scale)), + function_count=max(1, round(71 * scale)), + macro_count=max(1, round(37 * scale)), + test_count=max(1, round(2_945 * scale)), + audit_count=max(1, round(5_056 * scale)), shared_diamond_layers=INSPECTION_DIAMOND_LAYERS, shared_diamond_width=INSPECTION_DIAMOND_WIDTH, ) + + +def prepare_inspection_benchmark_project(*, project_dir: Path) -> None: + """Write and compile the shared-dependency benchmark, then warm lineage and scope caches.""" + + write_inspection_benchmark_project(project_dir=project_dir) compiled: InspectionCommandMeasurement = run_fresh_process_inspection_command( project_dir=project_dir, label="inspection-compile", @@ -2190,16 +2197,19 @@ class BuiltBenchmark(NamedTuple): _BUILD_BENCHMARK_EDIT: tuple[str, str] = ("amount + 0 WHEN", "amount + 100 WHEN") -def prepare_build_benchmark_project(*, project_dir: Path) -> None: - """Write a 1,000-model benchmark without tests or audits and warm its compile cache.""" +def write_build_benchmark_project( + *, project_dir: Path, model_count: int = BUILD_BENCHMARK_MODEL_COUNT +) -> None: + """Write the build benchmark without tests or audits, scaled to model_count models.""" + scale: float = model_count / BUILD_BENCHMARK_MODEL_COUNT write_semantic_compile_project( project_dir=project_dir, - model_count=BUILD_BENCHMARK_MODEL_COUNT, - source_count=238, - seed_count=47, - function_count=24, - macro_count=12, + model_count=model_count, + source_count=max(1, round(238 * scale)), + seed_count=max(1, round(47 * scale)), + function_count=max(1, round(24 * scale)), + macro_count=max(1, round(12 * scale)), test_count=0, audit_count=0, ) @@ -2207,6 +2217,12 @@ def prepare_build_benchmark_project(*, project_dir: Path) -> None: config.write_text( config.read_text(encoding="utf-8") + _BUILD_BENCHMARK_CONNECTION, encoding="utf-8" ) + + +def prepare_build_benchmark_project(*, project_dir: Path) -> None: + """Write a 1,000-model benchmark without tests or audits and warm its compile cache.""" + + write_build_benchmark_project(project_dir=project_dir) _ = run_fresh_process_inspection_command( project_dir=project_dir, label="build-benchmark-compile", diff --git a/tests/unit/scripts/release_performance/__init__.py b/tests/unit/scripts/release_performance/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/release_performance/_helpers/__init__.py b/tests/unit/scripts/release_performance/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/release_performance/_helpers/_test_types.py b/tests/unit/scripts/release_performance/_helpers/_test_types.py new file mode 100644 index 0000000000..3e0b2aa9e6 --- /dev/null +++ b/tests/unit/scripts/release_performance/_helpers/_test_types.py @@ -0,0 +1,62 @@ +"""Test-case models for the release performance comparison.""" + +from dataclasses import dataclass + +from scripts.release_performance.models import CommandComparison, SkippedCommand + + +@dataclass(frozen=True) +class MetricVerdictTestCase: + description: str + comparison: CommandComparison + expected_regressed: tuple[bool, bool, bool] + + +@dataclass(frozen=True) +class SkipReasonTestCase: + description: str + minimum_version: str | None + baseline_version: str + candidate_version: str + expected_reason: str | None + + +@dataclass(frozen=True) +class ComparisonMarkdownTestCase: + description: str + commands: tuple[CommandComparison, ...] + skipped: tuple[SkippedCommand, ...] + expected_fragments: tuple[str, ...] + unexpected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class PreviousVersionTestCase: + description: str + candidate: str + versions: tuple[str, ...] + expected_version: str | None + + +@dataclass(frozen=True) +class ReleaseVersionsTestCase: + description: str + files: tuple[dict[str, object], ...] + expected_versions: tuple[str, ...] + + +@dataclass(frozen=True) +class CompatibleWheelsTestCase: + description: str + machine: str + system: str + expected_filenames: tuple[str, ...] + + +@dataclass(frozen=True) +class ChooseBaselineTestCase: + description: str + candidate: str + files: tuple[dict[str, object], ...] + tags: tuple[str, ...] + expected_baseline: str | None diff --git a/tests/unit/scripts/release_performance/_helpers/helpers.py b/tests/unit/scripts/release_performance/_helpers/helpers.py new file mode 100644 index 0000000000..7fa7c1385f --- /dev/null +++ b/tests/unit/scripts/release_performance/_helpers/helpers.py @@ -0,0 +1,31 @@ +"""Builders for release performance comparison test data.""" + +from scripts.release_performance.models import CommandComparison, CommandSample + +MIB: int = 1024 * 1024 + + +def comparison( + *, + name: str, + baseline: tuple[tuple[float, float, int], ...], + candidate: tuple[tuple[float, float, int], ...], +) -> CommandComparison: + """Build a comparison from (wall seconds, CPU seconds, peak RSS MiB) samples.""" + + return CommandComparison( + name=name, + baseline=tuple(_sample(values=values) for values in baseline), + candidate=tuple(_sample(values=values) for values in candidate), + ) + + +def release_file(*, filename: str) -> dict[str, object]: + """Build one PEP 691 file entry that is not yanked.""" + + return {"filename": filename, "yanked": False, "url": filename} + + +def _sample(*, values: tuple[float, float, int]) -> CommandSample: + wall, cpu, rss_mib = values + return CommandSample(wall_seconds=wall, cpu_seconds=cpu, peak_rss_bytes=rss_mib * MIB) diff --git a/tests/unit/scripts/release_performance/_helpers/test_report.py b/tests/unit/scripts/release_performance/_helpers/test_report.py new file mode 100644 index 0000000000..6a0cfc8b7f --- /dev/null +++ b/tests/unit/scripts/release_performance/_helpers/test_report.py @@ -0,0 +1,105 @@ +"""Tests for the release performance job summary.""" + +import pytest + +from scripts.release_performance._helpers.report import comparison_markdown +from scripts.release_performance.models import ( + ReleaseComparison, + RunnerContext, + SkippedCommand, +) +from tests.unit.scripts.release_performance._helpers._test_types import ( + ComparisonMarkdownTestCase, +) +from tests.unit.scripts.release_performance._helpers.helpers import comparison + + +@pytest.mark.parametrize( + "test_case", + ( + ComparisonMarkdownTestCase( + description="a passing comparison lists every command and the limits", + commands=( + comparison( + name="compile (warm cache)", + baseline=((6.0, 6.5, 500),) * 3, + candidate=((6.1, 6.4, 505),) * 3, + ), + ), + skipped=(), + expected_fragments=( + "### Release performance: 0.126.2 (candidate) vs 0.126.1 (baseline)", + "Same runner: Example CPU, 4 CPUs; load average 0.50/0.40/0.30 before", + "3 interleaved runs per version", + "Limits: wall +25% (ignored under 0.5 s), CPU +25% (ignored under 0.5 s), " + "peak RSS +25% (ignored under 32 MiB).", + "| `compile (warm cache)` | 6.00 → 6.10 | 1.017x | 6.50 → 6.40 | 0.985x " + "| 500 → 505 | 1.010x | ✅ ok |", + "**Result: passed.**", + ), + unexpected_fragments=("❌", "skipped"), + ), + ComparisonMarkdownTestCase( + description="a regression is bolded and named in the failed result", + commands=( + comparison( + name="plan --json", + baseline=((27.0, 28.0, 900),) * 3, + candidate=((56.0, 57.0, 1_800),) * 3, + ), + ), + skipped=(), + expected_fragments=( + "| `plan --json` | 27.00 → 56.00 | **2.074x** | 28.00 → 57.00 | **2.036x** " + "| 900 → 1800 | **2.000x** | ❌ regressed |", + "**Result: failed.** plan --json: wall 2.074x exceeds 1.25x; plan --json: CPU " + "2.036x exceeds 1.25x; plan --json: peak RSS 2.000x exceeds 1.25x.", + ), + unexpected_fragments=("**Result: passed.**",), + ), + ComparisonMarkdownTestCase( + description="a skipped command is reported with its reason instead of omitted", + commands=(), + skipped=( + SkippedCommand( + name="scope --json", + reason="requires sqlbuild 0.126.2 or later; baseline is 0.126.1", + ), + ), + expected_fragments=( + "| `scope --json` | – | – | – | – | – | – | ⚠️ skipped: requires sqlbuild " + "0.126.2 or later; baseline is 0.126.1 |", + "**1 command(s) skipped** because a compared version predates them", + ), + unexpected_fragments=("❌",), + ), + ), + ids=lambda case: case.description, +) +def test_given_release_comparison_when_rendering_summary_then_reports_each_command( + test_case: ComparisonMarkdownTestCase, +) -> None: + release: ReleaseComparison = ReleaseComparison( + baseline_version="0.126.1", + candidate_version="0.126.2", + runs=3, + runner=RunnerContext( + cpu_model="Example CPU", + cpu_count=4, + load_average_before=(0.5, 0.4, 0.3), + load_average_after=(0.6, 0.5, 0.4), + ), + commands=test_case.commands, + skipped=test_case.skipped, + ) + + markdown: str = comparison_markdown(comparison=release) + + for fragment in test_case.expected_fragments: + assert fragment in markdown + for fragment in test_case.unexpected_fragments: + assert fragment not in markdown + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/release_performance/_helpers/test_verdict.py b/tests/unit/scripts/release_performance/_helpers/test_verdict.py new file mode 100644 index 0000000000..85a0d051ca --- /dev/null +++ b/tests/unit/scripts/release_performance/_helpers/test_verdict.py @@ -0,0 +1,138 @@ +"""Tests for the release performance regression verdict.""" + +import pytest + +from scripts.release_performance._helpers.verdict import ( + metric_verdicts, + regression_messages, + skip_reason, +) +from scripts.release_performance.models import BenchmarkCommand, MetricVerdict +from tests.unit.scripts.release_performance._helpers._test_types import ( + MetricVerdictTestCase, + SkipReasonTestCase, +) +from tests.unit.scripts.release_performance._helpers.helpers import comparison + + +@pytest.mark.parametrize( + "test_case", + ( + MetricVerdictTestCase( + description="identical medians pass", + comparison=comparison( + name="dag --json", + baseline=((9.0, 9.0, 500),) * 3, + candidate=((9.0, 9.0, 500),) * 3, + ), + expected_regressed=(False, False, False), + ), + MetricVerdictTestCase( + description="slower planning with more peak memory fails CPU and RSS below wall limit", + comparison=comparison( + name="plan --json", + baseline=((122.7, 125.6, 973),) * 3, + candidate=((150.3, 157.3, 1_742),) * 3, + ), + expected_regressed=(False, True, True), + ), + MetricVerdictTestCase( + description="a sub-second command doubling stays under the absolute floor", + comparison=comparison( + name="scope --json", + baseline=((0.3, 0.3, 70),) * 3, + candidate=((0.7, 0.7, 70),) * 3, + ), + expected_regressed=(False, False, False), + ), + MetricVerdictTestCase( + description="a small allocation growth stays under the memory floor", + comparison=comparison( + name="lineage hub downstream", + baseline=((2.0, 2.0, 40),) * 3, + candidate=((2.0, 2.0, 70),) * 3, + ), + expected_regressed=(False, False, False), + ), + MetricVerdictTestCase( + description="exactly the ratio limit passes and just above it fails", + comparison=comparison( + name="compile (warm cache)", + baseline=((10.0, 10.0, 400),) * 3, + candidate=((12.5, 12.6, 400),) * 3, + ), + expected_regressed=(False, True, False), + ), + MetricVerdictTestCase( + description="one slow outlier does not move the median", + comparison=comparison( + name="build (empty warehouse)", + baseline=((60.0, 60.0, 700),) * 3, + candidate=((60.0, 60.0, 700), (95.0, 95.0, 1_400), (61.0, 61.0, 710)), + ), + expected_regressed=(False, False, False), + ), + ), + ids=lambda case: case.description, +) +def test_given_command_samples_when_judging_then_flags_only_material_regressions( + test_case: MetricVerdictTestCase, +) -> None: + verdicts: tuple[MetricVerdict, ...] = metric_verdicts(comparison=test_case.comparison) + + assert tuple(verdict.metric for verdict in verdicts) == ("wall", "CPU", "peak RSS") + assert tuple(verdict.regressed for verdict in verdicts) == test_case.expected_regressed + messages: tuple[str, ...] = regression_messages(commands=(test_case.comparison,)) + assert len(messages) == sum(test_case.expected_regressed) + for message in messages: + assert message.startswith(f"{test_case.comparison.name}: ") + + +@pytest.mark.parametrize( + "test_case", + ( + SkipReasonTestCase( + description="a command without a minimum version always runs", + minimum_version=None, + baseline_version="0.119.1", + candidate_version="0.121.0", + expected_reason=None, + ), + SkipReasonTestCase( + description="a command newer than the baseline is skipped with the baseline named", + minimum_version="0.126.0", + baseline_version="0.125.1", + candidate_version="0.126.0", + expected_reason="requires sqlbuild 0.126.0 or later; baseline is 0.125.1", + ), + SkipReasonTestCase( + description="versions compare numerically rather than as text", + minimum_version="0.99.0", + baseline_version="0.126.1", + candidate_version="0.126.2", + expected_reason=None, + ), + ), + ids=lambda case: case.description, +) +def test_given_command_minimum_version_when_checking_versions_then_explains_skips( + test_case: SkipReasonTestCase, +) -> None: + command: BenchmarkCommand = BenchmarkCommand( + name="dag --json", + project="inspection", + sqb_args=("dag", "--json"), + minimum_version=test_case.minimum_version, + ) + + reason: str | None = skip_reason( + command=command, + baseline_version=test_case.baseline_version, + candidate_version=test_case.candidate_version, + ) + + assert reason == test_case.expected_reason + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/release_performance/_helpers/test_versions.py b/tests/unit/scripts/release_performance/_helpers/test_versions.py new file mode 100644 index 0000000000..0aae6d9e73 --- /dev/null +++ b/tests/unit/scripts/release_performance/_helpers/test_versions.py @@ -0,0 +1,184 @@ +"""Tests for resolving the published baseline version.""" + +import pytest + +from scripts.release_performance._helpers.versions import ( + choose_baseline, + compatible_wheels, + previous_version, + release_versions, +) +from tests.unit.scripts.release_performance._helpers._test_types import ( + ChooseBaselineTestCase, + CompatibleWheelsTestCase, + PreviousVersionTestCase, + ReleaseVersionsTestCase, +) +from tests.unit.scripts.release_performance._helpers.helpers import release_file + +_RELEASE_FILES: tuple[str, ...] = ( + "sqlbuild-0.126.1-cp312-abi3-macosx_10_12_x86_64.whl", + "sqlbuild-0.126.1-cp312-abi3-macosx_11_0_arm64.whl", + "sqlbuild-0.126.1-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "sqlbuild-0.126.1-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "sqlbuild-0.126.1-cp312-abi3-win_amd64.whl", + "sqlbuild-0.126.1.tar.gz", + "sqlbuild-0.126.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", +) + + +@pytest.mark.parametrize( + "test_case", + ( + PreviousVersionTestCase( + description="the highest version below a release candidate", + candidate="0.126.3", + versions=("0.126.1", "0.126.2", "0.125.9", "0.126.2"), + expected_version="0.126.2", + ), + PreviousVersionTestCase( + description="an already published candidate compares with the release before it", + candidate="0.126.2", + versions=("0.126.1", "0.126.2"), + expected_version="0.126.1", + ), + PreviousVersionTestCase( + description="versions order numerically rather than as text", + candidate="0.100.0", + versions=("0.99.0", "0.9.9", "0.100.0"), + expected_version="0.99.0", + ), + PreviousVersionTestCase( + description="no earlier release", + candidate="0.1.0", + versions=("0.1.0",), + expected_version=None, + ), + ), + ids=lambda case: case.description, +) +def test_given_known_versions_when_choosing_baseline_then_returns_previous_release( + test_case: PreviousVersionTestCase, +) -> None: + assert ( + previous_version(candidate=test_case.candidate, versions=test_case.versions) + == test_case.expected_version + ) + + +@pytest.mark.parametrize( + "test_case", + ( + ReleaseVersionsTestCase( + description="wheels and sdists of every release", + files=tuple(release_file(filename=name) for name in _RELEASE_FILES), + expected_versions=("0.126.0", "0.126.1"), + ), + ReleaseVersionsTestCase( + description="a yanked release and a pre-release are not baselines", + files=( + {"filename": "sqlbuild-0.126.0.tar.gz", "yanked": False}, + {"filename": "sqlbuild-0.126.1.tar.gz", "yanked": "broken wheel"}, + {"filename": "sqlbuild-0.127.0rc1.tar.gz", "yanked": False}, + ), + expected_versions=("0.126.0",), + ), + ), + ids=lambda case: case.description, +) +def test_given_simple_index_when_listing_versions_then_returns_installable_releases( + test_case: ReleaseVersionsTestCase, +) -> None: + assert release_versions(index={"files": list(test_case.files)}) == test_case.expected_versions + + +@pytest.mark.parametrize( + "test_case", + ( + CompatibleWheelsTestCase( + description="linux x86_64 runner", + machine="x86_64", + system="linux", + expected_filenames=( + "sqlbuild-0.126.1-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + ), + ), + CompatibleWheelsTestCase( + description="apple silicon", + machine="arm64", + system="darwin", + expected_filenames=("sqlbuild-0.126.1-cp312-abi3-macosx_11_0_arm64.whl",), + ), + CompatibleWheelsTestCase( + description="unsupported platform", + machine="riscv64", + system="linux", + expected_filenames=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_release_files_when_selecting_wheel_urls_then_matches_runner_platform( + test_case: CompatibleWheelsTestCase, +) -> None: + index: dict[str, object] = {"files": [release_file(filename=name) for name in _RELEASE_FILES]} + + wheels: tuple[dict[str, object], ...] = compatible_wheels( + index=index, version="0.126.1", machine=test_case.machine, system=test_case.system + ) + + assert tuple(entry["filename"] for entry in wheels) == test_case.expected_filenames + + +@pytest.mark.parametrize( + "test_case", + ( + ChooseBaselineTestCase( + description="a tagged release that is still publishing is the baseline", + candidate="0.126.3", + files=(release_file(filename="sqlbuild-0.126.1.tar.gz"),), + tags=("0.126.1", "0.126.2"), + expected_baseline="0.126.2", + ), + ChooseBaselineTestCase( + description="a yanked release is skipped even though its tag exists", + candidate="0.126.3", + files=( + release_file(filename="sqlbuild-0.126.1.tar.gz"), + {"filename": "sqlbuild-0.126.2.tar.gz", "yanked": "broken build", "url": ""}, + ), + tags=("0.126.1", "0.126.2"), + expected_baseline="0.126.1", + ), + ChooseBaselineTestCase( + description="a published release without a local tag is still a baseline", + candidate="0.126.3", + files=( + release_file(filename="sqlbuild-0.126.1.tar.gz"), + release_file(filename="sqlbuild-0.126.2.tar.gz"), + ), + tags=(), + expected_baseline="0.126.2", + ), + ChooseBaselineTestCase( + description="nothing precedes the first release", + candidate="0.1.0", + files=(release_file(filename="sqlbuild-0.1.0.tar.gz"),), + tags=("0.1.0",), + expected_baseline=None, + ), + ), + ids=lambda case: case.description, +) +def test_given_index_and_release_tags_when_choosing_baseline_then_skips_yanked_releases( + test_case: ChooseBaselineTestCase, +) -> None: + baseline: str | None = choose_baseline( + candidate=test_case.candidate, index={"files": list(test_case.files)}, tags=test_case.tags + ) + + assert baseline == test_case.expected_baseline + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"])