From 9524a22cfe94fe8245120ec461dd0d142c797428 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 29 Sep 2026 06:30:53 +0100 Subject: [PATCH 1/3] ci: compare release candidates with the previous release Release Please now dispatches a same-runner comparison of the candidate wheel with the previous published release and posts the required Verify status only after it passes. Wall time, CPU time and peak RSS medians of compile, plan, lineage, dag, scope and build on the generated benchmark fail above +25% and the absolute floors. --- .github/workflows/release-performance.yml | 72 +++++ .github/workflows/release-please.yml | 17 +- Makefile | 1 + scripts/compare_release_performance.py | 15 + .../_helpers/report.py | 6 +- scripts/release_performance/__init__.py | 0 .../release_performance/_helpers/__init__.py | 0 .../release_performance/_helpers/benchmark.py | 193 ++++++++++++ .../release_performance/_helpers/report.py | 94 ++++++ .../release_performance/_helpers/verdict.py | 122 ++++++++ .../release_performance/_helpers/versions.py | 286 ++++++++++++++++++ .../release_performance/_helpers/workflow.py | 172 +++++++++++ scripts/release_performance/constants.py | 106 +++++++ scripts/release_performance/exceptions.py | 5 + scripts/release_performance/main/__init__.py | 0 scripts/release_performance/main/compare.py | 56 ++++ scripts/release_performance/models.py | 86 ++++++ .../scripts/release_performance/__init__.py | 0 .../release_performance/_test_types.py | 14 + .../test_compare_release_performance.py | 85 ++++++ .../main/compile/COMPILE_BENCHMARKS.md | 33 ++ .../cli/commands/main/compile/helpers.py | 48 ++- .../scripts/release_performance/__init__.py | 0 .../release_performance/_helpers/__init__.py | 0 .../_helpers/_test_types.py | 53 ++++ .../release_performance/_helpers/helpers.py | 31 ++ .../_helpers/test_report.py | 105 +++++++ .../_helpers/test_verdict.py | 138 +++++++++ .../_helpers/test_versions.py | 132 ++++++++ 29 files changed, 1848 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/release-performance.yml create mode 100644 scripts/compare_release_performance.py create mode 100644 scripts/release_performance/__init__.py create mode 100644 scripts/release_performance/_helpers/__init__.py create mode 100644 scripts/release_performance/_helpers/benchmark.py create mode 100644 scripts/release_performance/_helpers/report.py create mode 100644 scripts/release_performance/_helpers/verdict.py create mode 100644 scripts/release_performance/_helpers/versions.py create mode 100644 scripts/release_performance/_helpers/workflow.py create mode 100644 scripts/release_performance/constants.py create mode 100644 scripts/release_performance/exceptions.py create mode 100644 scripts/release_performance/main/__init__.py create mode 100644 scripts/release_performance/main/compare.py create mode 100644 scripts/release_performance/models.py create mode 100644 tests/e2e/scripts/release_performance/__init__.py create mode 100644 tests/e2e/scripts/release_performance/_test_types.py create mode 100644 tests/e2e/scripts/release_performance/test_compare_release_performance.py create mode 100644 tests/unit/scripts/release_performance/__init__.py create mode 100644 tests/unit/scripts/release_performance/_helpers/__init__.py create mode 100644 tests/unit/scripts/release_performance/_helpers/_test_types.py create mode 100644 tests/unit/scripts/release_performance/_helpers/helpers.py create mode 100644 tests/unit/scripts/release_performance/_helpers/test_report.py create mode 100644 tests/unit/scripts/release_performance/_helpers/test_verdict.py create mode 100644 tests/unit/scripts/release_performance/_helpers/test_versions.py diff --git a/.github/workflows/release-performance.yml b/.github/workflows/release-performance.yml new file mode 100644 index 0000000000..d3314d26cb --- /dev/null +++ b/.github/workflows/release-performance.yml @@ -0,0 +1,72 @@ +name: Release Performance + +# Release Please dispatches this workflow for each release pull request and reports its result +# through the required Verify status. Pull requests that change the comparison itself run it too. +on: + workflow_dispatch: + inputs: + baseline: + description: "Published sqlbuild version to compare against (default: the release before the candidate)" + required: false + type: string + pull_request: + paths: + - .github/workflows/release-performance.yml + - scripts/compare_release_performance.py + - scripts/release_performance/** + +permissions: + contents: read + +concurrency: + group: release-performance-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + compare: + name: Compare with the previous release + runs-on: ubuntu-24.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + with: + # Release tags identify the previous release before its wheels reach PyPI. + fetch-depth: 0 + - uses: astral-sh/setup-uv@v5 + with: + version: "0.12.3" + enable-cache: true + cache-dependency-glob: | + uv.lock + Cargo.lock + - uses: dtolnay/rust-toolchain@master + with: + toolchain: stable + - name: Install Python + run: uv python install 3.12 + - name: Install dependencies + run: uv sync --locked --all-groups --all-extras + - name: Build candidate wheel + run: uv build --wheel --out-dir "${RUNNER_TEMP}/candidate" + - name: Compare candidate and previous release on this runner + env: + BASELINE: ${{ inputs.baseline }} + run: | + wheels=("${RUNNER_TEMP}"/candidate/sqlbuild-*.whl) + args=( + --candidate "${wheels[0]}" + --work-dir "${RUNNER_TEMP}/release-performance" + --output "${RUNNER_TEMP}/release-performance.json" + ) + if [[ -n "${BASELINE}" ]]; then + args+=(--baseline "${BASELINE}") + fi + uv run python -m scripts.compare_release_performance "${args[@]}" + - name: Upload comparison evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: release-performance-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ runner.temp }}/release-performance.json + if-no-files-found: warn + retention-days: 90 diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index 6e3c5a8890..dd98539b8f 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -113,8 +113,10 @@ jobs: fi head_sha="$(git rev-parse HEAD)" - workflows=(pull-request-metadata.yml version-guard.yml) - contexts=("Pull Request Metadata" "Block automatic 1.0 releases") + # Verify is the required status, so it only succeeds after every trusted check passes, + # including the same-runner comparison with the previous release. + workflows=(pull-request-metadata.yml version-guard.yml release-performance.yml) + contexts=("Pull Request Metadata" "Block automatic 1.0 releases" "Release performance") for context in "${contexts[@]}"; do gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ -f state=pending -f "context=${context}" \ @@ -122,11 +124,12 @@ jobs: done gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ -f state=pending -f context=Verify \ - -f description="Validating trusted release metadata" >/dev/null + -f description="Validating trusted release metadata and performance" >/dev/null gh workflow run pull-request-metadata.yml --ref "${RELEASE_BRANCH}" \ -f "pr_number=${release_pr}" gh workflow run version-guard.yml --ref "${RELEASE_BRANCH}" + gh workflow run release-performance.yml --ref "${RELEASE_BRANCH}" for index in "${!workflows[@]}"; do workflow="${workflows[${index}]}" context="${contexts[${index}]}" @@ -142,10 +145,13 @@ jobs: gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ -f state=failure -f "context=${context}" \ -f description="Trusted release check did not start" >/dev/null + gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ + -f state=failure -f context=Verify \ + -f description="${context} did not start" >/dev/null exit 1 fi run_url="$(gh run view "${run_id}" --json url --jq .url)" - if gh run watch "${run_id}" --exit-status; then + if gh run watch "${run_id}" --exit-status --interval 30; then gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ -f state=success -f "context=${context}" -f target_url="${run_url}" \ -f description="Trusted release check passed" >/dev/null @@ -153,6 +159,9 @@ jobs: gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ -f state=failure -f "context=${context}" -f target_url="${run_url}" \ -f description="Trusted release check failed" >/dev/null + gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ + -f state=failure -f context=Verify -f target_url="${run_url}" \ + -f description="${context} failed" >/dev/null exit 1 fi done diff --git a/Makefile b/Makefile index e0dcabcf55..5e1dfc37ec 100644 --- a/Makefile +++ b/Makefile @@ -121,6 +121,7 @@ E2E_DUCKDB_CLI_DATA_PATHS := \ E2E_DUCKDB_CLI_PATHS := \ tests/e2e/scripts/cli_preview \ tests/e2e/scripts/compile_performance_ratio \ + tests/e2e/scripts/release_performance \ tests/e2e/src/sqlbuild/cli/commands/main/adapters \ tests/e2e/src/sqlbuild/cli/commands/main/audit \ tests/e2e/src/sqlbuild/cli/commands/main/bigquery \ diff --git a/scripts/compare_release_performance.py b/scripts/compare_release_performance.py new file mode 100644 index 0000000000..e6b3ec1daa --- /dev/null +++ b/scripts/compare_release_performance.py @@ -0,0 +1,15 @@ +"""Direct wrapper for comparing a release candidate with the previous published release.""" + +from __future__ import annotations + +from scripts.release_performance.main.compare import compare_release_performance + + +def main(argv: list[str] | None = None) -> int: + """Compare candidate and baseline command performance on the same machine.""" + + return compare_release_performance(argv) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/compile_performance_ratio/_helpers/report.py b/scripts/compile_performance_ratio/_helpers/report.py index 586f218f68..3df30ab101 100644 --- a/scripts/compile_performance_ratio/_helpers/report.py +++ b/scripts/compile_performance_ratio/_helpers/report.py @@ -15,7 +15,7 @@ def comparison_markdown(*, comparison: CompileComparison, runs: int, max_ratio: lines: list[str] = [ f"### {comparison.kind} {comparison.models} models: head vs base (same runner)", "", - f"Runner CPU: {_cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.", + f"Runner CPU: {cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.", "", "| Metric | Base | Head | Ratio |", "|---|---:|---:|---:|", @@ -41,7 +41,9 @@ def append_summary(*, path: Path | None, markdown: str) -> None: summary.write(markdown + "\n") -def _cpu_model() -> str: +def cpu_model() -> str: + """Return the runner CPU model name.""" + cpuinfo: Path = Path("/proc/cpuinfo") if cpuinfo.exists(): for line in cpuinfo.read_text(encoding="utf-8").splitlines(): diff --git a/scripts/release_performance/__init__.py b/scripts/release_performance/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/release_performance/_helpers/__init__.py b/scripts/release_performance/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/release_performance/_helpers/benchmark.py b/scripts/release_performance/_helpers/benchmark.py new file mode 100644 index 0000000000..7dfd8cbb53 --- /dev/null +++ b/scripts/release_performance/_helpers/benchmark.py @@ -0,0 +1,193 @@ +"""Prepare identical benchmark projects per version and time interleaved command runs.""" + +from __future__ import annotations + +import json +import os +import shutil +import signal +import subprocess +import sys +from pathlib import Path +from typing import cast + +from scripts.release_performance.constants import ( + BUILD_PROJECT, + BUILD_WARMUPS, + COMMAND_TIMEOUT_SECONDS, + COMPILE_COMMAND, + EXCLUDED_ENVIRONMENT_KEYS, + EXCLUDED_ENVIRONMENT_PREFIX, + INSPECTION_PROJECT, + INSPECTION_WARMUPS, + STDERR_TAIL_CHARACTERS, + TIME_FORMAT, +) +from scripts.release_performance.exceptions import ReleasePerformanceError +from scripts.release_performance.models import ( + BenchmarkCommand, + CommandComparison, + CommandSample, + InstalledVersion, +) +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + write_build_benchmark_project, + write_inspection_benchmark_project, +) + +_MIB: int = 1024 * 1024 +_TEMPLATE_SUFFIX: str = "-template" + + +def write_pristine_projects(*, root: Path, inspection_models: int, build_models: int) -> Path: + """Generate the inspection and build benchmarks once so every version reads the same files.""" + + pristine: Path = root / "pristine" + write_inspection_benchmark_project( + project_dir=pristine / INSPECTION_PROJECT, model_count=inspection_models + ) + write_build_benchmark_project(project_dir=pristine / BUILD_PROJECT, model_count=build_models) + return pristine + + +def prepare_version_projects( + *, version: InstalledVersion, pristine: Path, root: Path +) -> dict[str, Path]: + """Copy the pristine projects for one version and warm its caches with untimed runs.""" + + projects: dict[str, Path] = { + INSPECTION_PROJECT: root / version.label / INSPECTION_PROJECT, + BUILD_PROJECT: root / version.label / BUILD_PROJECT, + } + logs: Path = root / version.label / "logs" + logs.mkdir(parents=True, exist_ok=True) + for name, warmups in ((INSPECTION_PROJECT, INSPECTION_WARMUPS), (BUILD_PROJECT, BUILD_WARMUPS)): + _ = shutil.copytree(pristine / name, projects[name]) + for index, sqb_args in enumerate(warmups): + output: Path = _run_sqb( + version=version, + project_dir=projects[name], + sqb_args=sqb_args, + log_prefix=logs / f"warmup-{name}-{index}", + )[1] + if sqb_args[0] == COMPILE_COMMAND: + _assert_clean_compile(version=version, output=output) + _ = shutil.copytree( + projects[BUILD_PROJECT], projects[BUILD_PROJECT].with_name(BUILD_PROJECT + _TEMPLATE_SUFFIX) + ) + return projects + + +def compare_command( + *, + command: BenchmarkCommand, + versions: tuple[InstalledVersion, InstalledVersion], + projects: dict[str, dict[str, Path]], + root: Path, + runs: int, +) -> CommandComparison: + """Time baseline and candidate alternately runs times each and keep every sample.""" + + samples: dict[str, list[CommandSample]] = {version.label: [] for version in versions} + for run in range(runs): + for version in versions: + project_dir: Path = projects[version.label][command.project] + if command.project == BUILD_PROJECT: + _reset_build_project(project_dir=project_dir) + sample: CommandSample = _run_sqb( + version=version, + project_dir=project_dir, + sqb_args=command.sqb_args, + log_prefix=root / version.label / "logs" / "measured", + )[0] + samples[version.label].append(sample) + print( + f"{command.name} run {run + 1}/{runs} {version.label} {version.version}: " + f"wall {sample.wall_seconds:.2f}s, CPU {sample.cpu_seconds:.2f}s, " + f"peak RSS {sample.peak_rss_bytes / _MIB:.0f} MiB", + file=sys.stderr, + flush=True, + ) + baseline, candidate = versions + return CommandComparison( + name=command.name, + baseline=tuple(samples[baseline.label]), + candidate=tuple(samples[candidate.label]), + ) + + +def _reset_build_project(*, project_dir: Path) -> None: + shutil.rmtree(project_dir, ignore_errors=True) + _ = shutil.copytree(project_dir.with_name(BUILD_PROJECT + _TEMPLATE_SUFFIX), project_dir) + + +def _assert_clean_compile(*, version: InstalledVersion, output: Path) -> None: + payload: dict[str, object] = cast(dict[str, object], json.loads(output.read_bytes())) + diagnostics: list[object] = cast(list[object], payload.get("diagnostics", [])) + if diagnostics: + raise ReleasePerformanceError( + f"The benchmark is not valid for {version.label} {version.version}: compile reported " + f"{len(diagnostics)} diagnostics, first {json.dumps(diagnostics[0])[:500]}" + ) + + +def _run_sqb( + *, + version: InstalledVersion, + project_dir: Path, + sqb_args: tuple[str, ...], + log_prefix: Path, +) -> tuple[CommandSample, Path]: + measurement: Path = log_prefix.with_suffix(".time") + output: Path = log_prefix.with_suffix(".out") + errors: Path = log_prefix.with_suffix(".err") + command: list[str] = [ + "/usr/bin/time", + "--quiet", + "--output", + str(measurement), + "--format", + TIME_FORMAT, + str(version.sqb), + "--project-dir", + str(project_dir), + "--no-color", + *sqb_args, + ] + with output.open("wb") as stdout, errors.open("wb") as stderr: + with subprocess.Popen( + command, stdout=stdout, stderr=stderr, env=_environment(), start_new_session=True + ) as process: + try: + returncode: int = process.wait(timeout=COMMAND_TIMEOUT_SECONDS) + except subprocess.TimeoutExpired: + os.killpg(process.pid, signal.SIGKILL) + _ = process.wait() + raise ReleasePerformanceError( + f"{version.label} {version.version} `sqb {' '.join(sqb_args)}` exceeded " + f"{COMMAND_TIMEOUT_SECONDS:.0f}s" + ) from None + if returncode != 0: + tail: str = errors.read_text(encoding="utf-8", errors="replace")[-STDERR_TAIL_CHARACTERS:] + raise ReleasePerformanceError( + f"{version.label} {version.version} `sqb {' '.join(sqb_args)}` exited with " + f"{returncode}. If the command needs a feature newer than this version, declare " + f"minimum_version for it in scripts/release_performance/constants.py.\n{tail}" + ) + wall, rss_kib, user, system = measurement.read_text(encoding="utf-8").split()[-4:] + return ( + CommandSample( + wall_seconds=float(wall), + cpu_seconds=float(user) + float(system), + peak_rss_bytes=int(rss_kib) * 1024, + ), + output, + ) + + +def _environment() -> dict[str, str]: + return { + key: value + for key, value in os.environ.items() + if key not in EXCLUDED_ENVIRONMENT_KEYS and not key.startswith(EXCLUDED_ENVIRONMENT_PREFIX) + } diff --git a/scripts/release_performance/_helpers/report.py b/scripts/release_performance/_helpers/report.py new file mode 100644 index 0000000000..43eed207c9 --- /dev/null +++ b/scripts/release_performance/_helpers/report.py @@ -0,0 +1,94 @@ +"""Render a release performance comparison for logs and the CI job summary.""" + +from __future__ import annotations + +from pathlib import Path + +from scripts.release_performance._helpers.verdict import metric_verdicts, regression_messages +from scripts.release_performance.constants import ( + MAX_CPU_RATIO, + MAX_RSS_RATIO, + MAX_WALL_RATIO, + MIN_CPU_REGRESSION_SECONDS, + MIN_RSS_REGRESSION_BYTES, + MIN_WALL_REGRESSION_SECONDS, +) +from scripts.release_performance.models import MetricVerdict, ReleaseComparison + +_MIB: int = 1024 * 1024 + + +def comparison_markdown(*, comparison: ReleaseComparison) -> str: + """Summarize per-command medians, ratios and the overall verdict as Markdown.""" + + lines: list[str] = [ + f"### Release performance: {comparison.candidate_version} (candidate) vs " + f"{comparison.baseline_version} (baseline)", + "", + f"Same runner: {comparison.runner.cpu_model}, {comparison.runner.cpu_count} CPUs; " + f"load average {_load(comparison.runner.load_average_before)} before and " + f"{_load(comparison.runner.load_average_after)} after. " + f"{comparison.runs} interleaved runs per version; medians shown.", + "", + f"Limits: wall {_percent(MAX_WALL_RATIO)} (ignored under " + f"{MIN_WALL_REGRESSION_SECONDS:g} s), CPU {_percent(MAX_CPU_RATIO)} (ignored under " + f"{MIN_CPU_REGRESSION_SECONDS:g} s), peak RSS {_percent(MAX_RSS_RATIO)} (ignored under " + f"{MIN_RSS_REGRESSION_BYTES // _MIB} MiB).", + "", + "| Command | Wall (s) | Wall ratio | CPU (s) | CPU ratio | Peak RSS (MiB) | RSS ratio " + "| Result |", + "|---|---:|---:|---:|---:|---:|---:|---|", + ] + for command in comparison.commands: + wall, cpu, rss = metric_verdicts(comparison=command) + regressed: bool = wall.regressed or cpu.regressed or rss.regressed + lines.append( + f"| `{command.name}` " + f"| {wall.baseline:.2f} → {wall.candidate:.2f} | {_ratio(wall)} " + f"| {cpu.baseline:.2f} → {cpu.candidate:.2f} | {_ratio(cpu)} " + f"| {rss.baseline / _MIB:.0f} → {rss.candidate / _MIB:.0f} | {_ratio(rss)} " + f"| {'❌ regressed' if regressed else '✅ ok'} |" + ) + for skipped in comparison.skipped: + lines.append(f"| `{skipped.name}` | – | – | – | – | – | – | ⚠️ skipped: {skipped.reason} |") + lines.append("") + failures: tuple[str, ...] = regression_messages(commands=comparison.commands) + if failures: + lines.append("**Result: failed.** " + "; ".join(failures) + ".") + else: + lines.append("**Result: passed.** No command exceeded the release limits.") + if comparison.skipped: + lines.append("") + lines.append( + f"**{len(comparison.skipped)} command(s) skipped** because a compared version " + "predates them; they were not measured." + ) + return "\n".join(lines) + "\n" + + +def error_markdown(*, message: str) -> str: + """Report a comparison that could not complete.""" + + return f"### Release performance\n\n**Result: error.** {message}\n" + + +def append_summary(*, path: Path | None, markdown: str) -> None: + """Append to the CI job summary when one is configured.""" + + if path is None: + return + with path.open("a", encoding="utf-8") as summary: + _ = summary.write(markdown + "\n") + + +def _ratio(verdict: MetricVerdict) -> str: + text: str = f"{verdict.ratio:.3f}x" + return f"**{text}**" if verdict.regressed else text + + +def _percent(ratio: float) -> str: + return f"+{round((ratio - 1) * 100)}%" + + +def _load(values: tuple[float, float, float]) -> str: + return "/".join(f"{value:.2f}" for value in values) diff --git a/scripts/release_performance/_helpers/verdict.py b/scripts/release_performance/_helpers/verdict.py new file mode 100644 index 0000000000..d688ac2451 --- /dev/null +++ b/scripts/release_performance/_helpers/verdict.py @@ -0,0 +1,122 @@ +"""Decide whether the candidate regressed against the baseline beyond the release limits.""" + +from __future__ import annotations + +import statistics + +from scripts.release_performance.constants import ( + MAX_CPU_RATIO, + MAX_RSS_RATIO, + MAX_WALL_RATIO, + MIN_CPU_REGRESSION_SECONDS, + MIN_RSS_REGRESSION_BYTES, + MIN_WALL_REGRESSION_SECONDS, +) +from scripts.release_performance.models import ( + BenchmarkCommand, + CommandComparison, + CommandSample, + MetricVerdict, +) + + +def metric_verdicts(*, comparison: CommandComparison) -> tuple[MetricVerdict, ...]: + """Compare median wall time, CPU time and peak RSS of one command.""" + + baseline: CommandSample = median_sample(samples=comparison.baseline) + candidate: CommandSample = median_sample(samples=comparison.candidate) + return ( + _verdict( + command=comparison.name, + metric="wall", + baseline=baseline.wall_seconds, + candidate=candidate.wall_seconds, + max_ratio=MAX_WALL_RATIO, + floor=MIN_WALL_REGRESSION_SECONDS, + ), + _verdict( + command=comparison.name, + metric="CPU", + baseline=baseline.cpu_seconds, + candidate=candidate.cpu_seconds, + max_ratio=MAX_CPU_RATIO, + floor=MIN_CPU_REGRESSION_SECONDS, + ), + _verdict( + command=comparison.name, + metric="peak RSS", + baseline=float(baseline.peak_rss_bytes), + candidate=float(candidate.peak_rss_bytes), + max_ratio=MAX_RSS_RATIO, + floor=float(MIN_RSS_REGRESSION_BYTES), + ), + ) + + +def regression_messages(*, commands: tuple[CommandComparison, ...]) -> tuple[str, ...]: + """Describe every metric that exceeds both its ratio limit and its absolute floor.""" + + return tuple( + f"{verdict.command}: {verdict.metric} {verdict.ratio:.3f}x exceeds {verdict.max_ratio:.2f}x" + for verdict in all_verdicts(commands=commands) + if verdict.regressed + ) + + +def all_verdicts(*, commands: tuple[CommandComparison, ...]) -> tuple[MetricVerdict, ...]: + """Return the metric verdicts of every measured command in order.""" + + verdicts: list[MetricVerdict] = [] + for comparison in commands: + verdicts.extend(metric_verdicts(comparison=comparison)) + return tuple(verdicts) + + +def median_sample(*, samples: tuple[CommandSample, ...]) -> CommandSample: + """Return the per-metric median of samples.""" + + return CommandSample( + wall_seconds=statistics.median(sample.wall_seconds for sample in samples), + cpu_seconds=statistics.median(sample.cpu_seconds for sample in samples), + peak_rss_bytes=int(statistics.median(sample.peak_rss_bytes for sample in samples)), + ) + + +def skip_reason( + *, command: BenchmarkCommand, baseline_version: str, candidate_version: str +) -> str | None: + """Explain why a command cannot run on a compared version, or return None when it can.""" + + if command.minimum_version is None: + return None + minimum: tuple[int, ...] = version_key(version=command.minimum_version) + for label, version in (("baseline", baseline_version), ("candidate", candidate_version)): + if version_key(version=version) < minimum: + return f"requires sqlbuild {command.minimum_version} or later; {label} is {version}" + return None + + +def version_key(*, version: str) -> tuple[int, ...]: + """Order plain X.Y.Z release versions numerically.""" + + return tuple(int(part) for part in version.split(".")) + + +def _verdict( + *, + command: str, + metric: str, + baseline: float, + candidate: float, + max_ratio: float, + floor: float, +) -> MetricVerdict: + return MetricVerdict( + command=command, + metric=metric, + baseline=baseline, + candidate=candidate, + max_ratio=max_ratio, + floor=floor, + regressed=candidate > baseline * max_ratio and candidate - baseline > floor, + ) diff --git a/scripts/release_performance/_helpers/versions.py b/scripts/release_performance/_helpers/versions.py new file mode 100644 index 0000000000..a676ce7ef1 --- /dev/null +++ b/scripts/release_performance/_helpers/versions.py @@ -0,0 +1,286 @@ +"""Resolve and install the compared sqlbuild versions, each into its own virtual environment.""" + +from __future__ import annotations + +import hashlib +import json +import platform +import re +import shutil +import subprocess +import sys +import time +import urllib.request +from pathlib import Path +from typing import cast + +from scripts.release_performance._helpers.verdict import version_key +from scripts.release_performance.constants import ( + BASELINE_PUBLICATION_POLL_SECONDS, + PACKAGE_NAME, + PYPI_REQUEST_TIMEOUT_SECONDS, + PYPI_SIMPLE_JSON, + PYPI_SIMPLE_URL, + RELEASE_TAG_PATTERN, + STDERR_TAIL_CHARACTERS, + UNIVERSAL_WHEEL_MACHINE, + WHEEL_MACHINE_ALIASES, + WHEEL_PLATFORM_FAMILIES, +) +from scripts.release_performance.exceptions import ReleasePerformanceError + +_RELEASE_VERSION: re.Pattern[str] = re.compile(r"\d+\.\d+\.\d+") +_DISTRIBUTION_VERSION: re.Pattern[str] = re.compile( + rf"^{PACKAGE_NAME}-(\d+\.\d+\.\d+)(?:-.+\.whl|\.tar\.gz)$" +) + + +def release_versions(*, index: dict[str, object]) -> tuple[str, ...]: + """Return the X.Y.Z versions in a PEP 691 index that have a file that is not yanked.""" + + versions: set[str] = set() + files: list[dict[str, object]] = cast(list[dict[str, object]], index.get("files", [])) + for entry in files: + match: re.Match[str] | None = _DISTRIBUTION_VERSION.match(str(entry.get("filename", ""))) + if match is not None and not entry.get("yanked"): + versions.add(match.group(1)) + return tuple(sorted(versions, key=lambda version: version_key(version=version))) + + +def previous_version(*, candidate: str, versions: tuple[str, ...]) -> str | None: + """Return the highest version strictly below candidate.""" + + earlier: list[str] = [ + version + for version in versions + if version_key(version=version) < version_key(version=candidate) + ] + return max(earlier, key=lambda version: version_key(version=version)) if earlier else None + + +def compatible_wheels( + *, index: dict[str, object], version: str, machine: str, system: str +) -> tuple[dict[str, object], ...]: + """Return the non-yanked wheels of version whose platform tag matches this runner.""" + + files: list[dict[str, object]] = cast(list[dict[str, object]], index.get("files", [])) + prefix: str = f"{PACKAGE_NAME}-{version}-" + return tuple( + entry + for entry in files + if str(entry.get("filename", "")).startswith(prefix) + and str(entry.get("filename", "")).endswith(".whl") + and not entry.get("yanked") + and _platform_matches( + platform_tag=str(entry["filename"]).removesuffix(".whl").rsplit("-", 1)[-1], + machine=machine.lower(), + system=system, + ) + ) + + +def fetch_simple_index() -> dict[str, object]: + """Read the PyPI simple index uncompressed, which reflects new uploads before the CDN does.""" + + request: urllib.request.Request = urllib.request.Request( + PYPI_SIMPLE_URL, + headers={ + "Accept": PYPI_SIMPLE_JSON, + "Accept-Encoding": "identity", + "Cache-Control": "no-cache", + }, + ) + with urllib.request.urlopen(request, timeout=PYPI_REQUEST_TIMEOUT_SECONDS) as response: + return cast(dict[str, object], json.loads(response.read())) + + +def tagged_versions(*, repo_dir: Path) -> tuple[str, ...]: + """Return the X.Y.Z versions of the release tags in repo_dir, or none outside a clone.""" + + completed: subprocess.CompletedProcess[str] = subprocess.run( + ["git", "-C", str(repo_dir), "tag", "--list", RELEASE_TAG_PATTERN], + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + return () + return tuple( + tag.removeprefix("v") + for tag in completed.stdout.split() + if _RELEASE_VERSION.fullmatch(tag.removeprefix("v")) + ) + + +def resolve_baseline(*, candidate: str, repo_dir: Path) -> str: + """Return the highest published or release-tagged version below the candidate.""" + + known: tuple[str, ...] = ( + *release_versions(index=fetch_simple_index()), + *tagged_versions(repo_dir=repo_dir), + ) + baseline: str | None = previous_version(candidate=candidate, versions=known) + if baseline is None: + raise ReleasePerformanceError(f"No published sqlbuild release precedes {candidate}") + return baseline + + +def install_published(*, version: str, venv_dir: Path, python: str, wait_seconds: float) -> Path: + """Install a published version, falling back to its wheel URL while the CDN lags.""" + + _create_venv(venv_dir=venv_dir, python=python) + deadline: float = time.monotonic() + wait_seconds + while True: + attempt: subprocess.CompletedProcess[str] = _uv_pip_install( + venv_dir=venv_dir, requirements=(f"{PACKAGE_NAME}=={version}",) + ) + if attempt.returncode == 0: + return _sqb(venv_dir=venv_dir) + print( + f"Index install of {PACKAGE_NAME}=={version} failed; trying the wheel URL: " + f"{_tail(attempt.stderr)}", + file=sys.stderr, + ) + if _install_from_wheel_urls(version=version, venv_dir=venv_dir): + return _sqb(venv_dir=venv_dir) + if time.monotonic() >= deadline: + raise ReleasePerformanceError( + f"{PACKAGE_NAME} {version} is not installable from PyPI; if the release is still " + "publishing, rerun once its wheels are available" + ) + print( + f"{PACKAGE_NAME} {version} is not on PyPI yet; retrying in " + f"{BASELINE_PUBLICATION_POLL_SECONDS:.0f}s", + file=sys.stderr, + ) + time.sleep(BASELINE_PUBLICATION_POLL_SECONDS) + + +def install_wheel(*, wheel: Path, venv_dir: Path, python: str) -> Path: + """Install a locally built wheel into a fresh virtual environment.""" + + _create_venv(venv_dir=venv_dir, python=python) + completed: subprocess.CompletedProcess[str] = _uv_pip_install( + venv_dir=venv_dir, requirements=(str(wheel.resolve()),) + ) + if completed.returncode != 0: + raise ReleasePerformanceError(f"Installing {wheel} failed: {_tail(completed.stderr)}") + return _sqb(venv_dir=venv_dir) + + +def installed_version(*, sqb: Path) -> str: + """Return the version an installed sqb reports.""" + + completed: subprocess.CompletedProcess[str] = subprocess.run( + [str(sqb), "--version"], capture_output=True, text=True, check=False + ) + match: re.Match[str] | None = _RELEASE_VERSION.search(completed.stdout) + if completed.returncode != 0 or match is None: + raise ReleasePerformanceError(f"{sqb} --version failed: {_tail(completed.stderr)}") + return match.group(0) + + +def existing_sqb(*, venv_dir: Path) -> Path: + """Return the sqb entry point of an existing virtual environment.""" + + return _sqb(venv_dir=venv_dir) + + +def _install_from_wheel_urls(*, version: str, venv_dir: Path) -> bool: + try: + index: dict[str, object] = fetch_simple_index() + except OSError as error: + print(f"Reading {PYPI_SIMPLE_URL} failed: {error}", file=sys.stderr) + return False + wheels: tuple[dict[str, object], ...] = compatible_wheels( + index=index, version=version, machine=platform.machine(), system=sys.platform + ) + if not wheels: + return False + download_dir: Path = venv_dir.parent / f"{venv_dir.name}-wheels" + download_dir.mkdir(parents=True, exist_ok=True) + for entry in wheels: + _download_wheel(entry=entry, download_dir=download_dir) + completed: subprocess.CompletedProcess[str] = _uv_pip_install( + venv_dir=venv_dir, + requirements=("--find-links", str(download_dir), f"{PACKAGE_NAME}=={version}"), + ) + if completed.returncode != 0: + print(f"Wheel URL install failed: {_tail(completed.stderr)}", file=sys.stderr) + return completed.returncode == 0 + + +def _download_wheel(*, entry: dict[str, object], download_dir: Path) -> None: + request: urllib.request.Request = urllib.request.Request( + str(entry["url"]), headers={"Accept-Encoding": "identity"} + ) + with urllib.request.urlopen(request, timeout=PYPI_REQUEST_TIMEOUT_SECONDS) as response: + content: bytes = response.read() + hashes: dict[str, str] = cast(dict[str, str], entry.get("hashes", {})) + expected: str | None = hashes.get("sha256") + if expected is not None and hashlib.sha256(content).hexdigest() != expected: + raise ReleasePerformanceError(f"Downloaded {entry['filename']} does not match its sha256") + _ = (download_dir / str(entry["filename"])).write_bytes(content) + + +def _platform_matches(*, platform_tag: str, machine: str, system: str) -> bool: + family: str | None = WHEEL_PLATFORM_FAMILIES.get(system) + machines: tuple[str, ...] = ( + machine, + WHEEL_MACHINE_ALIASES.get(machine, machine), + UNIVERSAL_WHEEL_MACHINE, + ) + if family is None or family not in platform_tag: + return False + return any(name in platform_tag for name in machines) + + +def _create_venv(*, venv_dir: Path, python: str) -> None: + shutil.rmtree(venv_dir, ignore_errors=True) + completed: subprocess.CompletedProcess[str] = subprocess.run( + [_uv(), "venv", "--quiet", "--python", python, str(venv_dir)], + capture_output=True, + text=True, + check=False, + ) + if completed.returncode != 0: + raise ReleasePerformanceError( + f"Creating a Python {python} environment failed: {_tail(completed.stderr)}" + ) + + +def _uv_pip_install( + *, venv_dir: Path, requirements: tuple[str, ...] +) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + _uv(), + "pip", + "install", + "--quiet", + "--python", + str(venv_dir / "bin" / "python"), + *requirements, + ], + capture_output=True, + text=True, + check=False, + ) + + +def _sqb(*, venv_dir: Path) -> Path: + sqb: Path = venv_dir / "bin" / "sqb" + if not sqb.exists(): + raise ReleasePerformanceError(f"{venv_dir} has no sqb entry point") + return sqb + + +def _uv() -> str: + uv: str | None = shutil.which("uv") + if uv is None: + raise ReleasePerformanceError("uv is required to create the compared environments") + return uv + + +def _tail(text: str) -> str: + return text.strip()[-STDERR_TAIL_CHARACTERS:] diff --git a/scripts/release_performance/_helpers/workflow.py b/scripts/release_performance/_helpers/workflow.py new file mode 100644 index 0000000000..5aefcaca53 --- /dev/null +++ b/scripts/release_performance/_helpers/workflow.py @@ -0,0 +1,172 @@ +"""Install both versions, time the benchmark commands and report the release verdict.""" + +from __future__ import annotations + +import dataclasses +import json +import os +import sys +import tempfile +from pathlib import Path + +from scripts.compile_performance_ratio._helpers.report import cpu_model +from scripts.release_performance._helpers.benchmark import ( + compare_command, + prepare_version_projects, + write_pristine_projects, +) +from scripts.release_performance._helpers.report import ( + append_summary, + comparison_markdown, + error_markdown, +) +from scripts.release_performance._helpers.verdict import ( + all_verdicts, + regression_messages, + skip_reason, +) +from scripts.release_performance._helpers.versions import ( + existing_sqb, + install_published, + install_wheel, + installed_version, + resolve_baseline, +) +from scripts.release_performance.constants import ( + BASELINE_LABEL, + BASELINE_PUBLICATION_WAIT_SECONDS, + BENCHMARK_COMMANDS, + CANDIDATE_LABEL, + REPO_ROOT, +) +from scripts.release_performance.exceptions import ReleasePerformanceError +from scripts.release_performance.models import ( + CommandComparison, + ComparisonOptions, + InstalledVersion, + ReleaseComparison, + RunnerContext, + SkippedCommand, +) + + +def run_release_comparison(*, options: ComparisonOptions) -> int: + """Compare the candidate with the baseline release and enforce the release limits.""" + + summary_value: str | None = os.environ.get("GITHUB_STEP_SUMMARY") + summary: Path | None = Path(summary_value) if summary_value else None + try: + if options.work_dir is not None: + options.work_dir.mkdir(parents=True, exist_ok=True) + comparison: ReleaseComparison = _compare(options=options, root=options.work_dir) + else: + with tempfile.TemporaryDirectory(prefix="sqlbuild-release-performance-") as root: + comparison = _compare(options=options, root=Path(root)) + except (ReleasePerformanceError, OSError) as error: + print(f"Release performance comparison failed: {error}", file=sys.stderr) + _ = append_summary(path=summary, markdown=error_markdown(message=str(error))) + return 1 + markdown: str = comparison_markdown(comparison=comparison) + print(markdown) + _ = append_summary(path=summary, markdown=markdown) + if options.output is not None: + _write_evidence(path=options.output, comparison=comparison) + failures: tuple[str, ...] = regression_messages(commands=comparison.commands) + if failures: + print("Release performance regression: " + "; ".join(failures), file=sys.stderr) + return 1 + print("Release performance within the limits of the previous release", file=sys.stderr) + return 0 + + +def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: + print(f"Installing candidate {options.candidate}", file=sys.stderr) + candidate: InstalledVersion = _install( + label=CANDIDATE_LABEL, + spec=options.candidate, + root=root, + python=options.python, + wait_seconds=0.0, + ) + baseline_spec: str = options.baseline or resolve_baseline( + candidate=candidate.version, repo_dir=REPO_ROOT + ) + print(f"Installing baseline {baseline_spec}", file=sys.stderr) + baseline: InstalledVersion = _install( + label=BASELINE_LABEL, + spec=baseline_spec, + root=root, + python=options.python, + wait_seconds=BASELINE_PUBLICATION_WAIT_SECONDS, + ) + print( + f"Comparing candidate {candidate.version} with baseline {baseline.version}", + file=sys.stderr, + ) + load_before: tuple[float, float, float] = os.getloadavg() + print("Generating benchmark projects", file=sys.stderr) + pristine: Path = write_pristine_projects( + root=root, inspection_models=options.inspection_models, build_models=options.build_models + ) + versions: tuple[InstalledVersion, InstalledVersion] = (baseline, candidate) + projects: dict[str, dict[str, Path]] = {} + for version in versions: + print(f"Warming caches for {version.label} {version.version}", file=sys.stderr) + projects[version.label] = prepare_version_projects( + version=version, pristine=pristine, root=root + ) + measured: list[CommandComparison] = [] + skipped: list[SkippedCommand] = [] + for command in BENCHMARK_COMMANDS: + reason: str | None = skip_reason( + command=command, baseline_version=baseline.version, candidate_version=candidate.version + ) + if reason is not None: + print(f"Skipping {command.name}: {reason}", file=sys.stderr) + skipped.append(SkippedCommand(name=command.name, reason=reason)) + continue + measured.append( + compare_command( + command=command, versions=versions, projects=projects, root=root, runs=options.runs + ) + ) + return ReleaseComparison( + baseline_version=baseline.version, + candidate_version=candidate.version, + runs=options.runs, + runner=RunnerContext( + cpu_model=cpu_model(), + cpu_count=os.cpu_count() or 0, + load_average_before=load_before, + load_average_after=os.getloadavg(), + ), + commands=tuple(measured), + skipped=tuple(skipped), + ) + + +def _install( + *, label: str, spec: str, root: Path, python: str, wait_seconds: float +) -> InstalledVersion: + path: Path = Path(spec) + venv_dir: Path = root / "venvs" / label + if path.is_dir(): + sqb: Path = existing_sqb(venv_dir=path) + elif spec.endswith(".whl"): + sqb = install_wheel(wheel=path, venv_dir=venv_dir, python=python) + else: + sqb = install_published( + version=spec, venv_dir=venv_dir, python=python, wait_seconds=wait_seconds + ) + return InstalledVersion(label=label, version=installed_version(sqb=sqb), sqb=sqb) + + +def _write_evidence(*, path: Path, comparison: ReleaseComparison) -> None: + evidence: dict[str, object] = { + **dataclasses.asdict(comparison), + "verdicts": [ + dataclasses.asdict(verdict) for verdict in all_verdicts(commands=comparison.commands) + ], + } + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(json.dumps(evidence, indent=2, default=str) + "\n", encoding="utf-8") diff --git a/scripts/release_performance/constants.py b/scripts/release_performance/constants.py new file mode 100644 index 0000000000..4880c25fb7 --- /dev/null +++ b/scripts/release_performance/constants.py @@ -0,0 +1,106 @@ +"""Stable constants for comparing a release candidate with the previous published release.""" + +from pathlib import Path + +from scripts.release_performance.models import BenchmarkCommand +from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( + SHARED_DIAMOND_HUB, + SHARED_DIAMOND_ROLLUP, +) + +MAX_WALL_RATIO: float = 1.25 +MAX_CPU_RATIO: float = 1.25 +MAX_RSS_RATIO: float = 1.25 +MIN_WALL_REGRESSION_SECONDS: float = 0.5 +MIN_CPU_REGRESSION_SECONDS: float = 0.5 +MIN_RSS_REGRESSION_BYTES: int = 32 * 1024 * 1024 + +DEFAULT_RUNS: int = 3 +DEFAULT_PYTHON: str = "3.12" +DEFAULT_INSPECTION_MODELS: int = 3_000 +DEFAULT_BUILD_MODELS: int = 1_000 +COMMAND_TIMEOUT_SECONDS: float = 900.0 + +PACKAGE_NAME: str = "sqlbuild" +PYPI_SIMPLE_URL: str = "https://pypi.org/simple/sqlbuild/" +PYPI_SIMPLE_JSON: str = "application/vnd.pypi.simple.v1+json" +PYPI_REQUEST_TIMEOUT_SECONDS: float = 60.0 +BASELINE_PUBLICATION_WAIT_SECONDS: float = 1_800.0 +BASELINE_PUBLICATION_POLL_SECONDS: float = 30.0 +RELEASE_TAG_PATTERN: str = "v*" +WHEEL_PLATFORM_FAMILIES: dict[str, str] = {"linux": "linux", "darwin": "macosx", "win32": "win"} +WHEEL_MACHINE_ALIASES: dict[str, str] = {"amd64": "x86_64", "arm64": "aarch64"} +UNIVERSAL_WHEEL_MACHINE: str = "universal2" +REPO_ROOT: Path = Path(__file__).resolve().parents[2] + +BASELINE_LABEL: str = "baseline" +CANDIDATE_LABEL: str = "candidate" +INSPECTION_PROJECT: str = "inspection" +BUILD_PROJECT: str = "build" +EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset({"VIRTUAL_ENV", "GITHUB_STEP_SUMMARY"}) +EXCLUDED_ENVIRONMENT_PREFIX: str = "DBT_" +TIME_FORMAT: str = "%e %M %U %S" +COMPILE_COMMAND: str = "compile" +STDERR_TAIL_CHARACTERS: int = 2_000 + +BENCHMARK_COMMANDS: tuple[BenchmarkCommand, ...] = ( + BenchmarkCommand( + name="compile (no cache)", + project=INSPECTION_PROJECT, + sqb_args=("compile", "--json", "--no-cache"), + ), + BenchmarkCommand( + name="compile (warm cache)", + project=INSPECTION_PROJECT, + sqb_args=("compile", "--json"), + ), + BenchmarkCommand( + name="plan --json", + project=INSPECTION_PROJECT, + sqb_args=("plan", "--json"), + ), + BenchmarkCommand( + name="lineage hub downstream", + project=INSPECTION_PROJECT, + sqb_args=("lineage", SHARED_DIAMOND_HUB, "--direction", "downstream"), + ), + BenchmarkCommand( + name="lineage rollup upstream", + project=INSPECTION_PROJECT, + sqb_args=("lineage", SHARED_DIAMOND_ROLLUP, "--direction", "upstream"), + ), + BenchmarkCommand( + name="lineage column trace", + project=INSPECTION_PROJECT, + sqb_args=("lineage", f"{SHARED_DIAMOND_ROLLUP}.amount", "--direction", "upstream"), + ), + BenchmarkCommand( + name="dag --json", + project=INSPECTION_PROJECT, + sqb_args=("dag", "--json"), + ), + BenchmarkCommand( + name="scope --json", + project=INSPECTION_PROJECT, + sqb_args=( + "scope", + f"model:{SHARED_DIAMOND_ROLLUP}", + "--json", + "--globals", + "all", + "--include-nearby", + ), + ), + BenchmarkCommand( + name="build (empty warehouse)", + project=BUILD_PROJECT, + sqb_args=("build",), + ), +) + +INSPECTION_WARMUPS: tuple[tuple[str, ...], ...] = ( + (COMPILE_COMMAND, "--json"), + ("lineage", SHARED_DIAMOND_HUB, "--depth", "1"), + ("scope", f"model:{SHARED_DIAMOND_ROLLUP}", "--json"), +) +BUILD_WARMUPS: tuple[tuple[str, ...], ...] = ((COMPILE_COMMAND, "--json"),) diff --git a/scripts/release_performance/exceptions.py b/scripts/release_performance/exceptions.py new file mode 100644 index 0000000000..b0386132d3 --- /dev/null +++ b/scripts/release_performance/exceptions.py @@ -0,0 +1,5 @@ +"""Release performance comparison errors.""" + + +class ReleasePerformanceError(RuntimeError): + """Raised when a compared version cannot be installed or cannot run the benchmark.""" diff --git a/scripts/release_performance/main/__init__.py b/scripts/release_performance/main/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/scripts/release_performance/main/compare.py b/scripts/release_performance/main/compare.py new file mode 100644 index 0000000000..73e65a04e2 --- /dev/null +++ b/scripts/release_performance/main/compare.py @@ -0,0 +1,56 @@ +"""Fail when a release candidate is materially slower or larger than the previous release.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from scripts.release_performance._helpers.workflow import run_release_comparison +from scripts.release_performance.constants import ( + DEFAULT_BUILD_MODELS, + DEFAULT_INSPECTION_MODELS, + DEFAULT_PYTHON, + DEFAULT_RUNS, +) +from scripts.release_performance.models import ComparisonOptions + + +def compare_release_performance(argv: list[str] | None = None) -> int: + """Compare the candidate with the baseline release and enforce the release limits.""" + + return run_release_comparison(options=_parse_args(argv)) + + +def _parse_args(argv: list[str] | None) -> ComparisonOptions: + parser: argparse.ArgumentParser = argparse.ArgumentParser( + description=( + "Time sqb commands for a release candidate and the previous published release on " + "this machine, and fail when the candidate is materially slower or larger. Each " + "version is a PyPI version, a wheel path, or an existing virtual environment." + ) + ) + parser.add_argument("--candidate", required=True) + parser.add_argument( + "--baseline", + default=None, + help="Defaults to the highest published or release-tagged version below the candidate.", + ) + parser.add_argument("--runs", type=int, default=DEFAULT_RUNS) + parser.add_argument("--python", default=DEFAULT_PYTHON) + parser.add_argument("--inspection-models", type=int, default=DEFAULT_INSPECTION_MODELS) + parser.add_argument("--build-models", type=int, default=DEFAULT_BUILD_MODELS) + parser.add_argument("--work-dir", type=Path, default=None) + parser.add_argument("--output", type=Path, default=None) + args: argparse.Namespace = parser.parse_args(argv) + if args.runs < 1: + parser.error("--runs must be at least 1") + return ComparisonOptions( + candidate=args.candidate, + baseline=args.baseline, + runs=args.runs, + python=args.python, + inspection_models=args.inspection_models, + build_models=args.build_models, + work_dir=args.work_dir, + output=args.output, + ) diff --git a/scripts/release_performance/models.py b/scripts/release_performance/models.py new file mode 100644 index 0000000000..2f71f3734a --- /dev/null +++ b/scripts/release_performance/models.py @@ -0,0 +1,86 @@ +"""Typed records for comparing a release candidate with the previous published release.""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class BenchmarkCommand: + name: str + project: str + sqb_args: tuple[str, ...] + minimum_version: str | None = None + + +@dataclass(frozen=True) +class InstalledVersion: + label: str + version: str + sqb: Path + + +@dataclass(frozen=True) +class CommandSample: + wall_seconds: float + cpu_seconds: float + peak_rss_bytes: int + + +@dataclass(frozen=True) +class CommandComparison: + name: str + baseline: tuple[CommandSample, ...] + candidate: tuple[CommandSample, ...] + + +@dataclass(frozen=True) +class SkippedCommand: + name: str + reason: str + + +@dataclass(frozen=True) +class MetricVerdict: + command: str + metric: str + baseline: float + candidate: float + max_ratio: float + floor: float + regressed: bool + + @property + def ratio(self) -> float: + return self.candidate / self.baseline if self.baseline else float("inf") + + +@dataclass(frozen=True) +class RunnerContext: + cpu_model: str + cpu_count: int + load_average_before: tuple[float, float, float] + load_average_after: tuple[float, float, float] + + +@dataclass(frozen=True) +class ReleaseComparison: + baseline_version: str + candidate_version: str + runs: int + runner: RunnerContext + commands: tuple[CommandComparison, ...] + skipped: tuple[SkippedCommand, ...] + + +@dataclass(frozen=True) +class ComparisonOptions: + candidate: str + baseline: str | None + runs: int + python: str + inspection_models: int + build_models: int + work_dir: Path | None + output: Path | None diff --git a/tests/e2e/scripts/release_performance/__init__.py b/tests/e2e/scripts/release_performance/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/e2e/scripts/release_performance/_test_types.py b/tests/e2e/scripts/release_performance/_test_types.py new file mode 100644 index 0000000000..5c697e59ae --- /dev/null +++ b/tests/e2e/scripts/release_performance/_test_types.py @@ -0,0 +1,14 @@ +"""Test types for the release performance comparison.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class ReleasePerformanceTestCase: + description: str + inspection_models: int + build_models: int + expected_return_code: int + expected_fragments: tuple[str, ...] diff --git a/tests/e2e/scripts/release_performance/test_compare_release_performance.py b/tests/e2e/scripts/release_performance/test_compare_release_performance.py new file mode 100644 index 0000000000..1395cc1af0 --- /dev/null +++ b/tests/e2e/scripts/release_performance/test_compare_release_performance.py @@ -0,0 +1,85 @@ +"""End-to-end tests for the release performance comparison against the previous release.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +from scripts.release_performance.constants import BENCHMARK_COMMANDS +from tests.e2e.scripts.release_performance._test_types import ReleasePerformanceTestCase + +_REPO_ROOT: Path = Path(__file__).resolve().parents[4] + + +@pytest.mark.parametrize( + "test_case", + ( + ReleasePerformanceTestCase( + description="the same build as candidate and baseline passes every command", + inspection_models=300, + build_models=100, + expected_return_code=0, + expected_fragments=( + "| `plan --json` |", + "| `build (empty warehouse)` |", + "| `lineage column trace` |", + "**Result: passed.**", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_two_installations_when_comparing_release_performance_then_reports_each_command( + test_case: ReleasePerformanceTestCase, tmp_path: Path +) -> None: + summary: Path = tmp_path / "summary.md" + evidence: Path = tmp_path / "evidence.json" + environment: dict[str, str] = dict(os.environ) + environment["GITHUB_STEP_SUMMARY"] = str(summary) + installation: str = str(Path(sys.executable).parent.parent) + + result: subprocess.CompletedProcess[str] = subprocess.run( + [ + sys.executable, + "-m", + "scripts.compare_release_performance", + "--candidate", + installation, + "--baseline", + installation, + "--runs", + "1", + "--inspection-models", + str(test_case.inspection_models), + "--build-models", + str(test_case.build_models), + "--work-dir", + str(tmp_path / "work"), + "--output", + str(evidence), + ], + cwd=_REPO_ROOT, + capture_output=True, + text=True, + env=environment, + check=False, + ) + + output: str = result.stdout + result.stderr + assert result.returncode == test_case.expected_return_code, output + markdown: str = summary.read_text(encoding="utf-8") + for fragment in test_case.expected_fragments: + assert fragment in markdown + recorded: dict[str, object] = json.loads(evidence.read_text(encoding="utf-8")) + assert [command["name"] for command in recorded["commands"]] == [ + command.name for command in BENCHMARK_COMMANDS + ] + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md index 180d6f4be7..2020f7b224 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md @@ -84,3 +84,36 @@ empty warehouse once per module, with wall and peak-RSS limits. Its existing-sta built project, change the queries of a few models (and rename one table in the second case so rename discovery matches it against stored fingerprints), and bound the next `sqb plan --json`. All cases carry `models_3000` identifiers so they run only in the 3,000-model fresh-process job. + +## Release comparison with the previous release + +Fixed ceilings miss gradual creep and slowdowns that stay under a loose limit, so every release +pull request also compares the candidate with the previous published release on the same runner. +`scripts/compare_release_performance.py` installs the candidate wheel and the baseline from PyPI +into separate Python 3.12 environments, generates the inspection and build benchmarks above once, +and gives each version its own copy with compile, lineage and scope caches warmed by that version. +It then runs each command alternately for baseline and candidate, three times each, and compares +the medians of wall time, CPU time (user+sys) and peak RSS: + +- warm and uncached `sqb compile --json`, `sqb plan --json`, `sqb dag --json` and + `sqb scope --json` on the 3,000-model inspection benchmark; +- `sqb lineage` downstream from the hub, upstream from the rollup, and the upstream column trace of + `shared_orders_rollup.amount`; +- `sqb build` of the 1,000-model build benchmark from an empty warehouse, restored before each run. + +A command fails when its candidate median exceeds the baseline by more than 25% and by more than +0.5 s (wall and CPU) or 32 MiB (peak RSS); the limits live in +`scripts/release_performance/constants.py`. The baseline defaults to the highest published or +release-tagged version below the candidate. A freshly tagged baseline is awaited until its wheels +are on PyPI, installing from the uncompressed simple index's wheel URL when the CDN still lags. A +command that needs a feature newer than a compared version declares `minimum_version` and appears +as skipped, with the reason, in the job summary; any other failure of either version fails the +check. + +Release Please dispatches `.github/workflows/release-performance.yml` with the metadata and version +checks and posts the required `Verify` status only after it passes, so a regressed release pull +request cannot auto-merge. Run it manually with an optional `baseline` input, or locally: + +```bash +uv run python -m scripts.compare_release_performance --candidate 0.121.0 --baseline 0.119.1 +``` diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index b40d029121..510052ba73 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -735,21 +735,28 @@ def run_fresh_process_inspection_command( ) -def prepare_inspection_benchmark_project(*, project_dir: Path) -> None: - """Write and compile the shared-dependency benchmark, then warm lineage and scope caches.""" +def write_inspection_benchmark_project(*, project_dir: Path, model_count: int = 3_000) -> None: + """Write the semantic benchmark scaled to model_count models plus the shared lattice.""" + scale: float = model_count / 3_000 write_semantic_compile_project( project_dir=project_dir, - model_count=3_000, - source_count=713, - seed_count=141, - function_count=71, - macro_count=37, - test_count=2_945, - audit_count=5_056, + model_count=model_count, + source_count=max(1, round(713 * scale)), + seed_count=max(1, round(141 * scale)), + function_count=max(1, round(71 * scale)), + macro_count=max(1, round(37 * scale)), + test_count=max(1, round(2_945 * scale)), + audit_count=max(1, round(5_056 * scale)), shared_diamond_layers=INSPECTION_DIAMOND_LAYERS, shared_diamond_width=INSPECTION_DIAMOND_WIDTH, ) + + +def prepare_inspection_benchmark_project(*, project_dir: Path) -> None: + """Write and compile the shared-dependency benchmark, then warm lineage and scope caches.""" + + write_inspection_benchmark_project(project_dir=project_dir) compiled: InspectionCommandMeasurement = run_fresh_process_inspection_command( project_dir=project_dir, label="inspection-compile", @@ -2190,16 +2197,19 @@ class BuiltBenchmark(NamedTuple): _BUILD_BENCHMARK_EDIT: tuple[str, str] = ("amount + 0 WHEN", "amount + 100 WHEN") -def prepare_build_benchmark_project(*, project_dir: Path) -> None: - """Write a 1,000-model benchmark without tests or audits and warm its compile cache.""" +def write_build_benchmark_project( + *, project_dir: Path, model_count: int = BUILD_BENCHMARK_MODEL_COUNT +) -> None: + """Write the build benchmark without tests or audits, scaled to model_count models.""" + scale: float = model_count / BUILD_BENCHMARK_MODEL_COUNT write_semantic_compile_project( project_dir=project_dir, - model_count=BUILD_BENCHMARK_MODEL_COUNT, - source_count=238, - seed_count=47, - function_count=24, - macro_count=12, + model_count=model_count, + source_count=max(1, round(238 * scale)), + seed_count=max(1, round(47 * scale)), + function_count=max(1, round(24 * scale)), + macro_count=max(1, round(12 * scale)), test_count=0, audit_count=0, ) @@ -2207,6 +2217,12 @@ def prepare_build_benchmark_project(*, project_dir: Path) -> None: config.write_text( config.read_text(encoding="utf-8") + _BUILD_BENCHMARK_CONNECTION, encoding="utf-8" ) + + +def prepare_build_benchmark_project(*, project_dir: Path) -> None: + """Write a 1,000-model benchmark without tests or audits and warm its compile cache.""" + + write_build_benchmark_project(project_dir=project_dir) _ = run_fresh_process_inspection_command( project_dir=project_dir, label="build-benchmark-compile", diff --git a/tests/unit/scripts/release_performance/__init__.py b/tests/unit/scripts/release_performance/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/release_performance/_helpers/__init__.py b/tests/unit/scripts/release_performance/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/release_performance/_helpers/_test_types.py b/tests/unit/scripts/release_performance/_helpers/_test_types.py new file mode 100644 index 0000000000..aef0ee34e1 --- /dev/null +++ b/tests/unit/scripts/release_performance/_helpers/_test_types.py @@ -0,0 +1,53 @@ +"""Test-case models for the release performance comparison.""" + +from dataclasses import dataclass + +from scripts.release_performance.models import CommandComparison, SkippedCommand + + +@dataclass(frozen=True) +class MetricVerdictTestCase: + description: str + comparison: CommandComparison + expected_regressed: tuple[bool, bool, bool] + + +@dataclass(frozen=True) +class SkipReasonTestCase: + description: str + minimum_version: str | None + baseline_version: str + candidate_version: str + expected_reason: str | None + + +@dataclass(frozen=True) +class ComparisonMarkdownTestCase: + description: str + commands: tuple[CommandComparison, ...] + skipped: tuple[SkippedCommand, ...] + expected_fragments: tuple[str, ...] + unexpected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class PreviousVersionTestCase: + description: str + candidate: str + versions: tuple[str, ...] + expected_version: str | None + + +@dataclass(frozen=True) +class ReleaseVersionsTestCase: + description: str + files: tuple[dict[str, object], ...] + expected_versions: tuple[str, ...] + + +@dataclass(frozen=True) +class CompatibleWheelsTestCase: + description: str + machine: str + system: str + expected_filenames: tuple[str, ...] diff --git a/tests/unit/scripts/release_performance/_helpers/helpers.py b/tests/unit/scripts/release_performance/_helpers/helpers.py new file mode 100644 index 0000000000..7fa7c1385f --- /dev/null +++ b/tests/unit/scripts/release_performance/_helpers/helpers.py @@ -0,0 +1,31 @@ +"""Builders for release performance comparison test data.""" + +from scripts.release_performance.models import CommandComparison, CommandSample + +MIB: int = 1024 * 1024 + + +def comparison( + *, + name: str, + baseline: tuple[tuple[float, float, int], ...], + candidate: tuple[tuple[float, float, int], ...], +) -> CommandComparison: + """Build a comparison from (wall seconds, CPU seconds, peak RSS MiB) samples.""" + + return CommandComparison( + name=name, + baseline=tuple(_sample(values=values) for values in baseline), + candidate=tuple(_sample(values=values) for values in candidate), + ) + + +def release_file(*, filename: str) -> dict[str, object]: + """Build one PEP 691 file entry that is not yanked.""" + + return {"filename": filename, "yanked": False, "url": filename} + + +def _sample(*, values: tuple[float, float, int]) -> CommandSample: + wall, cpu, rss_mib = values + return CommandSample(wall_seconds=wall, cpu_seconds=cpu, peak_rss_bytes=rss_mib * MIB) diff --git a/tests/unit/scripts/release_performance/_helpers/test_report.py b/tests/unit/scripts/release_performance/_helpers/test_report.py new file mode 100644 index 0000000000..6a0cfc8b7f --- /dev/null +++ b/tests/unit/scripts/release_performance/_helpers/test_report.py @@ -0,0 +1,105 @@ +"""Tests for the release performance job summary.""" + +import pytest + +from scripts.release_performance._helpers.report import comparison_markdown +from scripts.release_performance.models import ( + ReleaseComparison, + RunnerContext, + SkippedCommand, +) +from tests.unit.scripts.release_performance._helpers._test_types import ( + ComparisonMarkdownTestCase, +) +from tests.unit.scripts.release_performance._helpers.helpers import comparison + + +@pytest.mark.parametrize( + "test_case", + ( + ComparisonMarkdownTestCase( + description="a passing comparison lists every command and the limits", + commands=( + comparison( + name="compile (warm cache)", + baseline=((6.0, 6.5, 500),) * 3, + candidate=((6.1, 6.4, 505),) * 3, + ), + ), + skipped=(), + expected_fragments=( + "### Release performance: 0.126.2 (candidate) vs 0.126.1 (baseline)", + "Same runner: Example CPU, 4 CPUs; load average 0.50/0.40/0.30 before", + "3 interleaved runs per version", + "Limits: wall +25% (ignored under 0.5 s), CPU +25% (ignored under 0.5 s), " + "peak RSS +25% (ignored under 32 MiB).", + "| `compile (warm cache)` | 6.00 → 6.10 | 1.017x | 6.50 → 6.40 | 0.985x " + "| 500 → 505 | 1.010x | ✅ ok |", + "**Result: passed.**", + ), + unexpected_fragments=("❌", "skipped"), + ), + ComparisonMarkdownTestCase( + description="a regression is bolded and named in the failed result", + commands=( + comparison( + name="plan --json", + baseline=((27.0, 28.0, 900),) * 3, + candidate=((56.0, 57.0, 1_800),) * 3, + ), + ), + skipped=(), + expected_fragments=( + "| `plan --json` | 27.00 → 56.00 | **2.074x** | 28.00 → 57.00 | **2.036x** " + "| 900 → 1800 | **2.000x** | ❌ regressed |", + "**Result: failed.** plan --json: wall 2.074x exceeds 1.25x; plan --json: CPU " + "2.036x exceeds 1.25x; plan --json: peak RSS 2.000x exceeds 1.25x.", + ), + unexpected_fragments=("**Result: passed.**",), + ), + ComparisonMarkdownTestCase( + description="a skipped command is reported with its reason instead of omitted", + commands=(), + skipped=( + SkippedCommand( + name="scope --json", + reason="requires sqlbuild 0.126.2 or later; baseline is 0.126.1", + ), + ), + expected_fragments=( + "| `scope --json` | – | – | – | – | – | – | ⚠️ skipped: requires sqlbuild " + "0.126.2 or later; baseline is 0.126.1 |", + "**1 command(s) skipped** because a compared version predates them", + ), + unexpected_fragments=("❌",), + ), + ), + ids=lambda case: case.description, +) +def test_given_release_comparison_when_rendering_summary_then_reports_each_command( + test_case: ComparisonMarkdownTestCase, +) -> None: + release: ReleaseComparison = ReleaseComparison( + baseline_version="0.126.1", + candidate_version="0.126.2", + runs=3, + runner=RunnerContext( + cpu_model="Example CPU", + cpu_count=4, + load_average_before=(0.5, 0.4, 0.3), + load_average_after=(0.6, 0.5, 0.4), + ), + commands=test_case.commands, + skipped=test_case.skipped, + ) + + markdown: str = comparison_markdown(comparison=release) + + for fragment in test_case.expected_fragments: + assert fragment in markdown + for fragment in test_case.unexpected_fragments: + assert fragment not in markdown + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/release_performance/_helpers/test_verdict.py b/tests/unit/scripts/release_performance/_helpers/test_verdict.py new file mode 100644 index 0000000000..85a0d051ca --- /dev/null +++ b/tests/unit/scripts/release_performance/_helpers/test_verdict.py @@ -0,0 +1,138 @@ +"""Tests for the release performance regression verdict.""" + +import pytest + +from scripts.release_performance._helpers.verdict import ( + metric_verdicts, + regression_messages, + skip_reason, +) +from scripts.release_performance.models import BenchmarkCommand, MetricVerdict +from tests.unit.scripts.release_performance._helpers._test_types import ( + MetricVerdictTestCase, + SkipReasonTestCase, +) +from tests.unit.scripts.release_performance._helpers.helpers import comparison + + +@pytest.mark.parametrize( + "test_case", + ( + MetricVerdictTestCase( + description="identical medians pass", + comparison=comparison( + name="dag --json", + baseline=((9.0, 9.0, 500),) * 3, + candidate=((9.0, 9.0, 500),) * 3, + ), + expected_regressed=(False, False, False), + ), + MetricVerdictTestCase( + description="slower planning with more peak memory fails CPU and RSS below wall limit", + comparison=comparison( + name="plan --json", + baseline=((122.7, 125.6, 973),) * 3, + candidate=((150.3, 157.3, 1_742),) * 3, + ), + expected_regressed=(False, True, True), + ), + MetricVerdictTestCase( + description="a sub-second command doubling stays under the absolute floor", + comparison=comparison( + name="scope --json", + baseline=((0.3, 0.3, 70),) * 3, + candidate=((0.7, 0.7, 70),) * 3, + ), + expected_regressed=(False, False, False), + ), + MetricVerdictTestCase( + description="a small allocation growth stays under the memory floor", + comparison=comparison( + name="lineage hub downstream", + baseline=((2.0, 2.0, 40),) * 3, + candidate=((2.0, 2.0, 70),) * 3, + ), + expected_regressed=(False, False, False), + ), + MetricVerdictTestCase( + description="exactly the ratio limit passes and just above it fails", + comparison=comparison( + name="compile (warm cache)", + baseline=((10.0, 10.0, 400),) * 3, + candidate=((12.5, 12.6, 400),) * 3, + ), + expected_regressed=(False, True, False), + ), + MetricVerdictTestCase( + description="one slow outlier does not move the median", + comparison=comparison( + name="build (empty warehouse)", + baseline=((60.0, 60.0, 700),) * 3, + candidate=((60.0, 60.0, 700), (95.0, 95.0, 1_400), (61.0, 61.0, 710)), + ), + expected_regressed=(False, False, False), + ), + ), + ids=lambda case: case.description, +) +def test_given_command_samples_when_judging_then_flags_only_material_regressions( + test_case: MetricVerdictTestCase, +) -> None: + verdicts: tuple[MetricVerdict, ...] = metric_verdicts(comparison=test_case.comparison) + + assert tuple(verdict.metric for verdict in verdicts) == ("wall", "CPU", "peak RSS") + assert tuple(verdict.regressed for verdict in verdicts) == test_case.expected_regressed + messages: tuple[str, ...] = regression_messages(commands=(test_case.comparison,)) + assert len(messages) == sum(test_case.expected_regressed) + for message in messages: + assert message.startswith(f"{test_case.comparison.name}: ") + + +@pytest.mark.parametrize( + "test_case", + ( + SkipReasonTestCase( + description="a command without a minimum version always runs", + minimum_version=None, + baseline_version="0.119.1", + candidate_version="0.121.0", + expected_reason=None, + ), + SkipReasonTestCase( + description="a command newer than the baseline is skipped with the baseline named", + minimum_version="0.126.0", + baseline_version="0.125.1", + candidate_version="0.126.0", + expected_reason="requires sqlbuild 0.126.0 or later; baseline is 0.125.1", + ), + SkipReasonTestCase( + description="versions compare numerically rather than as text", + minimum_version="0.99.0", + baseline_version="0.126.1", + candidate_version="0.126.2", + expected_reason=None, + ), + ), + ids=lambda case: case.description, +) +def test_given_command_minimum_version_when_checking_versions_then_explains_skips( + test_case: SkipReasonTestCase, +) -> None: + command: BenchmarkCommand = BenchmarkCommand( + name="dag --json", + project="inspection", + sqb_args=("dag", "--json"), + minimum_version=test_case.minimum_version, + ) + + reason: str | None = skip_reason( + command=command, + baseline_version=test_case.baseline_version, + candidate_version=test_case.candidate_version, + ) + + assert reason == test_case.expected_reason + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/release_performance/_helpers/test_versions.py b/tests/unit/scripts/release_performance/_helpers/test_versions.py new file mode 100644 index 0000000000..8e24b7a5e2 --- /dev/null +++ b/tests/unit/scripts/release_performance/_helpers/test_versions.py @@ -0,0 +1,132 @@ +"""Tests for resolving the published baseline version.""" + +import pytest + +from scripts.release_performance._helpers.versions import ( + compatible_wheels, + previous_version, + release_versions, +) +from tests.unit.scripts.release_performance._helpers._test_types import ( + CompatibleWheelsTestCase, + PreviousVersionTestCase, + ReleaseVersionsTestCase, +) +from tests.unit.scripts.release_performance._helpers.helpers import release_file + +_RELEASE_FILES: tuple[str, ...] = ( + "sqlbuild-0.126.1-cp312-abi3-macosx_10_12_x86_64.whl", + "sqlbuild-0.126.1-cp312-abi3-macosx_11_0_arm64.whl", + "sqlbuild-0.126.1-cp312-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "sqlbuild-0.126.1-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "sqlbuild-0.126.1-cp312-abi3-win_amd64.whl", + "sqlbuild-0.126.1.tar.gz", + "sqlbuild-0.126.0-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", +) + + +@pytest.mark.parametrize( + "test_case", + ( + PreviousVersionTestCase( + description="the highest version below a release candidate", + candidate="0.126.3", + versions=("0.126.1", "0.126.2", "0.125.9", "0.126.2"), + expected_version="0.126.2", + ), + PreviousVersionTestCase( + description="an already published candidate compares with the release before it", + candidate="0.126.2", + versions=("0.126.1", "0.126.2"), + expected_version="0.126.1", + ), + PreviousVersionTestCase( + description="versions order numerically rather than as text", + candidate="0.100.0", + versions=("0.99.0", "0.9.9", "0.100.0"), + expected_version="0.99.0", + ), + PreviousVersionTestCase( + description="no earlier release", + candidate="0.1.0", + versions=("0.1.0",), + expected_version=None, + ), + ), + ids=lambda case: case.description, +) +def test_given_known_versions_when_choosing_baseline_then_returns_previous_release( + test_case: PreviousVersionTestCase, +) -> None: + assert ( + previous_version(candidate=test_case.candidate, versions=test_case.versions) + == test_case.expected_version + ) + + +@pytest.mark.parametrize( + "test_case", + ( + ReleaseVersionsTestCase( + description="wheels and sdists of every release", + files=tuple(release_file(filename=name) for name in _RELEASE_FILES), + expected_versions=("0.126.0", "0.126.1"), + ), + ReleaseVersionsTestCase( + description="a yanked release and a pre-release are not baselines", + files=( + {"filename": "sqlbuild-0.126.0.tar.gz", "yanked": False}, + {"filename": "sqlbuild-0.126.1.tar.gz", "yanked": "broken wheel"}, + {"filename": "sqlbuild-0.127.0rc1.tar.gz", "yanked": False}, + ), + expected_versions=("0.126.0",), + ), + ), + ids=lambda case: case.description, +) +def test_given_simple_index_when_listing_versions_then_returns_installable_releases( + test_case: ReleaseVersionsTestCase, +) -> None: + assert release_versions(index={"files": list(test_case.files)}) == test_case.expected_versions + + +@pytest.mark.parametrize( + "test_case", + ( + CompatibleWheelsTestCase( + description="linux x86_64 runner", + machine="x86_64", + system="linux", + expected_filenames=( + "sqlbuild-0.126.1-cp312-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + ), + ), + CompatibleWheelsTestCase( + description="apple silicon", + machine="arm64", + system="darwin", + expected_filenames=("sqlbuild-0.126.1-cp312-abi3-macosx_11_0_arm64.whl",), + ), + CompatibleWheelsTestCase( + description="unsupported platform", + machine="riscv64", + system="linux", + expected_filenames=(), + ), + ), + ids=lambda case: case.description, +) +def test_given_release_files_when_selecting_wheel_urls_then_matches_runner_platform( + test_case: CompatibleWheelsTestCase, +) -> None: + index: dict[str, object] = {"files": [release_file(filename=name) for name in _RELEASE_FILES]} + + wheels: tuple[dict[str, object], ...] = compatible_wheels( + index=index, version="0.126.1", machine=test_case.machine, system=test_case.system + ) + + assert tuple(entry["filename"] for entry in wheels) == test_case.expected_filenames + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) From 3d187c08842821ea0aa609641347c0a49e48fbff Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 29 Sep 2026 06:36:42 +0100 Subject: [PATCH 2/3] ci: ignore yanked releases when choosing the performance baseline A release tag only counts as a baseline while its version is not yet on PyPI, so a yanked release is never compared against. --- .../release_performance/_helpers/versions.py | 40 +++++++++----- .../_helpers/_test_types.py | 9 ++++ .../_helpers/test_versions.py | 52 +++++++++++++++++++ 3 files changed, 89 insertions(+), 12 deletions(-) diff --git a/scripts/release_performance/_helpers/versions.py b/scripts/release_performance/_helpers/versions.py index a676ce7ef1..616405439b 100644 --- a/scripts/release_performance/_helpers/versions.py +++ b/scripts/release_performance/_helpers/versions.py @@ -38,13 +38,19 @@ def release_versions(*, index: dict[str, object]) -> tuple[str, ...]: """Return the X.Y.Z versions in a PEP 691 index that have a file that is not yanked.""" - versions: set[str] = set() - files: list[dict[str, object]] = cast(list[dict[str, object]], index.get("files", [])) - for entry in files: - match: re.Match[str] | None = _DISTRIBUTION_VERSION.match(str(entry.get("filename", ""))) - if match is not None and not entry.get("yanked"): - versions.add(match.group(1)) - return tuple(sorted(versions, key=lambda version: version_key(version=version))) + return _index_versions(index=index, include_yanked=False) + + +def choose_baseline( + *, candidate: str, index: dict[str, object], tags: tuple[str, ...] +) -> str | None: + """Return the highest installable or still-publishing release below the candidate.""" + + listed: frozenset[str] = frozenset(_index_versions(index=index, include_yanked=True)) + publishing: tuple[str, ...] = tuple(tag for tag in tags if tag not in listed) + return previous_version( + candidate=candidate, versions=(*release_versions(index=index), *publishing) + ) def previous_version(*, candidate: str, versions: tuple[str, ...]) -> str | None: @@ -113,13 +119,13 @@ def tagged_versions(*, repo_dir: Path) -> tuple[str, ...]: def resolve_baseline(*, candidate: str, repo_dir: Path) -> str: - """Return the highest published or release-tagged version below the candidate.""" + """Return the highest published or not-yet-published tagged release below the candidate.""" - known: tuple[str, ...] = ( - *release_versions(index=fetch_simple_index()), - *tagged_versions(repo_dir=repo_dir), + baseline: str | None = choose_baseline( + candidate=candidate, + index=fetch_simple_index(), + tags=tagged_versions(repo_dir=repo_dir), ) - baseline: str | None = previous_version(candidate=candidate, versions=known) if baseline is None: raise ReleasePerformanceError(f"No published sqlbuild release precedes {candidate}") return baseline @@ -186,6 +192,16 @@ def existing_sqb(*, venv_dir: Path) -> Path: return _sqb(venv_dir=venv_dir) +def _index_versions(*, index: dict[str, object], include_yanked: bool) -> tuple[str, ...]: + versions: set[str] = set() + files: list[dict[str, object]] = cast(list[dict[str, object]], index.get("files", [])) + for entry in files: + match: re.Match[str] | None = _DISTRIBUTION_VERSION.match(str(entry.get("filename", ""))) + if match is not None and (include_yanked or not entry.get("yanked")): + versions.add(match.group(1)) + return tuple(sorted(versions, key=lambda version: version_key(version=version))) + + def _install_from_wheel_urls(*, version: str, venv_dir: Path) -> bool: try: index: dict[str, object] = fetch_simple_index() diff --git a/tests/unit/scripts/release_performance/_helpers/_test_types.py b/tests/unit/scripts/release_performance/_helpers/_test_types.py index aef0ee34e1..3e0b2aa9e6 100644 --- a/tests/unit/scripts/release_performance/_helpers/_test_types.py +++ b/tests/unit/scripts/release_performance/_helpers/_test_types.py @@ -51,3 +51,12 @@ class CompatibleWheelsTestCase: machine: str system: str expected_filenames: tuple[str, ...] + + +@dataclass(frozen=True) +class ChooseBaselineTestCase: + description: str + candidate: str + files: tuple[dict[str, object], ...] + tags: tuple[str, ...] + expected_baseline: str | None diff --git a/tests/unit/scripts/release_performance/_helpers/test_versions.py b/tests/unit/scripts/release_performance/_helpers/test_versions.py index 8e24b7a5e2..0aae6d9e73 100644 --- a/tests/unit/scripts/release_performance/_helpers/test_versions.py +++ b/tests/unit/scripts/release_performance/_helpers/test_versions.py @@ -3,11 +3,13 @@ import pytest from scripts.release_performance._helpers.versions import ( + choose_baseline, compatible_wheels, previous_version, release_versions, ) from tests.unit.scripts.release_performance._helpers._test_types import ( + ChooseBaselineTestCase, CompatibleWheelsTestCase, PreviousVersionTestCase, ReleaseVersionsTestCase, @@ -128,5 +130,55 @@ def test_given_release_files_when_selecting_wheel_urls_then_matches_runner_platf assert tuple(entry["filename"] for entry in wheels) == test_case.expected_filenames +@pytest.mark.parametrize( + "test_case", + ( + ChooseBaselineTestCase( + description="a tagged release that is still publishing is the baseline", + candidate="0.126.3", + files=(release_file(filename="sqlbuild-0.126.1.tar.gz"),), + tags=("0.126.1", "0.126.2"), + expected_baseline="0.126.2", + ), + ChooseBaselineTestCase( + description="a yanked release is skipped even though its tag exists", + candidate="0.126.3", + files=( + release_file(filename="sqlbuild-0.126.1.tar.gz"), + {"filename": "sqlbuild-0.126.2.tar.gz", "yanked": "broken build", "url": ""}, + ), + tags=("0.126.1", "0.126.2"), + expected_baseline="0.126.1", + ), + ChooseBaselineTestCase( + description="a published release without a local tag is still a baseline", + candidate="0.126.3", + files=( + release_file(filename="sqlbuild-0.126.1.tar.gz"), + release_file(filename="sqlbuild-0.126.2.tar.gz"), + ), + tags=(), + expected_baseline="0.126.2", + ), + ChooseBaselineTestCase( + description="nothing precedes the first release", + candidate="0.1.0", + files=(release_file(filename="sqlbuild-0.1.0.tar.gz"),), + tags=("0.1.0",), + expected_baseline=None, + ), + ), + ids=lambda case: case.description, +) +def test_given_index_and_release_tags_when_choosing_baseline_then_skips_yanked_releases( + test_case: ChooseBaselineTestCase, +) -> None: + baseline: str | None = choose_baseline( + candidate=test_case.candidate, index={"files": list(test_case.files)}, tags=test_case.tags + ) + + assert baseline == test_case.expected_baseline + + if __name__ == "__main__": pytest.main([__file__, "-vv"]) From bbd41c6d7810cb088ec3693d70687868ecb59e88 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Tue, 29 Sep 2026 06:36:42 +0100 Subject: [PATCH 3/3] ci: match dispatched release checks exactly and allow reruns Release Please passes a per-attempt request id to each trusted check and waits only on the run carrying it, never with a baseline override. The checks run in their own job so re-running it re-dispatches all three and repaints Verify on the current head, and the delivery token is minted just before enabling auto-merge. --- .github/workflows/pull-request-metadata.yml | 7 + .github/workflows/release-performance.yml | 12 ++ .github/workflows/release-please.yml | 129 +++++++++++------- .github/workflows/version-guard.yml | 8 ++ .../main/compile/COMPILE_BENCHMARKS.md | 10 +- 5 files changed, 112 insertions(+), 54 deletions(-) diff --git a/.github/workflows/pull-request-metadata.yml b/.github/workflows/pull-request-metadata.yml index 736a91f369..09d068efee 100644 --- a/.github/workflows/pull-request-metadata.yml +++ b/.github/workflows/pull-request-metadata.yml @@ -1,4 +1,7 @@ name: Pull Request Metadata +run-name: >- + ${{ inputs.request_id && format('{0} [{1}]', github.workflow, inputs.request_id) + || github.event.pull_request.title || github.event.head_commit.message || github.workflow }} on: pull_request: @@ -9,6 +12,10 @@ on: description: Pull request number to validate required: true type: number + request_id: + description: "Release Please attempt that dispatched this run; leave empty for manual runs" + required: false + type: string permissions: contents: read diff --git a/.github/workflows/release-performance.yml b/.github/workflows/release-performance.yml index d3314d26cb..22492427f9 100644 --- a/.github/workflows/release-performance.yml +++ b/.github/workflows/release-performance.yml @@ -1,4 +1,7 @@ name: Release Performance +run-name: >- + ${{ inputs.request_id && format('{0} [{1}]', github.workflow, inputs.request_id) + || github.event.pull_request.title || github.event.head_commit.message || github.workflow }} # Release Please dispatches this workflow for each release pull request and reports its result # through the required Verify status. Pull requests that change the comparison itself run it too. @@ -9,6 +12,10 @@ on: description: "Published sqlbuild version to compare against (default: the release before the candidate)" required: false type: string + request_id: + description: "Release Please attempt that dispatched this run; leave empty for manual runs" + required: false + type: string pull_request: paths: - .github/workflows/release-performance.yml @@ -51,7 +58,12 @@ jobs: - name: Compare candidate and previous release on this runner env: BASELINE: ${{ inputs.baseline }} + REQUEST_ID: ${{ inputs.request_id }} run: | + if [[ -n "${REQUEST_ID}" && -n "${BASELINE}" ]]; then + echo "Release Please runs always compare with the previous release; drop baseline" >&2 + exit 1 + fi wheels=("${RUNNER_TEMP}"/candidate/sqlbuild-*.whl) args=( --candidate "${wheels[0]}" diff --git a/.github/workflows/release-please.yml b/.github/workflows/release-please.yml index dd98539b8f..e4b2a4b414 100644 --- a/.github/workflows/release-please.yml +++ b/.github/workflows/release-please.yml @@ -19,6 +19,10 @@ concurrency: jobs: release-please: runs-on: ubuntu-24.04 + outputs: + prs_created: ${{ steps.release.outputs.prs_created }} + release_branch: ${{ steps.release_branch.outputs.name }} + release_pr: ${{ steps.release_branch.outputs.pr }} steps: - name: Create delivery token id: delivery-token @@ -56,11 +60,13 @@ jobs: RELEASE_PRS: ${{ steps.release.outputs.prs }} run: | release_branch="$(jq -r '.[0].headBranchName // empty' <<< "${RELEASE_PRS}")" - if [[ -z "${release_branch}" ]]; then - echo "Release Please reported a created PR without a head branch" >&2 + release_pr="$(jq -r '.[0].number // empty' <<< "${RELEASE_PRS}")" + if [[ -z "${release_branch}" || -z "${release_pr}" ]]; then + echo "Release Please reported a created PR without a head branch or number" >&2 exit 1 fi echo "name=${release_branch}" >> "${GITHUB_OUTPUT}" + echo "pr=${release_pr}" >> "${GITHUB_OUTPUT}" - uses: actions/checkout@v4 if: steps.release.outputs.prs_created == 'true' @@ -99,93 +105,114 @@ jobs: git commit -m "chore: refresh release lockfiles" git push origin "HEAD:${RELEASE_BRANCH}" + # A separate job so that "Re-run failed jobs" re-dispatches and re-evaluates every trusted check + # and repaints their statuses and Verify on the current head of the release branch. + release-checks: + name: Release pull request checks + needs: release-please + if: needs.release-please.outputs.prs_created == 'true' + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@v4 + with: + ref: ${{ needs.release-please.outputs.release_branch }} + - name: Run release pull request checks - if: steps.release.outputs.prs_created == 'true' env: GH_TOKEN: ${{ github.token }} - RELEASE_BRANCH: ${{ steps.release_branch.outputs.name }} - RELEASE_PRS: ${{ steps.release.outputs.prs }} + RELEASE_BRANCH: ${{ needs.release-please.outputs.release_branch }} + RELEASE_PR: ${{ needs.release-please.outputs.release_pr }} + REQUEST_ID: ${{ github.run_id }}-${{ github.run_attempt }} run: | - release_pr="$(jq -r '.[0].number // empty' <<< "${RELEASE_PRS}")" - if [[ -z "${release_pr}" ]]; then - echo "Release Please reported a created PR without a number" >&2 - exit 1 - fi head_sha="$(git rev-parse HEAD)" + recovery="rerun the failed Release Please job to re-check" + post_status() { + gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ + -f "state=$1" -f "context=$2" -f "description=$3" ${4:+-f "target_url=$4"} >/dev/null + } + fail_verify() { + post_status failure Verify "$1; ${recovery}" "${2:-}" + { + echo "### Release pull request checks failed" + echo + echo "$1. After fixing or confirming a transient cause, use **Re-run failed jobs**" + echo "on this workflow run: it re-dispatches every trusted check and repaints Verify." + } >> "${GITHUB_STEP_SUMMARY}" + exit 1 + } # Verify is the required status, so it only succeeds after every trusted check passes, - # including the same-runner comparison with the previous release. + # including the same-runner comparison with the previous release. Each dispatched run + # carries this attempt's request id in its run name and is matched on it exactly. workflows=(pull-request-metadata.yml version-guard.yml release-performance.yml) contexts=("Pull Request Metadata" "Block automatic 1.0 releases" "Release performance") for context in "${contexts[@]}"; do - gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ - -f state=pending -f "context=${context}" \ - -f description="Trusted release check is running" >/dev/null + post_status pending "${context}" "Trusted release check is running" done - gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ - -f state=pending -f context=Verify \ - -f description="Validating trusted release metadata and performance" >/dev/null + post_status pending Verify "Validating trusted release metadata and performance" + dispatched_after="$(date -u -d '-2 minutes' +%Y-%m-%dT%H:%M:%SZ)" gh workflow run pull-request-metadata.yml --ref "${RELEASE_BRANCH}" \ - -f "pr_number=${release_pr}" - gh workflow run version-guard.yml --ref "${RELEASE_BRANCH}" - gh workflow run release-performance.yml --ref "${RELEASE_BRANCH}" + -f "pr_number=${RELEASE_PR}" -f "request_id=${REQUEST_ID}" + gh workflow run version-guard.yml --ref "${RELEASE_BRANCH}" \ + -f "request_id=${REQUEST_ID}" + gh workflow run release-performance.yml --ref "${RELEASE_BRANCH}" \ + -f "request_id=${REQUEST_ID}" for index in "${!workflows[@]}"; do workflow="${workflows[${index}]}" context="${contexts[${index}]}" run_id="" for _ in {1..30}; do run_id="$(gh run list --workflow "${workflow}" --branch "${RELEASE_BRANCH}" \ - --event workflow_dispatch --limit 20 --json databaseId,headSha \ - --jq ".[] | select(.headSha == \"${head_sha}\") | .databaseId" | head -n 1)" + --event workflow_dispatch --limit 50 \ + --json databaseId,headSha,displayTitle,createdAt \ + --jq ".[] | select(.headSha == \"${head_sha}\" + and (.displayTitle | endswith(\" [${REQUEST_ID}]\")) + and .createdAt >= \"${dispatched_after}\") | .databaseId" | head -n 1)" [[ -n "${run_id}" ]] && break sleep 2 done if [[ -z "${run_id}" ]]; then - gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ - -f state=failure -f "context=${context}" \ - -f description="Trusted release check did not start" >/dev/null - gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ - -f state=failure -f context=Verify \ - -f description="${context} did not start" >/dev/null - exit 1 + post_status failure "${context}" "Trusted release check did not start" + fail_verify "${context} did not start" fi run_url="$(gh run view "${run_id}" --json url --jq .url)" if gh run watch "${run_id}" --exit-status --interval 30; then - gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ - -f state=success -f "context=${context}" -f target_url="${run_url}" \ - -f description="Trusted release check passed" >/dev/null + post_status success "${context}" "Trusted release check passed" "${run_url}" else - gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ - -f state=failure -f "context=${context}" -f target_url="${run_url}" \ - -f description="Trusted release check failed" >/dev/null - gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ - -f state=failure -f context=Verify -f target_url="${run_url}" \ - -f description="${context} failed" >/dev/null - exit 1 + post_status failure "${context}" "Trusted release check failed" "${run_url}" + fail_verify "${context} failed" "${run_url}" fi done - unexpected_files="$(gh pr view "${release_pr}" --repo "${GITHUB_REPOSITORY}" \ + unexpected_files="$(gh pr view "${RELEASE_PR}" --repo "${GITHUB_REPOSITORY}" \ --json files --jq '[.files[].path | select(. != ".release-please-manifest.json" and . != "CHANGELOG.md" and . != "pyproject.toml" and . != "uv.lock" and . != "Cargo.lock" and . != "crates/sqlbuild-rules-native/Cargo.toml")] | join("\n")')" if [[ -n "${unexpected_files}" ]]; then echo "Release pull request contains non-metadata changes:" >&2 echo "${unexpected_files}" >&2 - gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ - -f state=failure -f context=Verify \ - -f description="Release pull request contains non-metadata changes" >/dev/null + post_status failure Verify "Release pull request contains non-metadata changes" exit 1 fi - gh api --method POST "repos/${GITHUB_REPOSITORY}/statuses/${head_sha}" \ - -f state=success -f context=Verify \ - -f description="Product changes passed verification before release metadata update" >/dev/null + post_status success Verify \ + "Product changes passed verification before release metadata update" + + # The checks can wait longer than an installation token lives, so mint one just before use. + - name: Create delivery token + id: delivery-token + uses: actions/create-github-app-token@v2 + with: + app-id: ${{ vars.CHIO_DELIVERY_APP_ID }} + private-key: ${{ secrets.CHIO_DELIVERY_PRIVATE_KEY }} + owner: ${{ github.repository_owner }} + repositories: ${{ github.event.repository.name }} + permission-actions: write + permission-contents: write + permission-issues: write + permission-pull-requests: write - name: Enable release pull request auto-merge - if: steps.release.outputs.prs_created == 'true' env: # The delivery app merges, so the resulting push to main starts the tag and publish runs. GH_TOKEN: ${{ steps.delivery-token.outputs.token }} - RELEASE_PRS: ${{ steps.release.outputs.prs }} - run: | - release_pr="$(jq -r '.[0].number // empty' <<< "${RELEASE_PRS}")" - gh pr merge "${release_pr}" --repo "${GITHUB_REPOSITORY}" --auto --squash + RELEASE_PR: ${{ needs.release-please.outputs.release_pr }} + run: gh pr merge "${RELEASE_PR}" --repo "${GITHUB_REPOSITORY}" --auto --squash diff --git a/.github/workflows/version-guard.yml b/.github/workflows/version-guard.yml index 44c6345efe..064bf19e53 100644 --- a/.github/workflows/version-guard.yml +++ b/.github/workflows/version-guard.yml @@ -1,4 +1,7 @@ name: Version Guard +run-name: >- + ${{ inputs.request_id && format('{0} [{1}]', github.workflow, inputs.request_id) + || github.event.pull_request.title || github.event.head_commit.message || github.workflow }} on: pull_request: @@ -8,6 +11,11 @@ on: branches: - main workflow_dispatch: + inputs: + request_id: + description: "Release Please attempt that dispatched this run; leave empty for manual runs" + required: false + type: string permissions: contents: read diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md index 2020f7b224..7df260b12a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md @@ -103,8 +103,9 @@ the medians of wall time, CPU time (user+sys) and peak RSS: A command fails when its candidate median exceeds the baseline by more than 25% and by more than 0.5 s (wall and CPU) or 32 MiB (peak RSS); the limits live in -`scripts/release_performance/constants.py`. The baseline defaults to the highest published or -release-tagged version below the candidate. A freshly tagged baseline is awaited until its wheels +`scripts/release_performance/constants.py`. The baseline defaults to the highest version below +the candidate that is installable from PyPI, or release-tagged but not yet on PyPI; yanked releases +never count. A freshly tagged baseline is awaited until its wheels are on PyPI, installing from the uncompressed simple index's wheel URL when the CDN still lags. A command that needs a feature newer than a compared version declares `minimum_version` and appears as skipped, with the reason, in the job summary; any other failure of either version fails the @@ -112,7 +113,10 @@ check. Release Please dispatches `.github/workflows/release-performance.yml` with the metadata and version checks and posts the required `Verify` status only after it passes, so a regressed release pull -request cannot auto-merge. Run it manually with an optional `baseline` input, or locally: +request cannot auto-merge. Each dispatch carries a request id in its run name, and Release Please +waits only on the runs it started. After a transient failure, use **Re-run failed jobs** on the +Release Please run: it re-dispatches all three checks and repaints `Verify` on the current head. +Run the comparison manually with an optional `baseline` input, or locally: ```bash uv run python -m scripts.compare_release_performance --candidate 0.121.0 --baseline 0.119.1