diff --git a/.github/workflows/release-performance.yml b/.github/workflows/release-performance.yml index 22492427f9..87637a9459 100644 --- a/.github/workflows/release-performance.yml +++ b/.github/workflows/release-performance.yml @@ -21,6 +21,10 @@ on: - .github/workflows/release-performance.yml - scripts/compare_release_performance.py - scripts/release_performance/** + - scripts/compile_performance_ratio/_helpers/edit.py + - scripts/compile_performance_ratio/constants.py + - scripts/compile_performance_ratio/exceptions.py + - scripts/cold_compile_performance/_helpers/dense_project.py permissions: contents: read diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index fff57401b8..7cc298cad3 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -451,13 +451,14 @@ jobs: - name: Install base dependencies working-directory: base run: uv sync --locked --all-extras - - name: Compare base and head cold compiles + - name: Compare base and head cold, warm and one-model edit compiles env: KIND: ${{ matrix.kind }} MAX_RATIO: ${{ matrix.max_ratio }} run: | uv run python -m scripts.compare_compile_performance \ --kind "${KIND}" \ + --modes cold warm edit \ --max-ratio "${MAX_RATIO}" \ --models 3000 \ --base-root base \ diff --git a/scripts/compare_compile_performance.py b/scripts/compare_compile_performance.py index ef851a0327..7a48483bfe 100644 --- a/scripts/compare_compile_performance.py +++ b/scripts/compare_compile_performance.py @@ -6,7 +6,7 @@ def main(argv: list[str] | None = None) -> int: - """Compare base and head cold compiles on the same machine.""" + """Compare base and head compiles on the same machine.""" return compare_compile_performance(argv) diff --git a/scripts/compile_performance_ratio/_helpers/edit.py b/scripts/compile_performance_ratio/_helpers/edit.py new file mode 100644 index 0000000000..894c2cb13a --- /dev/null +++ b/scripts/compile_performance_ratio/_helpers/edit.py @@ -0,0 +1,34 @@ +"""Apply a deterministic one-model edit that any generator's benchmark project accepts.""" + +from __future__ import annotations + +from pathlib import Path + +from scripts.compile_performance_ratio.constants import ( + EDIT_COMMENT, + MODEL_FILE_PATTERN, + MODELS_DIRECTORY, + STATEMENT_START, +) +from scripts.compile_performance_ratio.exceptions import BenchmarkEditError + + +def apply_one_model_edit(*, project_dir: Path, revision: int) -> Path: + """Comment the middle model's query with the revision so each revision is a new edit.""" + + models: list[Path] = sorted( + (project_dir / MODELS_DIRECTORY).rglob(MODEL_FILE_PATTERN), + key=lambda path: path.relative_to(project_dir).as_posix(), + ) + if not models: + raise BenchmarkEditError(f"{project_dir} has no model files to edit") + target: Path = models[len(models) // 2] + edited, count = STATEMENT_START.subn( + EDIT_COMMENT.format(revision=revision) + r"\1", + target.read_text(encoding="utf-8"), + count=1, + ) + if count != 1: + raise BenchmarkEditError(f"{target} has no line starting a WITH or SELECT query to edit") + _ = target.write_text(edited, encoding="utf-8") + return target diff --git a/scripts/compile_performance_ratio/_helpers/measure.py b/scripts/compile_performance_ratio/_helpers/measure.py index 000408bcfc..45123ec561 100644 --- a/scripts/compile_performance_ratio/_helpers/measure.py +++ b/scripts/compile_performance_ratio/_helpers/measure.py @@ -8,15 +8,20 @@ import shutil import statistics import subprocess +import sys import time from pathlib import Path from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project +from scripts.compile_performance_ratio._helpers.edit import apply_one_model_edit from scripts.compile_performance_ratio.constants import ( + ANALYSIS_CACHE_MISSES, BASE_GENERATOR_ENTRY, BASE_LABEL, + COLD_MODE, COMPILE_ENTRY, DENSE_KIND, + EDIT_MODE, ERROR_TAIL_CHARACTERS, EXCLUDED_ENVIRONMENT_KEYS, EXCLUDED_ENVIRONMENT_PREFIX, @@ -27,8 +32,10 @@ FRESH_SOURCE_SHARE, FRESH_TEST_SHARE, HEAD_LABEL, + MODE_TITLES, PYTHONPATH_KEY, REPORTED_PHASES, + WARM_MODE, ) from scripts.compile_performance_ratio.exceptions import CompileComparisonError from scripts.compile_performance_ratio.models import CompileComparison, CompileRun @@ -103,26 +110,57 @@ def compare_builds( base_python: Path, head_python: Path, runs: int, -) -> CompileComparison: - """Alternate base and head cold compiles after one untimed warm-up compile per build.""" + modes: tuple[str, ...], +) -> tuple[CompileComparison, ...]: + """Alternate base and head compiles per mode; warm and edit reuse the cold project.""" builds: tuple[tuple[str, Path, Path], ...] = ( (BASE_LABEL, base_python, base_project_dir), (HEAD_LABEL, head_python, head_project_dir), ) - for label, python, project_dir in builds: - _compile_once(label=label, python=python, project_dir=project_dir) - results: dict[str, list[CompileRun]] = {BASE_LABEL: [], HEAD_LABEL: []} - for _ in range(runs): - for label, python, project_dir in builds: - results[label].append( - _compile_once(label=label, python=python, project_dir=project_dir) + comparisons: list[CompileComparison] = [] + cache_primed: bool = False + for mode in modes: + print(f"Comparing {MODE_TITLES[mode]} over {runs} alternating runs", file=sys.stderr) + if mode == COLD_MODE: + for label, python, project_dir in builds: + _ = _compile_once(label=label, python=python, project_dir=project_dir, mode=mode) + cache_primed = False + elif not cache_primed: + for label, python, project_dir in builds: + _ = _compile_once( + label=label, python=python, project_dir=project_dir, mode=WARM_MODE + ) + cache_primed = True + results: dict[str, list[CompileRun]] = {BASE_LABEL: [], HEAD_LABEL: []} + for revision in range(runs): + for label, python, project_dir in builds: + if mode == EDIT_MODE: + _ = apply_one_model_edit(project_dir=project_dir, revision=revision) + run: CompileRun = _compile_once( + label=label, python=python, project_dir=project_dir, mode=mode + ) + _check_cache_use(run=run, mode=mode) + results[label].append(run) + comparisons.append( + _comparison( + kind=kind, + models=models, + mode=mode, + base=results[BASE_LABEL], + head=results[HEAD_LABEL], ) - base: list[CompileRun] = results[BASE_LABEL] - head: list[CompileRun] = results[HEAD_LABEL] + ) + return tuple(comparisons) + + +def _comparison( + *, kind: str, models: int, mode: str, base: list[CompileRun], head: list[CompileRun] +) -> CompileComparison: return CompileComparison( kind=kind, models=models, + mode=mode, base_wall_seconds=statistics.median(run.wall_seconds for run in base), head_wall_seconds=statistics.median(run.wall_seconds for run in head), base_cpu_seconds=statistics.median(run.cpu_seconds for run in base), @@ -132,8 +170,27 @@ def compare_builds( ) -def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun: - shutil.rmtree(project_dir / "target", ignore_errors=True) +def _check_cache_use(*, run: CompileRun, mode: str) -> None: + misses: int | None = run.analysis_cache_misses + if misses is None or mode == COLD_MODE: + return + if mode == WARM_MODE and misses != 0: + raise CompileComparisonError( + f"{run.label} warm compile missed the analysis cache for {misses} models, so it " + "did not measure an unchanged warm compile" + ) + if mode == EDIT_MODE and misses == 0: + raise CompileComparisonError( + f"{run.label} one-model edit compile reported no analysis cache miss, so the edit " + "did not invalidate the edited model" + ) + + +def _compile_once(*, label: str, python: Path, project_dir: Path, mode: str) -> CompileRun: + cache_args: tuple[str, ...] = () + if mode == COLD_MODE: + shutil.rmtree(project_dir / "target", ignore_errors=True) + cache_args = ("--no-cache",) environment: dict[str, str] = _compile_environment() before: resource.struct_rusage = resource.getrusage(resource.RUSAGE_CHILDREN) started: float = time.perf_counter() @@ -147,7 +204,7 @@ def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun: "--no-color", "compile", "--json", - "--no-cache", + *cache_args, ], capture_output=True, text=True, @@ -158,20 +215,22 @@ def _compile_once(*, label: str, python: Path, project_dir: Path) -> CompileRun: after: resource.struct_rusage = resource.getrusage(resource.RUSAGE_CHILDREN) if completed.returncode != 0: raise CompileComparisonError( - f"{label} compile failed with exit {completed.returncode}: " + f"{label} {mode} compile failed with exit {completed.returncode}: " f"{completed.stderr[-ERROR_TAIL_CHARACTERS:]}" ) payload: dict[str, object] = json.loads(completed.stdout) timings: object = payload.get("compile_timings", {}) + numeric: dict[str, int] = { + str(name): int(value) + for name, value in (timings.items() if isinstance(timings, dict) else ()) + if isinstance(value, (int, float)) + } return CompileRun( label=label, wall_seconds=wall_seconds, cpu_seconds=(after.ru_utime - before.ru_utime) + (after.ru_stime - before.ru_stime), - timings_ms={ - name: int(value) - for name, value in (timings.items() if isinstance(timings, dict) else ()) - if name in REPORTED_PHASES and isinstance(value, (int, float)) - }, + timings_ms={name: value for name, value in numeric.items() if name in REPORTED_PHASES}, + analysis_cache_misses=numeric.get(ANALYSIS_CACHE_MISSES), ) diff --git a/scripts/compile_performance_ratio/_helpers/report.py b/scripts/compile_performance_ratio/_helpers/report.py index 7324d7d66a..a92535a050 100644 --- a/scripts/compile_performance_ratio/_helpers/report.py +++ b/scripts/compile_performance_ratio/_helpers/report.py @@ -5,39 +5,54 @@ import platform from pathlib import Path -from scripts.compile_performance_ratio.constants import REPORTED_PHASES +from scripts.compile_performance_ratio.constants import MODE_TITLES, REPORTED_PHASES from scripts.compile_performance_ratio.models import CompileComparison def comparison_markdown( - *, comparison: CompileComparison, runs: int, max_ratio: float, per_side_projects: bool + *, + comparisons: tuple[CompileComparison, ...], + runs: int, + max_ratio: float, + per_side_projects: bool, + failures: tuple[str, ...], ) -> str: - """Summarize medians, ratios and per-phase timings as a Markdown table.""" + """Summarize medians, ratios and per-phase timings of every mode as Markdown tables.""" generation: str = ( "Projects generated per side: base with the base checkout's generator, head with head's." if per_side_projects else "One project generated with head's generator and compiled by both builds." ) - lines: list[str] = [ - f"### {comparison.kind} {comparison.models} models: head vs base (same runner)", - "", - f"Runner CPU: {cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.", - "", - generation, - "", - "| Metric | Base | Head | Ratio |", - "|---|---:|---:|---:|", - f"| Wall (s) | {comparison.base_wall_seconds:.2f} | " - f"{comparison.head_wall_seconds:.2f} | {comparison.wall_ratio:.3f} |", - f"| CPU (s) | {comparison.base_cpu_seconds:.2f} | " - f"{comparison.head_cpu_seconds:.2f} | {comparison.cpu_ratio:.3f} |", - ] - for phase in REPORTED_PHASES: - base: float = comparison.base_timings_ms.get(phase, 0) - head: float = comparison.head_timings_ms.get(phase, 0) - ratio: str = f"{head / base:.3f}" if base else "n/a" - lines.append(f"| {phase} | {base:.0f} | {head:.0f} | {ratio} |") + lines: list[str] = [] + for comparison in comparisons: + lines.extend( + ( + f"### {comparison.kind} {comparison.models} models, " + f"{MODE_TITLES[comparison.mode]}: head vs base (same runner)", + "", + f"Runner CPU: {cpu_model()}; {runs} alternating runs each; limit {max_ratio:.2f}x.", + "", + generation, + "", + "| Metric | Base | Head | Ratio |", + "|---|---:|---:|---:|", + f"| Wall (s) | {comparison.base_wall_seconds:.2f} | " + f"{comparison.head_wall_seconds:.2f} | {comparison.wall_ratio:.3f} |", + f"| CPU (s) | {comparison.base_cpu_seconds:.2f} | " + f"{comparison.head_cpu_seconds:.2f} | {comparison.cpu_ratio:.3f} |", + ) + ) + for phase in REPORTED_PHASES: + base: float = comparison.base_timings_ms.get(phase, 0) + head: float = comparison.head_timings_ms.get(phase, 0) + ratio: str = f"{head / base:.3f}" if base else "n/a" + lines.append(f"| {phase} | {base:.0f} | {head:.0f} | {ratio} |") + lines.append("") + if failures: + lines.append("**Result: failed.** " + "; ".join(failures) + ".") + else: + lines.append("**Result: passed.** Every mode is within the same-runner limit.") return "\n".join(lines) + "\n" diff --git a/scripts/compile_performance_ratio/_helpers/verdict.py b/scripts/compile_performance_ratio/_helpers/verdict.py new file mode 100644 index 0000000000..1caa0296b7 --- /dev/null +++ b/scripts/compile_performance_ratio/_helpers/verdict.py @@ -0,0 +1,44 @@ +"""Decide whether head compiled slower than base beyond the same-runner ratio limit.""" + +from __future__ import annotations + +import math + +from scripts.compile_performance_ratio.models import CompileComparison + + +def ratio_failures( + *, comparisons: tuple[CompileComparison, ...], modes: tuple[str, ...], max_ratio: float +) -> tuple[str, ...]: + """Describe every requested mode that is unmeasured or whose wall or CPU ratio is too high.""" + + measured: dict[str, CompileComparison] = { + comparison.mode: comparison for comparison in comparisons + } + failures: list[str] = [] + for mode in modes: + comparison: CompileComparison | None = measured.get(mode) + if comparison is None: + failures.append(f"{mode}: no measurement") + continue + prefix: str = f"{comparison.kind} {comparison.models} {mode}" + for metric, base, head, ratio in ( + ( + "wall", + comparison.base_wall_seconds, + comparison.head_wall_seconds, + comparison.wall_ratio, + ), + ("CPU", comparison.base_cpu_seconds, comparison.head_cpu_seconds, comparison.cpu_ratio), + ): + if not _measured(base) or not _measured(head): + failures.append( + f"{prefix}: {metric} measurement missing (base {base:.2f} s, head {head:.2f} s)" + ) + elif ratio > max_ratio: + failures.append(f"{prefix}: {metric} ratio {ratio:.3f} exceeds {max_ratio:.2f}") + return tuple(failures) + + +def _measured(seconds: float) -> bool: + return math.isfinite(seconds) and seconds > 0 diff --git a/scripts/compile_performance_ratio/constants.py b/scripts/compile_performance_ratio/constants.py index 3698f796a3..00b18fbd69 100644 --- a/scripts/compile_performance_ratio/constants.py +++ b/scripts/compile_performance_ratio/constants.py @@ -1,10 +1,26 @@ """Stable constants for same-runner compile performance comparisons.""" +import re + DEFAULT_RUNS: int = 3 DEFAULT_MAX_RATIO: float = 1.10 DENSE_KIND: str = "dense" FRESH_KIND: str = "fresh" PROJECT_KINDS: tuple[str, ...] = (DENSE_KIND, FRESH_KIND) +COLD_MODE: str = "cold" +WARM_MODE: str = "warm" +EDIT_MODE: str = "edit" +COMPILE_MODES: tuple[str, ...] = (COLD_MODE, WARM_MODE, EDIT_MODE) +MODE_TITLES: dict[str, str] = { + COLD_MODE: "cold compile without cache", + WARM_MODE: "unchanged warm compile", + EDIT_MODE: "one-model edit on a warm cache", +} +ANALYSIS_CACHE_MISSES: str = "analysis_cache_misses" +MODELS_DIRECTORY: str = "models" +MODEL_FILE_PATTERN: str = "*.sql" +EDIT_COMMENT: str = "-- Benchmark edit {revision}.\n" +STATEMENT_START: re.Pattern[str] = re.compile(r"^(WITH|SELECT)\b", re.MULTILINE) BASE_LABEL: str = "base" HEAD_LABEL: str = "head" EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset({"VIRTUAL_ENV"}) diff --git a/scripts/compile_performance_ratio/exceptions.py b/scripts/compile_performance_ratio/exceptions.py index 9b5b24f93e..548a49d0a7 100644 --- a/scripts/compile_performance_ratio/exceptions.py +++ b/scripts/compile_performance_ratio/exceptions.py @@ -3,3 +3,7 @@ class CompileComparisonError(RuntimeError): """Raised when a compared build fails to compile the benchmark project.""" + + +class BenchmarkEditError(RuntimeError): + """Raised when a benchmark project has no model query the one-model edit can change.""" diff --git a/scripts/compile_performance_ratio/main/compare.py b/scripts/compile_performance_ratio/main/compare.py index a38382e0f3..e8e29d1fed 100644 --- a/scripts/compile_performance_ratio/main/compare.py +++ b/scripts/compile_performance_ratio/main/compare.py @@ -14,7 +14,9 @@ write_benchmark_project, ) from scripts.compile_performance_ratio._helpers.report import append_summary, comparison_markdown +from scripts.compile_performance_ratio._helpers.verdict import ratio_failures from scripts.compile_performance_ratio.constants import ( + COMPILE_MODES, DEFAULT_MAX_RATIO, DEFAULT_RUNS, PROJECT_KINDS, @@ -23,7 +25,7 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: - """Compare base and head cold compiles and enforce the wall and CPU ratio limit.""" + """Compare base and head compiles per mode and enforce the wall and CPU ratio limit.""" args: argparse.Namespace = _parse_args(argv) summary_value: str | None = os.environ.get("GITHUB_STEP_SUMMARY") @@ -45,8 +47,8 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: project_dir=base_project_dir, models=args.models, ) - print(f"Comparing base and head over {args.runs} alternating runs", file=sys.stderr) - comparison: CompileComparison = compare_builds( + modes: tuple[str, ...] = tuple(dict.fromkeys(args.modes)) + comparisons: tuple[CompileComparison, ...] = compare_builds( kind=args.kind, models=args.models, base_project_dir=base_project_dir, @@ -54,22 +56,22 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: base_python=args.base_python, head_python=args.head_python, runs=args.runs, + modes=modes, ) + failures: tuple[str, ...] = ratio_failures( + comparisons=comparisons, modes=modes, max_ratio=args.max_ratio + ) markdown: str = comparison_markdown( - comparison=comparison, + comparisons=comparisons, runs=args.runs, max_ratio=args.max_ratio, per_side_projects=per_side_projects, + failures=failures, ) print(markdown) _ = append_summary(path=Path(summary_value) if summary_value else None, markdown=markdown) - exceeded: list[str] = [ - f"{name} ratio {ratio:.3f} exceeds {args.max_ratio:.2f}" - for name, ratio in (("wall", comparison.wall_ratio), ("CPU", comparison.cpu_ratio)) - if ratio > args.max_ratio - ] - if exceeded: - print("Compile performance regression: " + "; ".join(exceeded), file=sys.stderr) + if failures: + print("Compile performance regression: " + "; ".join(failures), file=sys.stderr) return 1 print("Compile performance within the same-runner limit", file=sys.stderr) return 0 @@ -77,7 +79,7 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: def _parse_args(argv: list[str] | None) -> argparse.Namespace: parser: argparse.ArgumentParser = argparse.ArgumentParser( - description="Compare base and head cold compiles on the same machine." + description="Compare base and head cold, warm and one-model edit compiles on one machine." ) parser.add_argument("--kind", choices=PROJECT_KINDS, required=True) parser.add_argument("--models", type=int, required=True) @@ -89,6 +91,13 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: default=None, help="Base checkout whose own generator writes the project the base build compiles.", ) + parser.add_argument( + "--modes", + nargs="+", + choices=COMPILE_MODES, + default=list(COMPILE_MODES), + help="Compile modes to compare, in order; warm and edit reuse the cold project.", + ) parser.add_argument("--runs", type=int, default=DEFAULT_RUNS) parser.add_argument("--max-ratio", type=float, default=DEFAULT_MAX_RATIO) return parser.parse_args(argv) diff --git a/scripts/compile_performance_ratio/models.py b/scripts/compile_performance_ratio/models.py index b42166aaee..b4f503c470 100644 --- a/scripts/compile_performance_ratio/models.py +++ b/scripts/compile_performance_ratio/models.py @@ -11,12 +11,14 @@ class CompileRun: wall_seconds: float cpu_seconds: float timings_ms: dict[str, int] + analysis_cache_misses: int | None = None @dataclass(frozen=True) class CompileComparison: kind: str models: int + mode: str base_wall_seconds: float head_wall_seconds: float base_cpu_seconds: float @@ -26,8 +28,12 @@ class CompileComparison: @property def wall_ratio(self) -> float: + if self.base_wall_seconds <= 0: + return float("inf") return self.head_wall_seconds / self.base_wall_seconds @property def cpu_ratio(self) -> float: + if self.base_cpu_seconds <= 0: + return float("inf") return self.head_cpu_seconds / self.base_cpu_seconds diff --git a/scripts/release_performance/_helpers/benchmark.py b/scripts/release_performance/_helpers/benchmark.py index 7b12925a98..5812031eb3 100644 --- a/scripts/release_performance/_helpers/benchmark.py +++ b/scripts/release_performance/_helpers/benchmark.py @@ -11,6 +11,8 @@ from pathlib import Path from typing import cast +from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project +from scripts.compile_performance_ratio._helpers.edit import apply_one_model_edit from scripts.compile_performance_ratio._helpers.measure import run_checkout_generator from scripts.release_performance.constants import ( BASELINE_GENERATOR_ENTRY, @@ -18,12 +20,16 @@ BUILD_WARMUPS, COMMAND_TIMEOUT_SECONDS, COMPILE_COMMAND, + DENSE_GENERATOR_ENTRY, + DENSE_PROJECT, + DENSE_WARMUPS, EXCLUDED_ENVIRONMENT_KEYS, EXCLUDED_ENVIRONMENT_PREFIX, INSPECTION_PROJECT, INSPECTION_WARMUPS, PRISTINE_DIRECTORY, STDERR_TAIL_CHARACTERS, + TARGET_DIRECTORY, TIME_FORMAT, ) from scripts.release_performance.exceptions import ReleasePerformanceError @@ -58,33 +64,48 @@ def write_baseline_pristine_projects( ) -> Path: """Generate the baseline's benchmarks with the baseline source's own generator.""" - completed: subprocess.CompletedProcess[str] = run_checkout_generator( - checkout=source, - python=Path(sys.executable).absolute(), + _run_baseline_generator( + source=source, entry=BASELINE_GENERATOR_ENTRY, arguments=(str(root), str(inspection_models), str(build_models)), - environment=_environment(), + description="benchmark", ) - if completed.returncode != 0: - raise ReleasePerformanceError( - f"Baseline benchmark generation with the generator in {source} failed with exit " - f"{completed.returncode}:\n{completed.stderr[-STDERR_TAIL_CHARACTERS:]}" - ) return root / PRISTINE_DIRECTORY +def write_dense_project(*, pristine: Path, models: int) -> None: + """Generate the dense all-rules benchmark beside the other pristine projects.""" + + write_dense_compile_project(project_dir=pristine / DENSE_PROJECT, model_count=models) + + +def write_baseline_dense_project(*, source: Path, pristine: Path, models: int) -> None: + """Generate the baseline's dense benchmark with the baseline source's own generator.""" + + _run_baseline_generator( + source=source, + entry=DENSE_GENERATOR_ENTRY, + arguments=(str(pristine / DENSE_PROJECT), str(models)), + description="dense benchmark", + ) + + def prepare_version_projects( *, version: InstalledVersion, pristine: Path, root: Path ) -> dict[str, Path]: """Copy the pristine projects for one version and warm its caches with untimed runs.""" + project_warmups: tuple[tuple[str, tuple[tuple[str, ...], ...]], ...] = ( + (INSPECTION_PROJECT, INSPECTION_WARMUPS), + (DENSE_PROJECT, DENSE_WARMUPS), + (BUILD_PROJECT, BUILD_WARMUPS), + ) projects: dict[str, Path] = { - INSPECTION_PROJECT: root / version.label / INSPECTION_PROJECT, - BUILD_PROJECT: root / version.label / BUILD_PROJECT, + name: root / version.label / name for name, _warmups in project_warmups } logs: Path = root / version.label / "logs" logs.mkdir(parents=True, exist_ok=True) - for name, warmups in ((INSPECTION_PROJECT, INSPECTION_WARMUPS), (BUILD_PROJECT, BUILD_WARMUPS)): + for name, warmups in project_warmups: _ = shutil.copytree(pristine / name, projects[name]) for index, sqb_args in enumerate(warmups): output: Path = _run_sqb( @@ -117,6 +138,10 @@ def compare_command( project_dir: Path = projects[version.label][command.project] if command.project == BUILD_PROJECT: _reset_build_project(project_dir=project_dir) + if command.removes_target: + shutil.rmtree(project_dir / TARGET_DIRECTORY, ignore_errors=True) + if command.edits_model: + _ = apply_one_model_edit(project_dir=project_dir, revision=run) sample: CommandSample = _run_sqb( version=version, project_dir=project_dir, @@ -136,9 +161,27 @@ def compare_command( name=command.name, baseline=tuple(samples[baseline.label]), candidate=tuple(samples[candidate.label]), + max_time_ratio=command.max_time_ratio, ) +def _run_baseline_generator( + *, source: Path, entry: str, arguments: tuple[str, ...], description: str +) -> None: + completed: subprocess.CompletedProcess[str] = run_checkout_generator( + checkout=source, + python=Path(sys.executable).absolute(), + entry=entry, + arguments=arguments, + environment=_environment(), + ) + if completed.returncode != 0: + raise ReleasePerformanceError( + f"Baseline {description} generation with the generator in {source} failed with exit " + f"{completed.returncode}:\n{completed.stderr[-STDERR_TAIL_CHARACTERS:]}" + ) + + def _reset_build_project(*, project_dir: Path) -> None: shutil.rmtree(project_dir, ignore_errors=True) _ = shutil.copytree(project_dir.with_name(BUILD_PROJECT + _TEMPLATE_SUFFIX), project_dir) diff --git a/scripts/release_performance/_helpers/report.py b/scripts/release_performance/_helpers/report.py index 2d09f050d1..3cf7120652 100644 --- a/scripts/release_performance/_helpers/report.py +++ b/scripts/release_performance/_helpers/report.py @@ -6,6 +6,7 @@ from scripts.release_performance._helpers.verdict import metric_verdicts, regression_messages from scripts.release_performance.constants import ( + MAX_COMPILE_TIME_RATIO, MAX_CPU_RATIO, MAX_RSS_RATIO, MAX_WALL_RATIO, @@ -37,7 +38,8 @@ def comparison_markdown(*, comparison: ReleaseComparison) -> str: f"Limits: wall {_percent(MAX_WALL_RATIO)} (ignored under " f"{MIN_WALL_REGRESSION_SECONDS:g} s), CPU {_percent(MAX_CPU_RATIO)} (ignored under " f"{MIN_CPU_REGRESSION_SECONDS:g} s), peak RSS {_percent(MAX_RSS_RATIO)} (ignored under " - f"{MIN_RSS_REGRESSION_BYTES // _MIB} MiB).", + f"{MIN_RSS_REGRESSION_BYTES // _MIB} MiB). Cold, warm and one-model edit compiles use " + f"{_percent(MAX_COMPILE_TIME_RATIO)} for wall and CPU.", "", "| Command | Wall (s) | Wall ratio | CPU (s) | CPU ratio | Peak RSS (MiB) | RSS ratio " "| Result |", diff --git a/scripts/release_performance/_helpers/verdict.py b/scripts/release_performance/_helpers/verdict.py index 23a6fb0777..054deaed0b 100644 --- a/scripts/release_performance/_helpers/verdict.py +++ b/scripts/release_performance/_helpers/verdict.py @@ -21,17 +21,18 @@ def metric_verdicts(*, comparison: CommandComparison) -> tuple[MetricVerdict, ...]: - """Compare median wall and CPU time and worst-run peak RSS of one command.""" + """Compare median wall and CPU time, against the command's own time ratio if set, and RSS.""" baseline: CommandSample = summary_sample(samples=comparison.baseline) candidate: CommandSample = summary_sample(samples=comparison.candidate) + time_ratio: float | None = comparison.max_time_ratio return ( _verdict( command=comparison.name, metric="wall", baseline=baseline.wall_seconds, candidate=candidate.wall_seconds, - max_ratio=MAX_WALL_RATIO, + max_ratio=time_ratio or MAX_WALL_RATIO, floor=MIN_WALL_REGRESSION_SECONDS, ), _verdict( @@ -39,7 +40,7 @@ def metric_verdicts(*, comparison: CommandComparison) -> tuple[MetricVerdict, .. metric="CPU", baseline=baseline.cpu_seconds, candidate=candidate.cpu_seconds, - max_ratio=MAX_CPU_RATIO, + max_ratio=time_ratio or MAX_CPU_RATIO, floor=MIN_CPU_REGRESSION_SECONDS, ), _verdict( diff --git a/scripts/release_performance/_helpers/workflow.py b/scripts/release_performance/_helpers/workflow.py index 6258cf6359..92f7ec7f1d 100644 --- a/scripts/release_performance/_helpers/workflow.py +++ b/scripts/release_performance/_helpers/workflow.py @@ -10,10 +10,13 @@ from pathlib import Path from scripts.compile_performance_ratio._helpers.report import cpu_model +from scripts.compile_performance_ratio.exceptions import BenchmarkEditError from scripts.release_performance._helpers.benchmark import ( compare_command, prepare_version_projects, + write_baseline_dense_project, write_baseline_pristine_projects, + write_dense_project, write_pristine_projects, ) from scripts.release_performance._helpers.report import ( @@ -67,7 +70,7 @@ def run_release_comparison(*, options: ComparisonOptions) -> int: else: with tempfile.TemporaryDirectory(prefix="sqlbuild-release-performance-") as root: comparison = _compare(options=options, root=Path(root)) - except (ReleasePerformanceError, OSError) as error: + except (ReleasePerformanceError, BenchmarkEditError, OSError) as error: print(f"Release performance comparison failed: {error}", file=sys.stderr) _ = append_summary(path=summary, markdown=error_markdown(message=str(error))) return 1 @@ -122,21 +125,26 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: if options.baseline_source is not None else RELEASE_TAG_PREFIX + baseline.version ) + write_dense_project(pristine=pristine[CANDIDATE_LABEL], models=options.dense_models) print(f"Generating baseline benchmark projects with {baseline_generator}", file=sys.stderr) + baseline_source: Path = ( + options.baseline_source.absolute() + if options.baseline_source is not None + else release_source( + version=baseline.version, + repo_dir=REPO_ROOT, + destination=root / BASELINE_SOURCE_DIRECTORY, + ) + ) pristine[BASELINE_LABEL] = write_baseline_pristine_projects( - source=( - options.baseline_source.absolute() - if options.baseline_source is not None - else release_source( - version=baseline.version, - repo_dir=REPO_ROOT, - destination=root / BASELINE_SOURCE_DIRECTORY, - ) - ), + source=baseline_source, root=root / BASELINE_GENERATED_DIRECTORY, inspection_models=options.inspection_models, build_models=options.build_models, ) + write_baseline_dense_project( + source=baseline_source, pristine=pristine[BASELINE_LABEL], models=options.dense_models + ) versions: tuple[InstalledVersion, InstalledVersion] = (baseline, candidate) projects: dict[str, dict[str, Path]] = {} for version in versions: diff --git a/scripts/release_performance/constants.py b/scripts/release_performance/constants.py index 95301a25ba..3f766fdcce 100644 --- a/scripts/release_performance/constants.py +++ b/scripts/release_performance/constants.py @@ -11,6 +11,7 @@ MAX_WALL_RATIO: float = 1.25 MAX_CPU_RATIO: float = 1.25 MAX_RSS_RATIO: float = 1.25 +MAX_COMPILE_TIME_RATIO: float = 1.15 MIN_WALL_REGRESSION_SECONDS: float = 0.5 MIN_CPU_REGRESSION_SECONDS: float = 0.5 MIN_RSS_REGRESSION_BYTES: int = 32 * 1024 * 1024 @@ -19,6 +20,7 @@ DEFAULT_PYTHON: str = "3.12" DEFAULT_INSPECTION_MODELS: int = 3_000 DEFAULT_BUILD_MODELS: int = 1_000 +DEFAULT_DENSE_MODELS: int = 3_000 COMMAND_TIMEOUT_SECONDS: float = 900.0 PACKAGE_NAME: str = "sqlbuild" @@ -37,12 +39,14 @@ CANDIDATE_LABEL: str = "candidate" INSPECTION_PROJECT: str = "inspection" BUILD_PROJECT: str = "build" +DENSE_PROJECT: str = "dense" EXCLUDED_ENVIRONMENT_KEYS: frozenset[str] = frozenset({"VIRTUAL_ENV", "GITHUB_STEP_SUMMARY"}) EXCLUDED_ENVIRONMENT_PREFIX: str = "DBT_" TIME_FORMAT: str = "%e %M %U %S" COMPILE_COMMAND: str = "compile" STDERR_TAIL_CHARACTERS: int = 2_000 PRISTINE_DIRECTORY: str = "pristine" +TARGET_DIRECTORY: str = "target" BASELINE_GENERATED_DIRECTORY: str = "baseline-generated" BASELINE_SOURCE_DIRECTORY: str = "baseline-source" RELEASE_TAG_PREFIX: str = "v" @@ -52,17 +56,25 @@ "write_pristine_projects(" "root=Path(sys.argv[1]), inspection_models=int(sys.argv[2]), build_models=int(sys.argv[3]))" ) +DENSE_GENERATOR_ENTRY: str = ( + "import sys; from pathlib import Path; " + "from scripts.cold_compile_performance._helpers.dense_project import " + "write_dense_compile_project; " + "write_dense_compile_project(project_dir=Path(sys.argv[1]), model_count=int(sys.argv[2]))" +) BENCHMARK_COMMANDS: tuple[BenchmarkCommand, ...] = ( BenchmarkCommand( name="compile (no cache)", project=INSPECTION_PROJECT, sqb_args=("compile", "--json", "--no-cache"), + max_time_ratio=MAX_COMPILE_TIME_RATIO, ), BenchmarkCommand( name="compile (warm cache)", project=INSPECTION_PROJECT, sqb_args=("compile", "--json"), + max_time_ratio=MAX_COMPILE_TIME_RATIO, ), BenchmarkCommand( name="plan --json", @@ -101,6 +113,33 @@ "--include-nearby", ), ), + BenchmarkCommand( + name="compile (one-model edit)", + project=INSPECTION_PROJECT, + sqb_args=("compile", "--json"), + edits_model=True, + max_time_ratio=MAX_COMPILE_TIME_RATIO, + ), + BenchmarkCommand( + name="dense compile (warm cache)", + project=DENSE_PROJECT, + sqb_args=("compile", "--json"), + max_time_ratio=MAX_COMPILE_TIME_RATIO, + ), + BenchmarkCommand( + name="dense compile (one-model edit)", + project=DENSE_PROJECT, + sqb_args=("compile", "--json"), + edits_model=True, + max_time_ratio=MAX_COMPILE_TIME_RATIO, + ), + BenchmarkCommand( + name="dense compile (no cache)", + project=DENSE_PROJECT, + sqb_args=("compile", "--json", "--no-cache"), + removes_target=True, + max_time_ratio=MAX_COMPILE_TIME_RATIO, + ), BenchmarkCommand( name="build (empty warehouse)", project=BUILD_PROJECT, @@ -114,3 +153,4 @@ ("scope", f"model:{SHARED_DIAMOND_ROLLUP}", "--json"), ) BUILD_WARMUPS: tuple[tuple[str, ...], ...] = ((COMPILE_COMMAND, "--json"),) +DENSE_WARMUPS: tuple[tuple[str, ...], ...] = ((COMPILE_COMMAND, "--json"),) diff --git a/scripts/release_performance/main/compare.py b/scripts/release_performance/main/compare.py index d9993e8868..7592549490 100644 --- a/scripts/release_performance/main/compare.py +++ b/scripts/release_performance/main/compare.py @@ -8,6 +8,7 @@ from scripts.release_performance._helpers.workflow import run_release_comparison from scripts.release_performance.constants import ( DEFAULT_BUILD_MODELS, + DEFAULT_DENSE_MODELS, DEFAULT_INSPECTION_MODELS, DEFAULT_PYTHON, DEFAULT_RUNS, @@ -39,6 +40,7 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions: parser.add_argument("--python", default=DEFAULT_PYTHON) parser.add_argument("--inspection-models", type=int, default=DEFAULT_INSPECTION_MODELS) parser.add_argument("--build-models", type=int, default=DEFAULT_BUILD_MODELS) + parser.add_argument("--dense-models", type=int, default=DEFAULT_DENSE_MODELS) parser.add_argument( "--baseline-source", type=Path, @@ -60,6 +62,7 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions: python=args.python, inspection_models=args.inspection_models, build_models=args.build_models, + dense_models=args.dense_models, work_dir=args.work_dir, output=args.output, baseline_source=args.baseline_source, diff --git a/scripts/release_performance/models.py b/scripts/release_performance/models.py index 1b5ea3a827..fa3af73d32 100644 --- a/scripts/release_performance/models.py +++ b/scripts/release_performance/models.py @@ -12,6 +12,9 @@ class BenchmarkCommand: project: str sqb_args: tuple[str, ...] minimum_version: str | None = None + edits_model: bool = False + removes_target: bool = False + max_time_ratio: float | None = None @dataclass(frozen=True) @@ -33,6 +36,7 @@ class CommandComparison: name: str baseline: tuple[CommandSample, ...] candidate: tuple[CommandSample, ...] + max_time_ratio: float | None = None @dataclass(frozen=True) @@ -83,6 +87,7 @@ class ComparisonOptions: python: str inspection_models: int build_models: int + dense_models: int work_dir: Path | None output: Path | None baseline_source: Path | None = None diff --git a/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py b/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py index 407b597179..3d2d6708e6 100644 --- a/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py +++ b/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py @@ -33,7 +33,15 @@ models=20, max_ratio="3.0", expected_return_code=0, - expected_fragments=("| Wall (s) |", "| CPU (s) |", "| analysis_native_ms |"), + expected_fragments=( + "dense 20 models, cold compile without cache: head vs base", + "dense 20 models, unchanged warm compile: head vs base", + "dense 20 models, one-model edit on a warm cache: head vs base", + "| Wall (s) |", + "| CPU (s) |", + "| analysis_native_ms |", + "**Result: passed.**", + ), ), CompilePerformanceRatioTestCase( description="an unattainable limit fails with the exceeded ratios", @@ -41,7 +49,12 @@ models=20, max_ratio="0.01", expected_return_code=1, - expected_fragments=("Compile performance regression: wall ratio",), + expected_fragments=( + "Compile performance regression: dense 20 cold: wall ratio", + "dense 20 warm: CPU ratio", + "dense 20 edit: wall ratio", + "**Result: failed.**", + ), ), ), ids=lambda case: case.description, @@ -90,8 +103,9 @@ def test_given_two_builds_when_comparing_compile_performance_then_enforces_ratio base_generator=MARKER_GENERATOR, expected_return_code=0, expected_fragments=("Projects generated per side",), - expected_base_projects=("base", "base"), - expected_head_projects=("head", "head"), + # Cold warm-up and run, warm cache priming and run, then one edit run per side. + expected_base_projects=("base",) * 5, + expected_head_projects=("head",) * 5, ), PerSideCompilePerformanceRatioTestCase( description="a failing base generator stops before any compile", diff --git a/tests/e2e/scripts/release_performance/_test_types.py b/tests/e2e/scripts/release_performance/_test_types.py index 8143febd53..b599708150 100644 --- a/tests/e2e/scripts/release_performance/_test_types.py +++ b/tests/e2e/scripts/release_performance/_test_types.py @@ -10,6 +10,7 @@ class ReleasePerformanceTestCase: description: str inspection_models: int build_models: int + dense_models: int expected_outcomes: tuple[tuple[int, bool], ...] expected_fragments: tuple[str, ...] expected_marked_projects: tuple[str, ...] diff --git a/tests/e2e/scripts/release_performance/helpers.py b/tests/e2e/scripts/release_performance/helpers.py index 46729069b3..4ac62a6c50 100644 --- a/tests/e2e/scripts/release_performance/helpers.py +++ b/tests/e2e/scripts/release_performance/helpers.py @@ -19,6 +19,15 @@ def write_pristine_projects(*, root: Path, inspection_models: int, build_models: (project / "BASELINE_MARKER").write_text("baseline", encoding="utf-8") return pristine """ +_DENSE_MARKING_WRAPPER: str = """ + +_unmarked_write_dense_compile_project = write_dense_compile_project + + +def write_dense_compile_project(*, project_dir: Path, model_count: int) -> None: + _unmarked_write_dense_compile_project(project_dir=project_dir, model_count=model_count) + (project_dir / "BASELINE_MARKER").write_text("baseline", encoding="utf-8") +""" def write_marked_baseline_source(*, repo_root: Path, destination: Path) -> Path: @@ -32,4 +41,10 @@ def write_marked_baseline_source(*, repo_root: Path, destination: Path) -> Path: _ = benchmark.write_text( benchmark.read_text(encoding="utf-8") + _MARKING_WRAPPER, encoding="utf-8" ) + dense: Path = ( + destination / "scripts" / "cold_compile_performance" / "_helpers" / "dense_project.py" + ) + _ = dense.write_text( + dense.read_text(encoding="utf-8") + _DENSE_MARKING_WRAPPER, encoding="utf-8" + ) return destination diff --git a/tests/e2e/scripts/release_performance/test_compare_release_performance.py b/tests/e2e/scripts/release_performance/test_compare_release_performance.py index a28887ff85..00293abaa0 100644 --- a/tests/e2e/scripts/release_performance/test_compare_release_performance.py +++ b/tests/e2e/scripts/release_performance/test_compare_release_performance.py @@ -27,8 +27,15 @@ description="the same build as candidate and baseline reports every command", inspection_models=300, build_models=100, + dense_models=20, expected_outcomes=((0, True), (1, False)), expected_fragments=( + "| `compile (no cache)` |", + "| `compile (warm cache)` |", + "| `compile (one-model edit)` |", + "| `dense compile (warm cache)` |", + "| `dense compile (one-model edit)` |", + "| `dense compile (no cache)` |", "| `plan --json` |", "| `build (empty warehouse)` |", "| `lineage column trace` |", @@ -38,6 +45,7 @@ expected_marked_projects=( "baseline/build", "baseline/build-template", + "baseline/dense", "baseline/inspection", ), ), @@ -71,6 +79,8 @@ def test_given_two_installations_when_comparing_release_performance_then_reports str(test_case.inspection_models), "--build-models", str(test_case.build_models), + "--dense-models", + str(test_case.dense_models), "--baseline-source", str(baseline_source), "--work-dir", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md index e94d8bf957..5055d2f863 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md @@ -85,27 +85,48 @@ built project, change the queries of a few models (and rename one table in the s rename discovery matches it against stored fingerprints), and bound the next `sqb plan --json`. All cases carry `models_3000` identifiers so they run only in the 3,000-model fresh-process job. +## Same-runner comparison with the base branch + +Absolute budgets must absorb the spread between hosted runners, so every pull request that touches +the compiler also runs `scripts/compare_compile_performance.py` on the dense and the fresh +3,000-model benchmark. Base and head each compile a project written by their own generator, on the +same runner, three alternating runs per mode, and the medians of wall and CPU time must stay within +1.10x of base for each mode: + +- cold: `sqb compile --json --no-cache` after removing `target`, after one untimed warm-up; +- warm: an unchanged `sqb compile --json` after one untimed run that writes the cache; +- edit: `sqb compile --json` after the same one-model edit as the release comparison below. + +The warm and edit modes reuse the cold project. A warm run that misses the analysis cache, or an +edit run that does not, stops the comparison because it would measure the wrong path. + ## Release comparison with the previous release Fixed ceilings miss gradual creep and slowdowns that stay under a loose limit, so every release pull request also compares the candidate with the previous published release on the same runner. `scripts/compare_release_performance.py` installs the candidate wheel and the baseline from PyPI -into separate Python 3.12 environments, generates the inspection and build benchmarks above once, -and gives each version its own copy with compile, lineage and scope caches warmed by that version. +into separate Python 3.12 environments, generates the inspection, build and dense benchmarks +with each version's own generator, and gives each version its own copy with compile, lineage and scope caches warmed by that version. It then runs each command alternately for baseline and candidate, five times each, and compares the medians of wall time and CPU time (user+sys) and the worst-run peak RSS. Peak RSS depends on whether concurrent phases overlap, so it lands on one of a few levels from run to run; a median flips between those levels, while the worst run is stable: -- warm and uncached `sqb compile --json`, `sqb plan --json`, `sqb dag --json` and - `sqb scope --json` on the 3,000-model inspection benchmark; +- uncached, warm and one-model edit `sqb compile --json`, `sqb plan --json`, `sqb dag --json` + and `sqb scope --json` on the 3,000-model inspection benchmark. The edit run adds a + revision-specific comment before the query of the middle model, so every run recompiles one + changed model against a warm cache; it runs after the other inspection commands; - `sqb lineage` downstream from the hub, upstream from the rollup, and the upstream column trace of `shared_orders_rollup.amount`; +- uncached, warm and one-model edit `sqb compile --json` on the 3,000-model dense all-rules + benchmark, which each version generates with its own `write_dense_compile_project`. The + uncached runs remove `target` first, so built-in and custom Rules run as in the dense guard; - `sqb build` of the 1,000-model build benchmark from an empty warehouse, restored before each run. A command fails when a candidate value exceeds the baseline by more than 25% and by more than -0.5 s (wall and CPU) or 32 MiB (peak RSS); the limits live in -`scripts/release_performance/constants.py`. The baseline defaults to the highest version below +0.5 s (wall and CPU) or 32 MiB (peak RSS). The six compile commands use a fixed 15% wall and CPU +limit instead, so small per-pull-request compile regressions cannot accumulate across a release +unnoticed. The limits live in `scripts/release_performance/constants.py`. The baseline defaults to the highest version below the candidate that is installable from PyPI, or release-tagged but not yet on PyPI; yanked releases never count. A freshly tagged baseline is awaited until its wheels are on PyPI, installing from the uncompressed simple index's wheel URL when the CDN still lags. A diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index 5179524f91..214f137ba9 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -29,7 +29,8 @@ run_dense_warm_edit_benchmark, ) -_GIB: int = 1024 * 1024 * 1024 +_MIB: int = 1024 * 1024 +_GIB: int = 1024 * _MIB @pytest.mark.performance @@ -40,28 +41,28 @@ DenseCompileGuardTestCase( "dense_models_1000_all_rules", 1000, - 14.0, - 3 * _GIB // 2, + 13.0, + 800 * _MIB, "6ea005eef07fe6d2ec47980bc573903efa060cfde55aa5488e63b29ce846280e", ), DenseCompileGuardTestCase( "dense_models_3000_all_rules", 3000, - 45.0, - 11 * _GIB // 4, + 25.0, + 3 * _GIB // 2, "3d26ef1a4d30ea96dd97d60b2e2f6c93473e354099138a01d74b136700c9f022", ), DenseCompileGuardTestCase( "dense_models_5000_all_rules", 5000, - 75.0, - 13 * _GIB // 4, + 42.0, + 9 * _GIB // 4, "bc6b60e90118d66f3217a5f04f104ae91f657a6356f4d19844614d1ba48b0012", ), DenseCompileGuardTestCase( "dense_models_10000_all_rules", 10000, - 145.0, + 77.0, 13 * _GIB // 4, "913a08c2277304efacf75e293728767a3fa416f6b32f5488dca6ab5baea80668", ), @@ -138,9 +139,9 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ "dense_models_3000_warm_and_one_edit", 3000, 1521, - 22.0, - 24.0, - 11 * _GIB // 4, + 11.0, + 12.0, + 3 * _GIB // 2, "3d26ef1a4d30ea96dd97d60b2e2f6c93473e354099138a01d74b136700c9f022", "f7f8a3d881496664ec7e293144d93b8b8e486a04eba830d9cc6500d1b28147f3", 3, @@ -149,9 +150,9 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ "dense_models_5000_warm_and_one_edit", 5000, 2521, - 36.0, - 38.0, - 13 * _GIB // 4, + 19.0, + 21.0, + 9 * _GIB // 4, "bc6b60e90118d66f3217a5f04f104ae91f657a6356f4d19844614d1ba48b0012", "571fe58493234057c7128b4929d3518532731192f6fef85da0496788826bb172", 3, diff --git a/tests/unit/scripts/compile_performance_ratio/__init__.py b/tests/unit/scripts/compile_performance_ratio/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/compile_performance_ratio/_helpers/__init__.py b/tests/unit/scripts/compile_performance_ratio/_helpers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py b/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py new file mode 100644 index 0000000000..1927e81a71 --- /dev/null +++ b/tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py @@ -0,0 +1,38 @@ +"""Test-case models for the same-runner compile performance ratio guard.""" + +from dataclasses import dataclass + +from scripts.compile_performance_ratio.models import CompileComparison + + +@dataclass(frozen=True) +class RatioFailuresTestCase: + description: str + comparisons: tuple[CompileComparison, ...] + modes: tuple[str, ...] + max_ratio: float + expected_failures: tuple[str, ...] + + +@dataclass(frozen=True) +class ComparisonMarkdownTestCase: + description: str + comparisons: tuple[CompileComparison, ...] + failures: tuple[str, ...] + expected_fragments: tuple[str, ...] + + +@dataclass(frozen=True) +class OneModelEditTestCase: + description: str + model_files: dict[str, str] + revisions: int + expected_paths: tuple[str, ...] + expected_files: dict[str, str] + + +@dataclass(frozen=True) +class OneModelEditErrorTestCase: + description: str + model_files: dict[str, str] + expected_message: str diff --git a/tests/unit/scripts/compile_performance_ratio/_helpers/helpers.py b/tests/unit/scripts/compile_performance_ratio/_helpers/helpers.py new file mode 100644 index 0000000000..9e88b6c75e --- /dev/null +++ b/tests/unit/scripts/compile_performance_ratio/_helpers/helpers.py @@ -0,0 +1,41 @@ +"""Builders for same-runner compile comparison test data.""" + +from pathlib import Path + +from scripts.compile_performance_ratio.models import CompileComparison + + +def comparison( + *, mode: str, wall: tuple[float, float], cpu: tuple[float, float] +) -> CompileComparison: + """Build a dense 3000-model comparison from (base, head) wall and CPU medians.""" + + return CompileComparison( + kind="dense", + models=3000, + mode=mode, + base_wall_seconds=wall[0], + head_wall_seconds=wall[1], + base_cpu_seconds=cpu[0], + head_cpu_seconds=cpu[1], + base_timings_ms={"total_ms": wall[0] * 1000}, + head_timings_ms={"total_ms": wall[1] * 1000}, + ) + + +def write_model_files(*, project_dir: Path, model_files: dict[str, str]) -> None: + """Write model files at project-relative paths.""" + + for relative, contents in model_files.items(): + path: Path = project_dir / relative + path.parent.mkdir(parents=True, exist_ok=True) + _ = path.write_text(contents, encoding="utf-8") + + +def read_model_files(*, project_dir: Path) -> dict[str, str]: + """Read every model file keyed by its project-relative path.""" + + return { + path.relative_to(project_dir).as_posix(): path.read_text(encoding="utf-8") + for path in sorted((project_dir / "models").rglob("*.sql")) + } diff --git a/tests/unit/scripts/compile_performance_ratio/_helpers/test_edit.py b/tests/unit/scripts/compile_performance_ratio/_helpers/test_edit.py new file mode 100644 index 0000000000..dc5c217b60 --- /dev/null +++ b/tests/unit/scripts/compile_performance_ratio/_helpers/test_edit.py @@ -0,0 +1,95 @@ +"""Tests for the generator-independent one-model benchmark edit.""" + +from pathlib import Path + +import pytest + +from scripts.compile_performance_ratio._helpers.edit import apply_one_model_edit +from scripts.compile_performance_ratio.exceptions import BenchmarkEditError +from tests.unit.scripts.compile_performance_ratio._helpers._test_types import ( + OneModelEditErrorTestCase, + OneModelEditTestCase, +) +from tests.unit.scripts.compile_performance_ratio._helpers.helpers import ( + read_model_files, + write_model_files, +) + +_HEADER: str = 'MODEL (description "Orders.");\n\n' +_WITH_QUERY: str = "WITH base AS (SELECT 1 AS id)\nSELECT id FROM base\n" + + +@pytest.mark.parametrize( + "test_case", + ( + OneModelEditTestCase( + description="the middle model by path gets a comment before its query", + model_files={ + "models/a/orders_a.sql": _HEADER + "SELECT 1 AS id\n", + "models/b/orders_b.sql": _HEADER + _WITH_QUERY, + "models/c/orders_c.sql": _HEADER + "SELECT 3 AS id\n", + }, + revisions=1, + expected_paths=("models/b/orders_b.sql",), + expected_files={ + "models/a/orders_a.sql": _HEADER + "SELECT 1 AS id\n", + "models/b/orders_b.sql": _HEADER + "-- Benchmark edit 0.\n" + _WITH_QUERY, + "models/c/orders_c.sql": _HEADER + "SELECT 3 AS id\n", + }, + ), + OneModelEditTestCase( + description="each revision adds a distinct comment so every run is a new edit", + model_files={"models/orders.sql": _HEADER + "SELECT 1 AS id\n"}, + revisions=2, + expected_paths=("models/orders.sql", "models/orders.sql"), + expected_files={ + "models/orders.sql": _HEADER + + "-- Benchmark edit 0.\n-- Benchmark edit 1.\nSELECT 1 AS id\n" + }, + ), + ), + ids=lambda case: case.description, +) +def test_given_benchmark_project_when_editing_one_model_then_changes_only_the_middle_query( + test_case: OneModelEditTestCase, tmp_path: Path +) -> None: + write_model_files(project_dir=tmp_path, model_files=test_case.model_files) + + edited: tuple[str, ...] = tuple( + apply_one_model_edit(project_dir=tmp_path, revision=revision) + .relative_to(tmp_path) + .as_posix() + for revision in range(test_case.revisions) + ) + + assert edited == test_case.expected_paths + assert read_model_files(project_dir=tmp_path) == test_case.expected_files + + +@pytest.mark.parametrize( + "test_case", + ( + OneModelEditErrorTestCase( + description="a project without models is reported", + model_files={}, + expected_message="has no model files to edit", + ), + OneModelEditErrorTestCase( + description="a model without a query start is reported", + model_files={"models/orders.sql": _HEADER + " select 1 as id\n"}, + expected_message="has no line starting a WITH or SELECT query to edit", + ), + ), + ids=lambda case: case.description, +) +def test_given_project_without_editable_query_when_editing_one_model_then_raises( + test_case: OneModelEditErrorTestCase, tmp_path: Path +) -> None: + write_model_files(project_dir=tmp_path, model_files=test_case.model_files) + + with pytest.raises(BenchmarkEditError, match=test_case.expected_message): + _ = apply_one_model_edit(project_dir=tmp_path, revision=0) + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compile_performance_ratio/_helpers/test_report.py b/tests/unit/scripts/compile_performance_ratio/_helpers/test_report.py new file mode 100644 index 0000000000..a4509303be --- /dev/null +++ b/tests/unit/scripts/compile_performance_ratio/_helpers/test_report.py @@ -0,0 +1,60 @@ +"""Tests for the same-runner compile comparison job summary.""" + +import pytest + +from scripts.compile_performance_ratio._helpers.report import comparison_markdown +from tests.unit.scripts.compile_performance_ratio._helpers._test_types import ( + ComparisonMarkdownTestCase, +) +from tests.unit.scripts.compile_performance_ratio._helpers.helpers import comparison + + +@pytest.mark.parametrize( + "test_case", + ( + ComparisonMarkdownTestCase( + description="every mode gets its own table and a passing result", + comparisons=( + comparison(mode="cold", wall=(18.0, 18.0), cpu=(40.0, 40.0)), + comparison(mode="warm", wall=(7.6, 7.6), cpu=(9.0, 9.0)), + comparison(mode="edit", wall=(8.0, 8.4), cpu=(11.0, 11.0)), + ), + failures=(), + expected_fragments=( + "### dense 3000 models, cold compile without cache: head vs base (same runner)", + "### dense 3000 models, unchanged warm compile: head vs base (same runner)", + "### dense 3000 models, one-model edit on a warm cache: head vs base", + "| Wall (s) | 8.00 | 8.40 | 1.050 |", + "| total_ms | 7600 | 7600 | 1.000 |", + "**Result: passed.** Every mode is within the same-runner limit.", + ), + ), + ComparisonMarkdownTestCase( + description="failures are listed in the result", + comparisons=(comparison(mode="warm", wall=(7.6, 11.4), cpu=(9.0, 9.0)),), + failures=("dense 3000 warm: wall ratio 1.500 exceeds 1.10",), + expected_fragments=( + "| Wall (s) | 7.60 | 11.40 | 1.500 |", + "**Result: failed.** dense 3000 warm: wall ratio 1.500 exceeds 1.10.", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_mode_comparisons_when_rendering_summary_then_reports_each_mode( + test_case: ComparisonMarkdownTestCase, +) -> None: + markdown: str = comparison_markdown( + comparisons=test_case.comparisons, + runs=3, + max_ratio=1.10, + per_side_projects=True, + failures=test_case.failures, + ) + + for fragment in test_case.expected_fragments: + assert fragment in markdown + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/compile_performance_ratio/_helpers/test_verdict.py b/tests/unit/scripts/compile_performance_ratio/_helpers/test_verdict.py new file mode 100644 index 0000000000..189b2e651f --- /dev/null +++ b/tests/unit/scripts/compile_performance_ratio/_helpers/test_verdict.py @@ -0,0 +1,94 @@ +"""Tests for the same-runner compile ratio verdict across cold, warm and edit modes.""" + +import pytest + +from scripts.compile_performance_ratio._helpers.verdict import ratio_failures +from scripts.compile_performance_ratio.constants import COMPILE_MODES +from tests.unit.scripts.compile_performance_ratio._helpers._test_types import ( + RatioFailuresTestCase, +) +from tests.unit.scripts.compile_performance_ratio._helpers.helpers import comparison + + +@pytest.mark.parametrize( + "test_case", + ( + RatioFailuresTestCase( + description="every mode at parity passes", + comparisons=( + comparison(mode="cold", wall=(18.0, 18.1), cpu=(40.0, 40.2)), + comparison(mode="warm", wall=(7.6, 7.5), cpu=(9.1, 9.0)), + comparison(mode="edit", wall=(8.6, 8.7), cpu=(11.0, 11.1)), + ), + modes=COMPILE_MODES, + max_ratio=1.10, + expected_failures=(), + ), + RatioFailuresTestCase( + description="exactly the limit passes", + comparisons=(comparison(mode="warm", wall=(8.0, 8.8), cpu=(10.0, 11.0)),), + modes=("warm",), + max_ratio=1.10, + expected_failures=(), + ), + RatioFailuresTestCase( + description="a warm-only slowdown fails while cold and edit pass", + comparisons=( + comparison(mode="cold", wall=(18.0, 18.0), cpu=(40.0, 40.0)), + comparison(mode="warm", wall=(7.6, 11.4), cpu=(9.0, 13.5)), + comparison(mode="edit", wall=(8.6, 8.6), cpu=(11.0, 11.0)), + ), + modes=COMPILE_MODES, + max_ratio=1.10, + expected_failures=( + "dense 3000 warm: wall ratio 1.500 exceeds 1.10", + "dense 3000 warm: CPU ratio 1.500 exceeds 1.10", + ), + ), + RatioFailuresTestCase( + description="an edit CPU regression at wall parity fails on CPU alone", + comparisons=(comparison(mode="edit", wall=(8.0, 8.0), cpu=(10.0, 12.0)),), + modes=("edit",), + max_ratio=1.10, + expected_failures=("dense 3000 edit: CPU ratio 1.200 exceeds 1.10",), + ), + RatioFailuresTestCase( + description="a requested mode without a measurement fails", + comparisons=(comparison(mode="cold", wall=(18.0, 18.0), cpu=(40.0, 40.0)),), + modes=("cold", "edit"), + max_ratio=1.10, + expected_failures=("edit: no measurement",), + ), + RatioFailuresTestCase( + description="a zero base measurement fails as missing instead of dividing by zero", + comparisons=(comparison(mode="warm", wall=(0.0, 7.5), cpu=(9.0, 9.0)),), + modes=("warm",), + max_ratio=1.10, + expected_failures=( + "dense 3000 warm: wall measurement missing (base 0.00 s, head 7.50 s)", + ), + ), + RatioFailuresTestCase( + description="a zero head measurement fails as missing instead of passing as fast", + comparisons=(comparison(mode="cold", wall=(18.0, 18.0), cpu=(40.0, 0.0)),), + modes=("cold",), + max_ratio=1.10, + expected_failures=( + "dense 3000 cold: CPU measurement missing (base 40.00 s, head 0.00 s)", + ), + ), + ), + ids=lambda case: case.description, +) +def test_given_mode_comparisons_when_judging_then_reports_each_failed_limit( + test_case: RatioFailuresTestCase, +) -> None: + failures: tuple[str, ...] = ratio_failures( + comparisons=test_case.comparisons, modes=test_case.modes, max_ratio=test_case.max_ratio + ) + + assert failures == test_case.expected_failures + + +if __name__ == "__main__": + pytest.main([__file__, "-vv"]) diff --git a/tests/unit/scripts/release_performance/_helpers/_test_types.py b/tests/unit/scripts/release_performance/_helpers/_test_types.py index 5a7bf9c666..bc602ccbea 100644 --- a/tests/unit/scripts/release_performance/_helpers/_test_types.py +++ b/tests/unit/scripts/release_performance/_helpers/_test_types.py @@ -70,6 +70,14 @@ class BaselineGeneratorTestCase: expected_error_fragments: tuple[str, ...] +@dataclass(frozen=True) +class BaselineDenseGeneratorTestCase: + description: str + generator: str + expected_markers: tuple[str, ...] + expected_error_fragments: tuple[str, ...] + + @dataclass(frozen=True) class ReleaseSourceTestCase: description: str diff --git a/tests/unit/scripts/release_performance/_helpers/helpers.py b/tests/unit/scripts/release_performance/_helpers/helpers.py index 9b9b7d20b8..347c353e3c 100644 --- a/tests/unit/scripts/release_performance/_helpers/helpers.py +++ b/tests/unit/scripts/release_performance/_helpers/helpers.py @@ -18,6 +18,16 @@ def write_pristine_projects(*, root: Path, inspection_models: int, build_models: FAILING_GENERATOR: str = """def write_pristine_projects(*, root, inspection_models, build_models): raise RuntimeError("baseline generator exploded") """ +MARKER_DENSE_GENERATOR: str = """from pathlib import Path + + +def write_dense_compile_project(*, project_dir: Path, model_count: int) -> None: + project_dir.mkdir(parents=True) + (project_dir / "BASELINE_MARKER").write_text(str(model_count)) +""" +FAILING_DENSE_GENERATOR: str = """def write_dense_compile_project(*, project_dir, model_count): + raise RuntimeError("baseline dense generator exploded") +""" def comparison( @@ -25,6 +35,7 @@ def comparison( name: str, baseline: tuple[tuple[float, float, int], ...], candidate: tuple[tuple[float, float, int], ...], + max_time_ratio: float | None = None, ) -> CommandComparison: """Build a comparison from (wall seconds, CPU seconds, peak RSS MiB) samples.""" @@ -32,6 +43,7 @@ def comparison( name=name, baseline=tuple(_sample(values=values) for values in baseline), candidate=tuple(_sample(values=values) for values in candidate), + max_time_ratio=max_time_ratio, ) @@ -57,6 +69,17 @@ def write_baseline_source(*, root: Path, generator: str) -> Path: return root +def write_baseline_dense_source(*, root: Path, generator: str) -> Path: + """Write a source tree whose only content is a dense benchmark generator.""" + + helpers: Path = root / "scripts" / "cold_compile_performance" / "_helpers" + helpers.mkdir(parents=True) + for package in (root / "scripts", helpers.parent, helpers): + _ = (package / "__init__.py").write_text("", encoding="utf-8") + _ = (helpers / "dense_project.py").write_text(generator, encoding="utf-8") + return root + + def tagged_repository(*, root: Path, tag: str) -> Path: """Create a one-commit git repository with a tracked file and the given tag.""" diff --git a/tests/unit/scripts/release_performance/_helpers/test_benchmark.py b/tests/unit/scripts/release_performance/_helpers/test_benchmark.py index bce6daa78c..1458f4e231 100644 --- a/tests/unit/scripts/release_performance/_helpers/test_benchmark.py +++ b/tests/unit/scripts/release_performance/_helpers/test_benchmark.py @@ -6,17 +6,24 @@ import pytest -from scripts.release_performance._helpers.benchmark import write_baseline_pristine_projects +from scripts.release_performance._helpers.benchmark import ( + write_baseline_dense_project, + write_baseline_pristine_projects, +) from scripts.release_performance._helpers.versions import release_source from scripts.release_performance.exceptions import ReleasePerformanceError from tests.unit.scripts.release_performance._helpers._test_types import ( + BaselineDenseGeneratorTestCase, BaselineGeneratorTestCase, ReleaseSourceTestCase, ) from tests.unit.scripts.release_performance._helpers.helpers import ( + FAILING_DENSE_GENERATOR, FAILING_GENERATOR, + MARKER_DENSE_GENERATOR, MARKER_GENERATOR, tagged_repository, + write_baseline_dense_source, write_baseline_source, ) @@ -63,6 +70,48 @@ def test_given_baseline_source_when_generating_then_uses_its_own_generator( assert all(fragment in error for fragment in test_case.expected_error_fragments), error +@pytest.mark.parametrize( + "test_case", + [ + BaselineDenseGeneratorTestCase( + description="the baseline source's own dense generator writes the dense project", + generator=MARKER_DENSE_GENERATOR, + expected_markers=("40",), + expected_error_fragments=(), + ), + BaselineDenseGeneratorTestCase( + description="a failing baseline dense generator is reported with its stderr", + generator=FAILING_DENSE_GENERATOR, + expected_markers=(), + expected_error_fragments=( + "Baseline dense benchmark generation with the generator in", + "baseline dense generator exploded", + ), + ), + ], + ids=lambda case: case.description, +) +def test_given_baseline_source_when_generating_dense_project_then_uses_its_own_generator( + test_case: BaselineDenseGeneratorTestCase, tmp_path: Path +) -> None: + source: Path = write_baseline_dense_source( + root=tmp_path / "source", generator=test_case.generator + ) + pristine: Path = tmp_path / "generated" / "pristine" + error: str = "" + + try: + write_baseline_dense_project(source=source, pristine=pristine, models=40) + except ReleasePerformanceError as raised: + error = str(raised) + + markers: tuple[str, ...] = tuple( + marker.read_text(encoding="utf-8") for marker in pristine.glob("dense/BASELINE_MARKER") + ) + assert markers == test_case.expected_markers + assert all(fragment in error for fragment in test_case.expected_error_fragments), error + + @pytest.mark.parametrize( "test_case", [ diff --git a/tests/unit/scripts/release_performance/_helpers/test_report.py b/tests/unit/scripts/release_performance/_helpers/test_report.py index 5ba5bffe62..0fca1ead7c 100644 --- a/tests/unit/scripts/release_performance/_helpers/test_report.py +++ b/tests/unit/scripts/release_performance/_helpers/test_report.py @@ -34,7 +34,8 @@ "Benchmark projects are generated per side: the baseline runs projects from its " "own generator (v0.126.1), the candidate from the candidate's.", "Limits: wall +25% (ignored under 0.5 s), CPU +25% (ignored under 0.5 s), " - "peak RSS +25% (ignored under 32 MiB).", + "peak RSS +25% (ignored under 32 MiB). Cold, warm and one-model edit compiles " + "use +15% for wall and CPU.", "| `compile (warm cache)` | 6.00 → 6.10 | 1.017x | 6.50 → 6.40 | 0.985x " "| 500 → 505 | 1.010x | ✅ ok |", "**Result: passed.**", @@ -59,6 +60,24 @@ ), unexpected_fragments=("**Result: passed.**",), ), + ComparisonMarkdownTestCase( + description="a compile regression is judged against the compile time limit", + commands=( + comparison( + name="compile (one-model edit)", + baseline=((6.0, 7.0, 450),) * 3, + candidate=((7.2, 7.1, 450),) * 3, + max_time_ratio=1.15, + ), + ), + skipped=(), + expected_fragments=( + "| `compile (one-model edit)` | 6.00 → 7.20 | **1.200x** | 7.00 → 7.10 | 1.014x " + "| 450 → 450 | 1.000x | ❌ regressed |", + "**Result: failed.** compile (one-model edit): wall 1.200x exceeds 1.15x.", + ), + unexpected_fragments=("**Result: passed.**",), + ), ComparisonMarkdownTestCase( description="a skipped command is reported with its reason instead of omitted", commands=(), diff --git a/tests/unit/scripts/release_performance/_helpers/test_verdict.py b/tests/unit/scripts/release_performance/_helpers/test_verdict.py index 3a4bc8ca5e..b5b481a904 100644 --- a/tests/unit/scripts/release_performance/_helpers/test_verdict.py +++ b/tests/unit/scripts/release_performance/_helpers/test_verdict.py @@ -63,6 +63,36 @@ ), expected_regressed=(False, True, False), ), + MetricVerdictTestCase( + description="a compile command fails just above its own tighter time limit", + comparison=comparison( + name="compile (one-model edit)", + baseline=((10.0, 10.0, 400),) * 3, + candidate=((11.6, 11.4, 400),) * 3, + max_time_ratio=1.15, + ), + expected_regressed=(True, False, False), + ), + MetricVerdictTestCase( + description="a compile command under its tighter limit passes", + comparison=comparison( + name="compile (no cache)", + baseline=((8.0, 16.0, 600),) * 3, + candidate=((9.1, 18.3, 600),) * 3, + max_time_ratio=1.15, + ), + expected_regressed=(False, False, False), + ), + MetricVerdictTestCase( + description="a compile command above its tighter limit stays under the time floor", + comparison=comparison( + name="compile (warm cache)", + baseline=((2.0, 2.0, 400),) * 3, + candidate=((2.4, 2.4, 400),) * 3, + max_time_ratio=1.15, + ), + expected_regressed=(False, False, False), + ), MetricVerdictTestCase( description="a wall-only regression at CPU parity fails on wall alone", comparison=comparison(