From b57ec001ab3d388172eb63313a19f45ca2cc33e4 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:10:00 +0000 Subject: [PATCH 1/3] test(perf): judge the dense warm and edit guards on the median of three runs The warm and one-model edit budgets were checked against one compile each, so a single slow moment on a hosted runner failed changes that leave the compiler untouched. The guard now runs three warm compiles and three edits in the same job and compares their medians with the unchanged budgets. Every run still checks diagnostics, peak memory and cache behaviour; warm runs keep the cold fingerprint and the final edit still matches the from-scratch oracle. --- .../cli/commands/main/compile/helpers.py | 78 +++++++++++++------ .../compile/test_dense_compile_performance.py | 61 +++++++++------ 2 files changed, 91 insertions(+), 48 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index c14d1cc038..25b8b7629e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -916,49 +916,71 @@ def prepare_small_inspection_project(*, project_dir: Path, model_count: int) -> ) +# Each repeated edit writes a value the cache has not seen, so every edit recompiles the edited +# model; the last writes the value the expected edit fingerprint and the oracle describe. +_DENSE_EDIT_AMOUNTS: tuple[str, ...] = ("1001", "1002", "1000") +_DENSE_WARM_RUNS: int = 3 + + +class DenseWarmEditMeasurements(NamedTuple): + """Cached cold, repeated warm and one-model edit compiles, and a from-scratch oracle.""" + + cold: FreshProcessCompileBenchmarkResult + warm: tuple[FreshProcessCompileBenchmarkResult, ...] + edit: tuple[FreshProcessCompileBenchmarkResult, ...] + oracle: FreshProcessCompileBenchmarkResult + + def run_dense_warm_edit_benchmark( *, project_dir: Path, edited_model_path: Path, expected_warm_max_seconds: float, expected_edit_max_seconds: float, -) -> dict[str, FreshProcessCompileBenchmarkResult]: - """Measure cached cold, warm, and one-model edit plus a from-scratch oracle of the edit.""" +) -> DenseWarmEditMeasurements: + """Measure cached cold, three warm and three one-model edit compiles, then the edit's oracle.""" - measurements: dict[str, FreshProcessCompileBenchmarkResult] = { - "cold": _run_fresh_process_compile_benchmark( - project_dir=project_dir, - label="dense-cold", - expected_max_wall_seconds=4 * expected_warm_max_seconds, - compile_args=(), - ), - "warm": _run_fresh_process_compile_benchmark( + cold: FreshProcessCompileBenchmarkResult = _run_fresh_process_compile_benchmark( + project_dir=project_dir, + label="dense-cold", + expected_max_wall_seconds=4 * expected_warm_max_seconds, + compile_args=(), + ) + warm: tuple[FreshProcessCompileBenchmarkResult, ...] = tuple( + _run_fresh_process_compile_benchmark( project_dir=project_dir, - label="dense-warm", + label=f"dense-warm-{run}", expected_max_wall_seconds=expected_warm_max_seconds, compile_args=(), - ), - } - _replace_benchmark_text( - path=edited_model_path, - old="COALESCE(b.amount, 0) + CAST(b.id AS DOUBLE)", - new="COALESCE(b.amount, 1000) + CAST(b.id AS DOUBLE)", - ) - measurements["edit"] = _run_fresh_process_compile_benchmark( - project_dir=project_dir, - label="dense-edit", - expected_max_wall_seconds=expected_edit_max_seconds, - compile_args=(), + ) + for run in range(_DENSE_WARM_RUNS) ) + edits: list[FreshProcessCompileBenchmarkResult] = [] + previous: str = "0" + for amount in _DENSE_EDIT_AMOUNTS: + _replace_benchmark_text( + path=edited_model_path, + old=f"COALESCE(b.amount, {previous}) + CAST(b.id AS DOUBLE)", + new=f"COALESCE(b.amount, {amount}) + CAST(b.id AS DOUBLE)", + ) + edits.append( + _run_fresh_process_compile_benchmark( + project_dir=project_dir, + label=f"dense-edit-{len(edits)}", + expected_max_wall_seconds=expected_edit_max_seconds, + compile_args=(), + ) + ) + previous = amount oracle_dir: Path = project_dir.with_name(f"{project_dir.name}_oracle") _ = shutil.copytree(project_dir, oracle_dir, ignore=shutil.ignore_patterns("target")) - measurements["oracle"] = _run_fresh_process_compile_benchmark( + oracle: FreshProcessCompileBenchmarkResult = _run_fresh_process_compile_benchmark( project_dir=oracle_dir, label="dense-oracle", expected_max_wall_seconds=4 * expected_edit_max_seconds, compile_args=("--no-cache",), ) - return measurements + return DenseWarmEditMeasurements(cold=cold, warm=warm, edit=tuple(edits), oracle=oracle) class DeltaEditLoopMeasurements(NamedTuple): @@ -1091,6 +1113,12 @@ def median_cpu_seconds(results: tuple[FreshProcessCompileBenchmarkResult, ...]) return statistics.median(result.cpu_seconds for result in results) +def median_wall_seconds(results: tuple[FreshProcessCompileBenchmarkResult, ...]) -> float: + """The median wall seconds of same-runner compiles.""" + + return statistics.median(result.elapsed_seconds for result in results) + + def _append_benchmark_edit(path: Path, label: str) -> None: path.write_text( path.read_text(encoding="utf-8") + f"\n-- one {label} edit\n", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index 5558f90575..a233e82057 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -29,12 +29,14 @@ ) from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import ( DeltaEditLoopMeasurements, + DenseWarmEditMeasurements, FreshProcessCompileBenchmarkResult, _run_fresh_process_compile_benchmark, analysed_model_count, fresh_process_compile_cache_metrics, measure_model_sql_bytes, median_cpu_seconds, + median_wall_seconds, run_dense_delta_edit_loop_benchmark, run_dense_warm_edit_benchmark, ) @@ -185,13 +187,19 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches edited_model_path: Path = dense_model_path( project_dir=project_dir, index=test_case.edited_model_index ) - measurements: dict[str, FreshProcessCompileBenchmarkResult] = run_dense_warm_edit_benchmark( + measurements: DenseWarmEditMeasurements = run_dense_warm_edit_benchmark( project_dir=project_dir, edited_model_path=edited_model_path, expected_warm_max_seconds=test_case.expected_warm_max_seconds, expected_edit_max_seconds=test_case.expected_edit_max_seconds, ) - for label, result in measurements.items(): + labelled: tuple[tuple[str, FreshProcessCompileBenchmarkResult], ...] = ( + ("cold", measurements.cold), + *(("warm", result) for result in measurements.warm), + *(("edit", result) for result in measurements.edit), + ("oracle", measurements.oracle), + ) + for label, result in labelled: print( f"dense {label} models={test_case.model_count} wall={result.elapsed_seconds:.2f}s " f"cpu={result.cpu_seconds:.2f}s fingerprint={result.semantic_fingerprint}", @@ -200,30 +208,37 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches assert result.payload["diagnostics"] == [] assert result.payload["has_errors"] is False assert result.peak_rss_bytes < test_case.expected_max_rss_bytes - cold: FreshProcessCompileBenchmarkResult = measurements["cold"] - warm: FreshProcessCompileBenchmarkResult = measurements["warm"] - edit: FreshProcessCompileBenchmarkResult = measurements["edit"] - assert cold.semantic_fingerprint == test_case.expected_cold_fingerprint - assert warm.semantic_fingerprint == test_case.expected_cold_fingerprint - assert edit.semantic_fingerprint == test_case.expected_edit_fingerprint - assert measurements["oracle"].semantic_fingerprint == test_case.expected_edit_fingerprint - warm_timings: dict[str, int] = cast(dict[str, int], warm.payload["compile_timings"]) - edit_timings: dict[str, int] = cast(dict[str, int], edit.payload["compile_timings"]) - batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics(warm) - assert (batch_hits + entry_hits, misses, bypasses) == (test_case.model_count, 0, 0) - assert warm_timings["rule_cache_misses"] == 0 - batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics(edit) - assert 0 < misses < test_case.model_count - assert batch_hits + entry_hits + misses == test_case.model_count - # The edited model misses once in built-in model rules and once in SQL lint; custom rules - # replay their recorded facts, and every other model's built-in findings stay cached. - assert edit_timings["rule_cache_misses"] == test_case.expected_edit_rule_cache_misses + assert measurements.cold.semantic_fingerprint == test_case.expected_cold_fingerprint + for warm in measurements.warm: + assert warm.semantic_fingerprint == test_case.expected_cold_fingerprint + warm_timings: dict[str, int] = cast(dict[str, int], warm.payload["compile_timings"]) + batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics(warm) + assert (batch_hits + entry_hits, misses, bypasses) == (test_case.model_count, 0, 0) + assert warm_timings["rule_cache_misses"] == 0 + for edit in measurements.edit: + edit_timings: dict[str, int] = cast(dict[str, int], edit.payload["compile_timings"]) + batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics(edit) + assert 0 < misses < test_case.model_count + assert batch_hits + entry_hits + misses == test_case.model_count + # The edited model misses once in built-in model rules and once in SQL lint; custom + # rules replay their recorded facts, and every other model's built-in findings stay + # cached. + assert edit_timings["rule_cache_misses"] == test_case.expected_edit_rule_cache_misses + # Earlier edits write other values, so their output differs from both the cold and final edit. + for edit in measurements.edit[:-1]: + assert edit.semantic_fingerprint not in { + test_case.expected_cold_fingerprint, + test_case.expected_edit_fingerprint, + } + assert measurements.edit[-1].semantic_fingerprint == test_case.expected_edit_fingerprint + assert measurements.oracle.semantic_fingerprint == test_case.expected_edit_fingerprint oracle_timings: dict[str, int] = cast( - dict[str, int], measurements["oracle"].payload["compile_timings"] + dict[str, int], measurements.oracle.payload["compile_timings"] ) assert oracle_timings["rule_cache_hits"] == 0 - assert warm.elapsed_seconds < test_case.expected_warm_max_seconds - assert edit.elapsed_seconds < test_case.expected_edit_max_seconds + # Medians of same-job runs keep one slow runner moment from failing an unchanged compiler. + assert median_wall_seconds(measurements.warm) < test_case.expected_warm_max_seconds + assert median_wall_seconds(measurements.edit) < test_case.expected_edit_max_seconds if __name__ == "__main__": From e26ce105d3d1685ced19d6a5cb1bf78cdd971602 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 19:45:45 +0000 Subject: [PATCH 2/3] test(perf): scale the dense compile budgets by a runner calibration Hosted runners differ by CPU class, and the same compiler measures about a third slower on the slower class, so the dense guard's absolute budgets judged the runner rather than the code. Before each dense cold and warm/edit guard, the job now times a fixed interpreter workload that does not depend on SQLBuild, in a fresh process, and scales the wall budgets by its fastest repetition over the reference time, floored at 1.0 so a faster runner keeps today's budgets. The calibration times, CPU model, factor and effective budgets are printed with the measurements. Same-runner ratio jobs remain the regression detector. --- .../cli/commands/main/compile/helpers.py | 70 +++++++++++++++++++ .../compile/test_dense_compile_performance.py | 26 +++++-- 2 files changed, 90 insertions(+), 6 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index 25b8b7629e..cbc99162b5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -18,10 +18,12 @@ import zipfile from bisect import bisect_left from collections.abc import Callable, Iterator +from concurrent.futures import ProcessPoolExecutor from contextlib import contextmanager, redirect_stdout from dataclasses import replace from functools import partial from io import StringIO +from multiprocessing import get_context from pathlib import Path from types import FrameType, ModuleType from typing import Any, NamedTuple, cast @@ -41,6 +43,7 @@ from scripts.cold_compile_performance.main.semantic_compile_fingerprint import ( semantic_compile_fingerprint, ) +from scripts.compile_performance_ratio._helpers.report import cpu_model from scripts.compiler_differential.constants import FAILURE_BASE_FILES from scripts.delta_compile_series.models import DeltaCompileOutcome from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR @@ -1119,6 +1122,73 @@ def median_wall_seconds(results: tuple[FreshProcessCompileBenchmarkResult, ...]) return statistics.median(result.elapsed_seconds for result in results) +# The calibration workload's fastest repetition on the hosted runner class the warm and edit +# budgets were set on. A slower runner scales those budgets by its own fastest repetition over +# this; a faster one keeps them as they are. +_RUNNER_CALIBRATION_REFERENCE_SECONDS: float = 0.60 +_RUNNER_CALIBRATION_CHECKSUM: int = 2910182 +_RUNNER_CALIBRATION_REPETITIONS: int = 7 +_RUNNER_CALIBRATION_VALUES: int = 600_000 + + +class RunnerCalibration(NamedTuple): + """How much slower this runner is than the budgets' reference runner, from fixed work.""" + + cpu_model: str + seconds: tuple[float, ...] + factor: float + + +def _runner_calibration_workload() -> int: + """Generate, count, sort and format pseudo-random integers; return a checksum of the work.""" + + state: int = 1 + values: list[int] = [] + for _ in range(_RUNNER_CALIBRATION_VALUES): + state = (state * 6364136223846793005 + 1442695040888963407) % 2**64 + values.append(state >> 33) + counts: dict[int, int] = {} + for value in values: + counts[value & 0xFFFF] = counts.get(value & 0xFFFF, 0) + 1 + values.sort() + rows: list[tuple[int, str]] = [(value, str(value)) for value in values[::2]] + return sum(len(text) for _, text in rows) + len(counts) + + +def _time_runner_calibration() -> tuple[tuple[float, ...], int]: + """Each repetition's seconds and the workload checksum, which pins the work done.""" + + seconds: list[float] = [] + checksum: int = 0 + for _ in range(_RUNNER_CALIBRATION_REPETITIONS): + started: float = time.perf_counter() + checksum = _runner_calibration_workload() + seconds.append(time.perf_counter() - started) + return tuple(seconds), checksum + + +def measure_runner_calibration() -> RunnerCalibration: + """Time the fixed calibration workload in a fresh interpreter and derive the budget factor.""" + + # A fresh interpreter keeps the test process's heap and collector out of the timing. + with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) as pool: + seconds, checksum = pool.submit(_time_runner_calibration).result(timeout=120) + assert checksum == _RUNNER_CALIBRATION_CHECKSUM + # The fastest repetition measures the machine; a busy moment only ever adds time to it. + calibration: RunnerCalibration = RunnerCalibration( + cpu_model=cpu_model(), + seconds=seconds, + factor=max(1.0, min(seconds) / _RUNNER_CALIBRATION_REFERENCE_SECONDS), + ) + print( + f"runner calibration cpu={calibration.cpu_model!r} " + f"runs={[round(run, 3) for run in seconds]} fastest={min(seconds):.3f}s " + f"reference={_RUNNER_CALIBRATION_REFERENCE_SECONDS:.3f}s factor={calibration.factor:.3f}", + flush=True, + ) + return calibration + + def _append_benchmark_edit(path: Path, label: str) -> None: path.write_text( path.read_text(encoding="utf-8") + f"\n-- one {label} edit\n", diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index a233e82057..370d234382 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -35,6 +35,7 @@ analysed_model_count, fresh_process_compile_cache_metrics, measure_model_sql_bytes, + measure_runner_calibration, median_cpu_seconds, median_wall_seconds, run_dense_delta_edit_loop_benchmark, @@ -88,6 +89,9 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ tmp_path: Path, ) -> None: require_compile_memory_scope() + # Fixed work that does not depend on SQLBuild scales the budget on a slower runner class. + wall_budget: float = test_case.expected_max_wall_seconds * measure_runner_calibration().factor + print(f"dense cold models={test_case.model_count} wall_budget={wall_budget:.2f}s", flush=True) project_dir: Path = tmp_path / "dense_orders" write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count) config: RulesConfig = load_rules_config(project_dir=project_dir) @@ -109,7 +113,7 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ result: FreshProcessCompileBenchmarkResult = _run_fresh_process_compile_benchmark( project_dir=project_dir, label=f"dense-{test_case.model_count}", - expected_max_wall_seconds=test_case.expected_max_wall_seconds, + expected_max_wall_seconds=wall_budget, compile_args=("--no-cache",), ) print( @@ -143,7 +147,7 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ assert timings["custom_rules_ms"] > 0 assert timings["custom_rules_cpu_ms"] > 0 assert result.semantic_fingerprint == test_case.expected_fingerprint - assert result.elapsed_seconds < test_case.expected_max_wall_seconds + assert result.elapsed_seconds < wall_budget assert result.peak_rss_bytes < test_case.expected_max_rss_bytes @@ -182,6 +186,16 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches tmp_path: Path, ) -> None: require_compile_memory_scope() + # Fixed work that does not depend on SQLBuild scales the budgets on a slower runner class; + # the same-runner ratio jobs remain what catches a slower compiler. + factor: float = measure_runner_calibration().factor + warm_budget: float = test_case.expected_warm_max_seconds * factor + edit_budget: float = test_case.expected_edit_max_seconds * factor + print( + f"dense models={test_case.model_count} warm_budget={warm_budget:.2f}s " + f"edit_budget={edit_budget:.2f}s", + flush=True, + ) project_dir: Path = tmp_path / "dense_orders" write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count) edited_model_path: Path = dense_model_path( @@ -190,8 +204,8 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches measurements: DenseWarmEditMeasurements = run_dense_warm_edit_benchmark( project_dir=project_dir, edited_model_path=edited_model_path, - expected_warm_max_seconds=test_case.expected_warm_max_seconds, - expected_edit_max_seconds=test_case.expected_edit_max_seconds, + expected_warm_max_seconds=warm_budget, + expected_edit_max_seconds=edit_budget, ) labelled: tuple[tuple[str, FreshProcessCompileBenchmarkResult], ...] = ( ("cold", measurements.cold), @@ -237,8 +251,8 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches ) assert oracle_timings["rule_cache_hits"] == 0 # Medians of same-job runs keep one slow runner moment from failing an unchanged compiler. - assert median_wall_seconds(measurements.warm) < test_case.expected_warm_max_seconds - assert median_wall_seconds(measurements.edit) < test_case.expected_edit_max_seconds + assert median_wall_seconds(measurements.warm) < warm_budget + assert median_wall_seconds(measurements.edit) < edit_budget if __name__ == "__main__": From 3cf8e59f65f8f1491e7fe63002abf5881d521349 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 9 Nov 2026 21:41:29 +0000 Subject: [PATCH 3/3] test(perf): set the runner calibration reference from hosted runner samples Across EPYC 9V45, 9V74 and 7763 and Xeon 8573C and 8370C runners the calibration's fastest repetition ranged from 0.31 s to 0.67 s. The reference is the top of the classes the dense budgets already fit (0.55 s), so those keep factor 1.0 and only measurably slower runners get a larger budget. --- .../e2e/src/sqlbuild/cli/commands/main/compile/helpers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py index cbc99162b5..dda68d48df 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py @@ -1122,10 +1122,10 @@ def median_wall_seconds(results: tuple[FreshProcessCompileBenchmarkResult, ...]) return statistics.median(result.elapsed_seconds for result in results) -# The calibration workload's fastest repetition on the hosted runner class the warm and edit -# budgets were set on. A slower runner scales those budgets by its own fastest repetition over -# this; a faster one keeps them as they are. -_RUNNER_CALIBRATION_REFERENCE_SECONDS: float = 0.60 +# The slowest fastest-repetition seen on the hosted runner classes the dense budgets fit as they +# are. A measurably slower runner scales the budgets by its own fastest repetition over this; a +# faster one keeps them unchanged. +_RUNNER_CALIBRATION_REFERENCE_SECONDS: float = 0.55 _RUNNER_CALIBRATION_CHECKSUM: int = 2910182 _RUNNER_CALIBRATION_REPETITIONS: int = 7 _RUNNER_CALIBRATION_VALUES: int = 600_000