From 871726b007c34a2648cfe399a548915bb060d298 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Wed, 11 Nov 2026 09:24:23 +0000 Subject: [PATCH] test(perf): scale the cache compile guards to the runner and restore the 1.10x cold ratio The fresh-process cache guards failed on runner speed alone: cold wall and CPU spread 1.6x to 1.9x between hosted runner classes, while cold over the in-job cache-disabled control stayed within 1.05 on CPU. Their wall budgets now scale by the fixed calibration workload the dense guards already use. A slower compiler is caught by the same-runner base/head comparisons. Their cold and model analysis allowances were sized against main before native analysis; main now has it on both sides, and recent readings sit at 0.95 to 1.03, so they return to 1.10x. An injected slowdown of 20% of cold wall fails the restored limit at 1.234. --- .github/workflows/verify.yml | 14 +------ .../main/compile/COMPILE_BENCHMARKS.md | 12 +++--- .../compile/test_dense_compile_performance.py | 4 +- ...fresh_process_compile_cache_performance.py | 37 ++++++++++++------- .../test_fresh_process_compile_performance.py | 4 +- 5 files changed, 36 insertions(+), 35 deletions(-) diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index cbec652280..22711433e8 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -845,20 +845,10 @@ jobs: include: - kind: dense max_ratio: "1.10" - # Temporary native-analysis allowance on the cold compile only; restore 1.10 when the - # native analysis cold path is optimised. Measured 1.326/1.347 wall and 1.642/1.677 CPU - # over main, plus 10%. - cold_max_ratio: "1.49" - cold_max_cpu_ratio: "1.85" edit_max_ratio: "1.20" edit_max_cpu_ratio: "1.75" - kind: fresh max_ratio: "1.10" - # Temporary native-analysis allowance on the cold compile only; restore 1.10 when the - # native analysis cold path is optimised. Measured 1.285/1.153 wall and 1.551/1.404 CPU - # over main, plus 10%. - cold_max_ratio: "1.42" - cold_max_cpu_ratio: "1.71" - kind: dense label: contracts CPU vs base gate_phase: contracts_cpu_ms @@ -867,9 +857,7 @@ jobs: label: model analysis CPU vs base gate_phase: model_analysis_cpu_ms compile_args: --compile-arg=--select=* - # Temporary native-analysis allowance; restore 1.10 when the native analysis path is - # optimised. Measured 2.660/2.554 over main, plus 10%. - max_ratio: "2.93" + max_ratio: "1.10" - kind: dense label: custom rules CPU vs base gate_phase: custom_rules_cpu_ms diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md index ba7adcfdeb..90332a2565 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md @@ -88,11 +88,13 @@ All cases carry `models_3000` identifiers so they run only in the 3,000-model fr ## Same-runner comparison with the base branch -Absolute budgets must absorb the spread between hosted runners, so every pull request that touches -the compiler also runs `scripts/compare_compile_performance.py` on the dense and the fresh -3,000-model benchmark. Base and head each compile a project written by their own generator, on the -same runner, three alternating runs per mode, and the medians of wall and CPU time must stay within -1.10x of base for each mode: +Absolute budgets must absorb the spread between hosted runner classes, up to 1.9x on the same +compile: the dense and fresh-process cache guards scale their wall budgets by a fixed calibration +workload on slower classes. An absolute budget therefore cannot catch a slower compiler, so every +pull request that touches the compiler also runs `scripts/compare_compile_performance.py` on the +dense and the fresh 3,000-model benchmark. Base and head each compile a project written by their +own generator, on the same runner, three alternating runs per mode, and the medians of wall and CPU +time must stay within 1.10x of base for each mode: - cold: `sqb compile --json --no-cache` after removing `target`, after one untimed warm-up; - warm: an unchanged `sqb compile --json` after one untimed run that writes the cache; diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index e4526c4b7f..9d997e32e5 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -45,8 +45,8 @@ _MIB: int = 1024 * 1024 _GIB: int = 1024 * _MIB # Temporary allowance for the native parser switch: each cold wall budget below is its original -# value times the approved 1.49 dense cold wall ratio ceiling. Drop the multiplier, back to the -# original value, when the 1.10x same-runner ratio guards return. +# value times 1.49, the approved dense cold slowdown of native analysis. Drop the multiplier, back +# to the original value, when the native analysis cold path is optimised. _NATIVE_COLD_WALL_ALLOWANCE: float = 1.49 # A delta reruns the edited model or the edited SQL test's few models; a full compile all of them. _MAX_DELTA_ANALYSED_MODELS: int = 10 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py index 4fef919698..529eac3886 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py @@ -20,6 +20,7 @@ assert_complete_compile_cache_hit, assert_successful_compile_cache_payload, fresh_process_compile_cache_metrics, + measure_runner_calibration, run_fresh_process_compile_cache_benchmark, ) @@ -200,6 +201,21 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in test_case: FreshProcessCompileCachePerformanceGuardTestCase, ) -> None: assert_required_cgroup_memory_limit() + # Fixed work that does not depend on SQLBuild scales the wall budgets on a slower runner + # class; the in-job ratios below and the same-runner ratio jobs catch a slower compiler. + factor: float = measure_runner_calibration().factor + cold_budget: float = test_case.expected_cold_max_wall_seconds * factor + warm_budget: float = test_case.expected_warm_max_wall_seconds * factor + edit_budget: float = test_case.expected_edit_max_wall_seconds * factor + project_config_edit_budget: float = ( + test_case.expected_project_config_edit_max_wall_seconds * factor + ) + print( + f"fresh-process cache models={test_case.model_count} cold_budget={cold_budget:.2f}s " + f"warm_budget={warm_budget:.2f}s edit_budget={edit_budget:.2f}s " + f"project_config_edit_budget={project_config_edit_budget:.2f}s", + flush=True, + ) result: FreshProcessCompileCacheBenchmarkResult = run_fresh_process_compile_cache_benchmark( project_dir=tmp_path / f"semantic_cache_{test_case.model_count}", model_count=test_case.model_count, @@ -209,12 +225,10 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in macro_count=test_case.macro_count, test_count=test_case.test_count, audit_count=test_case.audit_count, - expected_cold_max_wall_seconds=test_case.expected_cold_max_wall_seconds, - expected_warm_max_wall_seconds=test_case.expected_warm_max_wall_seconds, - expected_edit_max_wall_seconds=test_case.expected_edit_max_wall_seconds, - expected_project_config_edit_max_wall_seconds=( - test_case.expected_project_config_edit_max_wall_seconds - ), + expected_cold_max_wall_seconds=cold_budget, + expected_warm_max_wall_seconds=warm_budget, + expected_edit_max_wall_seconds=edit_budget, + expected_project_config_edit_max_wall_seconds=project_config_edit_budget, macro_call_interval=test_case.macro_call_interval, scoped_macros=test_case.scoped_macros, compiler_only=test_case.compiler_only, @@ -246,7 +260,7 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in assert_successful_compile_cache_payload(measurement=measurement, test_case=test_case) assert measurement.peak_rss_bytes < test_case.expected_max_rss_bytes - assert result.cold.elapsed_seconds < test_case.expected_cold_max_wall_seconds + assert result.cold.elapsed_seconds < cold_budget cache_write_wall_ratio: float = ( result.cold.elapsed_seconds / result.cache_disabled.elapsed_seconds ) @@ -265,7 +279,7 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in result.after_macro_edit, result.after_project_config_edit, ): - assert measurement.elapsed_seconds < test_case.expected_warm_max_wall_seconds + assert measurement.elapsed_seconds < warm_budget for measurement in (result.warm, result.after_leaf_edit, result.after_macro_edit): assert ( measurement.elapsed_seconds / result.cold.elapsed_seconds @@ -276,15 +290,12 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in <= _MAX_AFTER_PROJECT_REWRITE_TO_COLD_RATIO ) for measurement in (result.leaf_edit, result.macro_edit): - assert measurement.elapsed_seconds < test_case.expected_edit_max_wall_seconds + assert measurement.elapsed_seconds < edit_budget assert ( measurement.elapsed_seconds / result.cold.elapsed_seconds <= test_case.expected_max_edit_to_cold_ratio ) - assert ( - result.project_config_edit.elapsed_seconds - < test_case.expected_project_config_edit_max_wall_seconds - ) + assert result.project_config_edit.elapsed_seconds < project_config_edit_budget assert result.cache_bytes < test_case.expected_max_cache_bytes assert result.cache_disabled.semantic_fingerprint == test_case.expected_cold_fingerprint diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py index cab25e452e..199387d04e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py @@ -23,8 +23,8 @@ _MIB: int = 1024 * 1024 _GIB: int = 1024 * _MIB # Temporary allowance for the native parser switch: each wall budget below is its original value -# times the approved 1.42 fresh cold wall ratio ceiling. Drop the multiplier, back to the original -# value, when the 1.10x same-runner ratio guards return. +# times 1.42, the approved fresh cold slowdown of native analysis. Drop the multiplier, back to the +# original value, when the native analysis cold path is optimised. _NATIVE_COLD_WALL_ALLOWANCE: float = 1.42