diff --git a/.github/workflows/verify.yml b/.github/workflows/verify.yml index cbec65228..22711433e 100644 --- a/.github/workflows/verify.yml +++ b/.github/workflows/verify.yml @@ -845,20 +845,10 @@ jobs: include: - kind: dense max_ratio: "1.10" - # Temporary native-analysis allowance on the cold compile only; restore 1.10 when the - # native analysis cold path is optimised. Measured 1.326/1.347 wall and 1.642/1.677 CPU - # over main, plus 10%. - cold_max_ratio: "1.49" - cold_max_cpu_ratio: "1.85" edit_max_ratio: "1.20" edit_max_cpu_ratio: "1.75" - kind: fresh max_ratio: "1.10" - # Temporary native-analysis allowance on the cold compile only; restore 1.10 when the - # native analysis cold path is optimised. Measured 1.285/1.153 wall and 1.551/1.404 CPU - # over main, plus 10%. - cold_max_ratio: "1.42" - cold_max_cpu_ratio: "1.71" - kind: dense label: contracts CPU vs base gate_phase: contracts_cpu_ms @@ -867,9 +857,7 @@ jobs: label: model analysis CPU vs base gate_phase: model_analysis_cpu_ms compile_args: --compile-arg=--select=* - # Temporary native-analysis allowance; restore 1.10 when the native analysis path is - # optimised. Measured 2.660/2.554 over main, plus 10%. - max_ratio: "2.93" + max_ratio: "1.10" - kind: dense label: custom rules CPU vs base gate_phase: custom_rules_cpu_ms diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md index ba7adcfde..90332a256 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md @@ -88,11 +88,13 @@ All cases carry `models_3000` identifiers so they run only in the 3,000-model fr ## Same-runner comparison with the base branch -Absolute budgets must absorb the spread between hosted runners, so every pull request that touches -the compiler also runs `scripts/compare_compile_performance.py` on the dense and the fresh -3,000-model benchmark. Base and head each compile a project written by their own generator, on the -same runner, three alternating runs per mode, and the medians of wall and CPU time must stay within -1.10x of base for each mode: +Absolute budgets must absorb the spread between hosted runner classes, up to 1.9x on the same +compile: the dense and fresh-process cache guards scale their wall budgets by a fixed calibration +workload on slower classes. An absolute budget therefore cannot catch a slower compiler, so every +pull request that touches the compiler also runs `scripts/compare_compile_performance.py` on the +dense and the fresh 3,000-model benchmark. Base and head each compile a project written by their +own generator, on the same runner, three alternating runs per mode, and the medians of wall and CPU +time must stay within 1.10x of base for each mode: - cold: `sqb compile --json --no-cache` after removing `target`, after one untimed warm-up; - warm: an unchanged `sqb compile --json` after one untimed run that writes the cache; diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index e4526c4b7..9d997e32e 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -45,8 +45,8 @@ _MIB: int = 1024 * 1024 _GIB: int = 1024 * _MIB # Temporary allowance for the native parser switch: each cold wall budget below is its original -# value times the approved 1.49 dense cold wall ratio ceiling. Drop the multiplier, back to the -# original value, when the 1.10x same-runner ratio guards return. +# value times 1.49, the approved dense cold slowdown of native analysis. Drop the multiplier, back +# to the original value, when the native analysis cold path is optimised. _NATIVE_COLD_WALL_ALLOWANCE: float = 1.49 # A delta reruns the edited model or the edited SQL test's few models; a full compile all of them. _MAX_DELTA_ANALYSED_MODELS: int = 10 diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py index 4fef91969..529eac388 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py @@ -20,6 +20,7 @@ assert_complete_compile_cache_hit, assert_successful_compile_cache_payload, fresh_process_compile_cache_metrics, + measure_runner_calibration, run_fresh_process_compile_cache_benchmark, ) @@ -200,6 +201,21 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in test_case: FreshProcessCompileCachePerformanceGuardTestCase, ) -> None: assert_required_cgroup_memory_limit() + # Fixed work that does not depend on SQLBuild scales the wall budgets on a slower runner + # class; the in-job ratios below and the same-runner ratio jobs catch a slower compiler. + factor: float = measure_runner_calibration().factor + cold_budget: float = test_case.expected_cold_max_wall_seconds * factor + warm_budget: float = test_case.expected_warm_max_wall_seconds * factor + edit_budget: float = test_case.expected_edit_max_wall_seconds * factor + project_config_edit_budget: float = ( + test_case.expected_project_config_edit_max_wall_seconds * factor + ) + print( + f"fresh-process cache models={test_case.model_count} cold_budget={cold_budget:.2f}s " + f"warm_budget={warm_budget:.2f}s edit_budget={edit_budget:.2f}s " + f"project_config_edit_budget={project_config_edit_budget:.2f}s", + flush=True, + ) result: FreshProcessCompileCacheBenchmarkResult = run_fresh_process_compile_cache_benchmark( project_dir=tmp_path / f"semantic_cache_{test_case.model_count}", model_count=test_case.model_count, @@ -209,12 +225,10 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in macro_count=test_case.macro_count, test_count=test_case.test_count, audit_count=test_case.audit_count, - expected_cold_max_wall_seconds=test_case.expected_cold_max_wall_seconds, - expected_warm_max_wall_seconds=test_case.expected_warm_max_wall_seconds, - expected_edit_max_wall_seconds=test_case.expected_edit_max_wall_seconds, - expected_project_config_edit_max_wall_seconds=( - test_case.expected_project_config_edit_max_wall_seconds - ), + expected_cold_max_wall_seconds=cold_budget, + expected_warm_max_wall_seconds=warm_budget, + expected_edit_max_wall_seconds=edit_budget, + expected_project_config_edit_max_wall_seconds=project_config_edit_budget, macro_call_interval=test_case.macro_call_interval, scoped_macros=test_case.scoped_macros, compiler_only=test_case.compiler_only, @@ -246,7 +260,7 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in assert_successful_compile_cache_payload(measurement=measurement, test_case=test_case) assert measurement.peak_rss_bytes < test_case.expected_max_rss_bytes - assert result.cold.elapsed_seconds < test_case.expected_cold_max_wall_seconds + assert result.cold.elapsed_seconds < cold_budget cache_write_wall_ratio: float = ( result.cold.elapsed_seconds / result.cache_disabled.elapsed_seconds ) @@ -265,7 +279,7 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in result.after_macro_edit, result.after_project_config_edit, ): - assert measurement.elapsed_seconds < test_case.expected_warm_max_wall_seconds + assert measurement.elapsed_seconds < warm_budget for measurement in (result.warm, result.after_leaf_edit, result.after_macro_edit): assert ( measurement.elapsed_seconds / result.cold.elapsed_seconds @@ -276,15 +290,12 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in <= _MAX_AFTER_PROJECT_REWRITE_TO_COLD_RATIO ) for measurement in (result.leaf_edit, result.macro_edit): - assert measurement.elapsed_seconds < test_case.expected_edit_max_wall_seconds + assert measurement.elapsed_seconds < edit_budget assert ( measurement.elapsed_seconds / result.cold.elapsed_seconds <= test_case.expected_max_edit_to_cold_ratio ) - assert ( - result.project_config_edit.elapsed_seconds - < test_case.expected_project_config_edit_max_wall_seconds - ) + assert result.project_config_edit.elapsed_seconds < project_config_edit_budget assert result.cache_bytes < test_case.expected_max_cache_bytes assert result.cache_disabled.semantic_fingerprint == test_case.expected_cold_fingerprint diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py index cab25e452..199387d04 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py @@ -23,8 +23,8 @@ _MIB: int = 1024 * 1024 _GIB: int = 1024 * _MIB # Temporary allowance for the native parser switch: each wall budget below is its original value -# times the approved 1.42 fresh cold wall ratio ceiling. Drop the multiplier, back to the original -# value, when the 1.10x same-runner ratio guards return. +# times 1.42, the approved fresh cold slowdown of native analysis. Drop the multiplier, back to the +# original value, when the native analysis cold path is optimised. _NATIVE_COLD_WALL_ALLOWANCE: float = 1.42