From a1d11abe8360df4ce34d228ef531af2467d19a5a Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Mon, 19 Oct 2026 14:45:31 +0100 Subject: [PATCH] test(compile): tighten the compile wall, memory and delta budgets to CI measurements Each absolute budget is now about 1.2x the slowest of two CI runs of this branch, and never below 1.15x the slowest recent CI run on the same code (runs based on 0.158.2 or later). That covers the fresh-process, cache, varied-cache, dense cold, warm and edit, and delta edit-loop wall budgets, the delta CPU ratios and the peak RSS limits. A limit already below that floor is unchanged. The fallback CPU limit, the 4 GiB cgroup and the same-runner ratio limits are unchanged. --- .../compile/test_dense_compile_performance.py | 28 +++++++++---------- ...fresh_process_compile_cache_performance.py | 24 ++++++++-------- .../test_fresh_process_compile_performance.py | 13 +++++---- .../test_varied_compile_cache_performance.py | 8 +++--- 4 files changed, 37 insertions(+), 36 deletions(-) diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py index c2c786313e..8e7d1e126a 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py @@ -53,28 +53,28 @@ DenseCompileGuardTestCase( "dense_models_1000_all_rules", 1000, - 7.5, + 6.0, 800 * _MIB, "de23b14d87e954a65c2da91918fce062d158419edd63cc8cf3dc938b3e47cd44", ), DenseCompileGuardTestCase( "dense_models_3000_all_rules", 3000, - 16.5, + 12.0, 3 * _GIB // 2, "196fca3e87989eb3e89e443b7ca93bb094198538d5bde32b9d6fcca58ffc8eed", ), DenseCompileGuardTestCase( "dense_models_5000_all_rules", 5000, - 26.0, + 18.0, 9 * _GIB // 4, "3477a472fa76d6236ebd0c9758d79c18f67f1bf55e9bbec05b002ba3744cfd22", ), DenseCompileGuardTestCase( "dense_models_10000_all_rules", 10000, - 51.0, + 35.0, 15 * _GIB // 4, "f2a574af6e8068a0634fa3c078b168dff9a475ada5d71502482ad2926c90e1c3", ), @@ -154,8 +154,8 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ "dense_models_3000_warm_and_one_edit", 3000, 1521, - 8.0, - 8.5, + 6.0, + 6.0, 3 * _GIB // 2, "196fca3e87989eb3e89e443b7ca93bb094198538d5bde32b9d6fcca58ffc8eed", "1afe754dd8c66bc36e38a40c66e9183ecbaf365fd8d87a784069cd615c3ce64a", @@ -165,8 +165,8 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_ "dense_models_5000_warm_and_one_edit", 5000, 2521, - 13.0, - 13.5, + 9.0, + 9.5, 9 * _GIB // 4, "3477a472fa76d6236ebd0c9758d79c18f67f1bf55e9bbec05b002ba3744cfd22", "dfbada4c59a3ab8ec7733784890d74c7b64e9b0adcb03364baaabef5bdba5574", @@ -240,9 +240,9 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches model_count=3000, edited_model_index=1521, full_pairs=4, - expected_max_full_seconds=15.0, - expected_max_delta_seconds=4.5, - expected_max_delta_cpu_ratio=0.375, + expected_max_full_seconds=7.0, + expected_max_delta_seconds=2.0, + expected_max_delta_cpu_ratio=0.30, expected_max_fallback_cpu_ratio=1.05, expected_max_rss_bytes=3 * _GIB // 2, ), @@ -251,9 +251,9 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches model_count=10000, edited_model_index=5021, full_pairs=3, - expected_max_full_seconds=60.0, - expected_max_delta_seconds=9.0, - expected_max_delta_cpu_ratio=0.30, + expected_max_full_seconds=28.0, + expected_max_delta_seconds=8.0, + expected_max_delta_cpu_ratio=0.275, expected_max_fallback_cpu_ratio=1.05, expected_max_rss_bytes=15 * _GIB // 4, ), diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py index 601275d93e..aeb16ae9de 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py @@ -50,14 +50,14 @@ macro_count=500, test_count=2_945, audit_count=5_056, - expected_cold_max_wall_seconds=10.5, - expected_warm_max_wall_seconds=6.0, - expected_edit_max_wall_seconds=6.0, + expected_cold_max_wall_seconds=9.0, + expected_warm_max_wall_seconds=4.5, + expected_edit_max_wall_seconds=4.5, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO, expected_max_cache_write_wall_overhead_ratio=_MAX_CACHE_WRITE_WALL_OVERHEAD_RATIO, - expected_max_rss_bytes=2 * _GIB, + expected_max_rss_bytes=1152 * _MIB, expected_max_cache_bytes=96 * _MIB, expected_cold_fingerprint=( "1914b788abcb452422eaf6c3a2c240ff390bab9530217285a5ca93334db7b6c3" @@ -84,14 +84,14 @@ macro_count=834, test_count=4_908, audit_count=8_427, - expected_cold_max_wall_seconds=18.0, - expected_warm_max_wall_seconds=10.5, - expected_edit_max_wall_seconds=10.5, + expected_cold_max_wall_seconds=14.5, + expected_warm_max_wall_seconds=7.0, + expected_edit_max_wall_seconds=7.0, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO, expected_max_cache_write_wall_overhead_ratio=_MAX_CACHE_WRITE_WALL_OVERHEAD_RATIO, - expected_max_rss_bytes=2 * _GIB, + expected_max_rss_bytes=1664 * _MIB, expected_max_cache_bytes=160 * _MIB, expected_cold_fingerprint=( "2f1e5dfd29025c25e0dffc6e74f4c2be020a5ef133ecd081f44e0c55c24858b4" @@ -118,15 +118,15 @@ macro_count=1_667, test_count=9_816, audit_count=16_855, - expected_cold_max_wall_seconds=34.5, - expected_warm_max_wall_seconds=19.5, - expected_edit_max_wall_seconds=19.5, + expected_cold_max_wall_seconds=28.0, + expected_warm_max_wall_seconds=14.0, + expected_edit_max_wall_seconds=13.5, expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO, expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO, expected_max_cache_write_cpu_overhead_ratio=_MAX_LARGE_CACHE_WRITE_CPU_OVERHEAD_RATIO, expected_max_cache_write_wall_overhead_ratio=_MAX_CACHE_WRITE_WALL_OVERHEAD_RATIO, # Temporary memory exception (2026-10-10); remove after conversion optimisation. - expected_max_rss_bytes=3 * _GIB, + expected_max_rss_bytes=3008 * _MIB, expected_max_cache_bytes=320 * _MIB, expected_cold_fingerprint=( "ecdd7f2cf961fab739aa806b6d90283916cffad2007e1e2f3ba90a466db6cb75" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py index 0431b89012..c1bd3fc98c 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py @@ -19,7 +19,8 @@ ) _LOGGER: logging.Logger = logging.getLogger(__name__) -_GIB: int = 1024 * 1024 * 1024 +_MIB: int = 1024 * 1024 +_GIB: int = 1024 * _MIB @pytest.mark.performance @@ -36,8 +37,8 @@ macro_count=37, test_count=2_945, audit_count=5_056, - expected_max_wall_seconds=9.5, - expected_max_rss_bytes=int(1.5 * _GIB), + expected_max_wall_seconds=8.0, + expected_max_rss_bytes=1088 * _MIB, expected_semantic_fingerprint=( "d935821df54043fabc3a6911c1926578178c9b744f2a589cbef99a10d5207543" ), @@ -51,8 +52,8 @@ macro_count=61, test_count=4_908, audit_count=8_427, - expected_max_wall_seconds=16.0, - expected_max_rss_bytes=int(1.75 * _GIB), + expected_max_wall_seconds=12.5, + expected_max_rss_bytes=1664 * _MIB, expected_semantic_fingerprint=( "874cbfd473846c6a335ec2f79cb6b7b0b77e8e8d58652815fad1e9d59d423c46" ), @@ -66,7 +67,7 @@ macro_count=123, test_count=9_816, audit_count=16_855, - expected_max_wall_seconds=31.0, + expected_max_wall_seconds=25.5, expected_max_rss_bytes=int(2.5 * _GIB), expected_semantic_fingerprint=( "5ccdabeaabfdfba1ecbae80259dd3c6eb801d683df140b7fb7a249b9555308c6" diff --git a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py index cfbe7d947a..af776b394f 100644 --- a/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py +++ b/tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py @@ -31,10 +31,10 @@ VariedCompileCacheTestCase( "models_1000_varied_shared_graph", 1000, - 24.0, - 10.0, - 13.0, - 2 * 1024**3, + 9.0, + 2.5, + 2.5, + 832 * 1024**2, 950, 900, 40,