Skip to content

Commit a1d11ab

Browse files
committed
test(compile): tighten the compile wall, memory and delta budgets to CI measurements
Each absolute budget is now about 1.2x the slowest of two CI runs of this branch, and never below 1.15x the slowest recent CI run on the same code (runs based on 0.158.2 or later). That covers the fresh-process, cache, varied-cache, dense cold, warm and edit, and delta edit-loop wall budgets, the delta CPU ratios and the peak RSS limits. A limit already below that floor is unchanged. The fallback CPU limit, the 4 GiB cgroup and the same-runner ratio limits are unchanged.
1 parent 0808c72 commit a1d11ab

4 files changed

Lines changed: 37 additions & 36 deletions

File tree

‎tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -53,28 +53,28 @@
5353
DenseCompileGuardTestCase(
5454
"dense_models_1000_all_rules",
5555
1000,
56-
7.5,
56+
6.0,
5757
800 * _MIB,
5858
"de23b14d87e954a65c2da91918fce062d158419edd63cc8cf3dc938b3e47cd44",
5959
),
6060
DenseCompileGuardTestCase(
6161
"dense_models_3000_all_rules",
6262
3000,
63-
16.5,
63+
12.0,
6464
3 * _GIB // 2,
6565
"196fca3e87989eb3e89e443b7ca93bb094198538d5bde32b9d6fcca58ffc8eed",
6666
),
6767
DenseCompileGuardTestCase(
6868
"dense_models_5000_all_rules",
6969
5000,
70-
26.0,
70+
18.0,
7171
9 * _GIB // 4,
7272
"3477a472fa76d6236ebd0c9758d79c18f67f1bf55e9bbec05b002ba3744cfd22",
7373
),
7474
DenseCompileGuardTestCase(
7575
"dense_models_10000_all_rules",
7676
10000,
77-
51.0,
77+
35.0,
7878
15 * _GIB // 4,
7979
"f2a574af6e8068a0634fa3c078b168dff9a475ada5d71502482ad2926c90e1c3",
8080
),
@@ -154,8 +154,8 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_
154154
"dense_models_3000_warm_and_one_edit",
155155
3000,
156156
1521,
157-
8.0,
158-
8.5,
157+
6.0,
158+
6.0,
159159
3 * _GIB // 2,
160160
"196fca3e87989eb3e89e443b7ca93bb094198538d5bde32b9d6fcca58ffc8eed",
161161
"1afe754dd8c66bc36e38a40c66e9183ecbaf365fd8d87a784069cd615c3ce64a",
@@ -165,8 +165,8 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_
165165
"dense_models_5000_warm_and_one_edit",
166166
5000,
167167
2521,
168-
13.0,
169-
13.5,
168+
9.0,
169+
9.5,
170170
9 * _GIB // 4,
171171
"3477a472fa76d6236ebd0c9758d79c18f67f1bf55e9bbec05b002ba3744cfd22",
172172
"dfbada4c59a3ab8ec7733784890d74c7b64e9b0adcb03364baaabef5bdba5574",
@@ -240,9 +240,9 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches
240240
model_count=3000,
241241
edited_model_index=1521,
242242
full_pairs=4,
243-
expected_max_full_seconds=15.0,
244-
expected_max_delta_seconds=4.5,
245-
expected_max_delta_cpu_ratio=0.375,
243+
expected_max_full_seconds=7.0,
244+
expected_max_delta_seconds=2.0,
245+
expected_max_delta_cpu_ratio=0.30,
246246
expected_max_fallback_cpu_ratio=1.05,
247247
expected_max_rss_bytes=3 * _GIB // 2,
248248
),
@@ -251,9 +251,9 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches
251251
model_count=10000,
252252
edited_model_index=5021,
253253
full_pairs=3,
254-
expected_max_full_seconds=60.0,
255-
expected_max_delta_seconds=9.0,
256-
expected_max_delta_cpu_ratio=0.30,
254+
expected_max_full_seconds=28.0,
255+
expected_max_delta_seconds=8.0,
256+
expected_max_delta_cpu_ratio=0.275,
257257
expected_max_fallback_cpu_ratio=1.05,
258258
expected_max_rss_bytes=15 * _GIB // 4,
259259
),

‎tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py‎

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -50,14 +50,14 @@
5050
macro_count=500,
5151
test_count=2_945,
5252
audit_count=5_056,
53-
expected_cold_max_wall_seconds=10.5,
54-
expected_warm_max_wall_seconds=6.0,
55-
expected_edit_max_wall_seconds=6.0,
53+
expected_cold_max_wall_seconds=9.0,
54+
expected_warm_max_wall_seconds=4.5,
55+
expected_edit_max_wall_seconds=4.5,
5656
expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO,
5757
expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO,
5858
expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO,
5959
expected_max_cache_write_wall_overhead_ratio=_MAX_CACHE_WRITE_WALL_OVERHEAD_RATIO,
60-
expected_max_rss_bytes=2 * _GIB,
60+
expected_max_rss_bytes=1152 * _MIB,
6161
expected_max_cache_bytes=96 * _MIB,
6262
expected_cold_fingerprint=(
6363
"1914b788abcb452422eaf6c3a2c240ff390bab9530217285a5ca93334db7b6c3"
@@ -84,14 +84,14 @@
8484
macro_count=834,
8585
test_count=4_908,
8686
audit_count=8_427,
87-
expected_cold_max_wall_seconds=18.0,
88-
expected_warm_max_wall_seconds=10.5,
89-
expected_edit_max_wall_seconds=10.5,
87+
expected_cold_max_wall_seconds=14.5,
88+
expected_warm_max_wall_seconds=7.0,
89+
expected_edit_max_wall_seconds=7.0,
9090
expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO,
9191
expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO,
9292
expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO,
9393
expected_max_cache_write_wall_overhead_ratio=_MAX_CACHE_WRITE_WALL_OVERHEAD_RATIO,
94-
expected_max_rss_bytes=2 * _GIB,
94+
expected_max_rss_bytes=1664 * _MIB,
9595
expected_max_cache_bytes=160 * _MIB,
9696
expected_cold_fingerprint=(
9797
"2f1e5dfd29025c25e0dffc6e74f4c2be020a5ef133ecd081f44e0c55c24858b4"
@@ -118,15 +118,15 @@
118118
macro_count=1_667,
119119
test_count=9_816,
120120
audit_count=16_855,
121-
expected_cold_max_wall_seconds=34.5,
122-
expected_warm_max_wall_seconds=19.5,
123-
expected_edit_max_wall_seconds=19.5,
121+
expected_cold_max_wall_seconds=28.0,
122+
expected_warm_max_wall_seconds=14.0,
123+
expected_edit_max_wall_seconds=13.5,
124124
expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO,
125125
expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO,
126126
expected_max_cache_write_cpu_overhead_ratio=_MAX_LARGE_CACHE_WRITE_CPU_OVERHEAD_RATIO,
127127
expected_max_cache_write_wall_overhead_ratio=_MAX_CACHE_WRITE_WALL_OVERHEAD_RATIO,
128128
# Temporary memory exception (2026-10-10); remove after conversion optimisation.
129-
expected_max_rss_bytes=3 * _GIB,
129+
expected_max_rss_bytes=3008 * _MIB,
130130
expected_max_cache_bytes=320 * _MIB,
131131
expected_cold_fingerprint=(
132132
"ecdd7f2cf961fab739aa806b6d90283916cffad2007e1e2f3ba90a466db6cb75"

‎tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_performance.py‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@
1919
)
2020

2121
_LOGGER: logging.Logger = logging.getLogger(__name__)
22-
_GIB: int = 1024 * 1024 * 1024
22+
_MIB: int = 1024 * 1024
23+
_GIB: int = 1024 * _MIB
2324

2425

2526
@pytest.mark.performance
@@ -36,8 +37,8 @@
3637
macro_count=37,
3738
test_count=2_945,
3839
audit_count=5_056,
39-
expected_max_wall_seconds=9.5,
40-
expected_max_rss_bytes=int(1.5 * _GIB),
40+
expected_max_wall_seconds=8.0,
41+
expected_max_rss_bytes=1088 * _MIB,
4142
expected_semantic_fingerprint=(
4243
"d935821df54043fabc3a6911c1926578178c9b744f2a589cbef99a10d5207543"
4344
),
@@ -51,8 +52,8 @@
5152
macro_count=61,
5253
test_count=4_908,
5354
audit_count=8_427,
54-
expected_max_wall_seconds=16.0,
55-
expected_max_rss_bytes=int(1.75 * _GIB),
55+
expected_max_wall_seconds=12.5,
56+
expected_max_rss_bytes=1664 * _MIB,
5657
expected_semantic_fingerprint=(
5758
"874cbfd473846c6a335ec2f79cb6b7b0b77e8e8d58652815fad1e9d59d423c46"
5859
),
@@ -66,7 +67,7 @@
6667
macro_count=123,
6768
test_count=9_816,
6869
audit_count=16_855,
69-
expected_max_wall_seconds=31.0,
70+
expected_max_wall_seconds=25.5,
7071
expected_max_rss_bytes=int(2.5 * _GIB),
7172
expected_semantic_fingerprint=(
7273
"5ccdabeaabfdfba1ecbae80259dd3c6eb801d683df140b7fb7a249b9555308c6"

‎tests/e2e/src/sqlbuild/cli/commands/main/compile/test_varied_compile_cache_performance.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,10 @@
3131
VariedCompileCacheTestCase(
3232
"models_1000_varied_shared_graph",
3333
1000,
34-
24.0,
35-
10.0,
36-
13.0,
37-
2 * 1024**3,
34+
9.0,
35+
2.5,
36+
2.5,
37+
832 * 1024**2,
3838
950,
3939
900,
4040
40,

0 commit comments

Comments
 (0)