Skip to content

Commit c87bca8

Browse files
authored
Merge branch 'main' into feat/recall-clickhouse-store
2 parents 4796ff1 + 2c22eb5 commit c87bca8

4 files changed

Lines changed: 27 additions & 15 deletions

File tree

‎tests/e2e/src/sqlbuild/cli/commands/main/compile/COMPILE_BENCHMARKS.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -75,9 +75,10 @@ behind a loose timing budget. It runs in the 3,000-model fresh-process compile j
7575

7676
The same inspection project also bounds `sqb plan --json` against an empty DuckDB warehouse. The
7777
plan budget is a ratio to a warm `sqb compile --json` measured in the same test on the same runner,
78-
plus a peak-RSS cap. A second case plans a 300-model copy of the benchmark and fails when the
79-
planning CPU of the large project (plan CPU minus compile CPU) exceeds twice its linear projection
80-
from the small one, so super-linear planning work fails on any runner.
78+
plus a peak-RSS cap. The CPU-time ratio is the tight limit; a looser wall-time ratio backstops lost
79+
parallelism and lock or I/O waits. A second case plans a 300-model copy of the benchmark and fails
80+
when the planning CPU of the large project (plan CPU minus compile CPU) exceeds twice its linear
81+
projection from the small one, so super-linear planning work fails on any runner.
8182

8283
`test_build_performance.py` builds a 1,000-model DuckDB benchmark without tests or audits from an
8384
empty warehouse once per module, with wall and peak-RSS limits. Its existing-state cases copy the

‎tests/e2e/src/sqlbuild/cli/commands/main/compile/_test_types.py‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -368,7 +368,8 @@ class InspectionCommandPerformanceGuardTestCase:
368368
@dataclass(frozen=True)
369369
class PlanComparedToCompileGuardTestCase:
370370
description: str
371-
expected_max_compile_ratio: float
371+
expected_max_compile_cpu_ratio: float
372+
expected_max_compile_wall_ratio: float
372373
expected_max_plan_wall_seconds: float
373374
expected_max_rss_bytes: int
374375

‎tests/e2e/src/sqlbuild/cli/commands/main/compile/test_fresh_process_compile_cache_performance.py‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,10 @@
3131
_MAX_AFTER_PROJECT_REWRITE_TO_COLD_RATIO: float = 0.8
3232
_MAX_EDIT_TO_COLD_RATIO: float = 0.70
3333
# Writing the cache during a cold compile must stay a small same-runner cost over --no-cache.
34-
_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO: float = 1.10
35-
_MAX_CACHE_WRITE_WALL_OVERHEAD_RATIO: float = 1.20
36-
_MAX_LARGE_CACHE_WRITE_CPU_OVERHEAD_RATIO: float = 1.15
34+
# CPU time is the tight guard because it is steady on shared runners; the looser wall-time
35+
# backstop still catches lost parallelism, lock waits and write I/O regressions.
36+
_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO: float = 1.25
37+
_MAX_CACHE_WRITE_WALL_OVERHEAD_RATIO: float = 1.35
3738

3839

3940
@pytest.mark.performance
@@ -123,7 +124,7 @@
123124
expected_edit_max_wall_seconds=13.5,
124125
expected_max_warm_to_cold_ratio=_MAX_WARM_TO_COLD_RATIO,
125126
expected_max_edit_to_cold_ratio=_MAX_EDIT_TO_COLD_RATIO,
126-
expected_max_cache_write_cpu_overhead_ratio=_MAX_LARGE_CACHE_WRITE_CPU_OVERHEAD_RATIO,
127+
expected_max_cache_write_cpu_overhead_ratio=_MAX_CACHE_WRITE_CPU_OVERHEAD_RATIO,
127128
expected_max_cache_write_wall_overhead_ratio=_MAX_CACHE_WRITE_WALL_OVERHEAD_RATIO,
128129
# Temporary memory exception (2026-10-10); remove after conversion optimisation.
129130
expected_max_rss_bytes=3008 * _MIB,
@@ -180,11 +181,12 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in
180181
}
181182
for label, measurement in measurements.items():
182183
_LOGGER.info(
183-
"fresh-process cache models=%d path=%s wall=%.3fs peak_rss_bytes=%d "
184+
"fresh-process cache models=%d path=%s wall=%.3fs cpu=%.3fs peak_rss_bytes=%d "
184185
"fingerprint=%s metrics=%s",
185186
test_case.model_count,
186187
label,
187188
measurement.elapsed_seconds,
189+
measurement.cpu_seconds,
188190
measurement.peak_rss_bytes,
189191
measurement.semantic_fingerprint,
190192
fresh_process_compile_cache_metrics(measurement),
@@ -204,8 +206,8 @@ def test_given_semantic_project_when_compiling_across_processes_then_cache_is_in
204206
cache_write_wall_ratio,
205207
cache_write_cpu_ratio,
206208
)
207-
assert cache_write_wall_ratio <= test_case.expected_max_cache_write_wall_overhead_ratio
208209
assert cache_write_cpu_ratio <= test_case.expected_max_cache_write_cpu_overhead_ratio
210+
assert cache_write_wall_ratio <= test_case.expected_max_cache_write_wall_overhead_ratio
209211
for measurement in (
210212
result.warm,
211213
result.after_leaf_edit,

‎tests/e2e/src/sqlbuild/cli/commands/main/compile/test_inspection_command_performance.py‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,8 @@ def test_given_shared_dependency_benchmark_when_inspecting_then_preserves_resour
202202
[
203203
PlanComparedToCompileGuardTestCase(
204204
description="models_3000_plan_json_relative_to_warm_compile",
205-
expected_max_compile_ratio=2.6,
205+
expected_max_compile_cpu_ratio=3.1,
206+
expected_max_compile_wall_ratio=3.3,
206207
expected_max_plan_wall_seconds=45.0,
207208
expected_max_rss_bytes=1_280 * _MIB,
208209
)
@@ -227,18 +228,25 @@ def test_given_shared_dependency_benchmark_when_planning_then_stays_near_compile
227228
expected_max_wall_seconds=test_case.expected_max_plan_wall_seconds,
228229
)
229230

230-
ratio: float = planned.elapsed_seconds / compiled.elapsed_seconds
231+
# CPU time is the tight guard because it is steady on shared runners; the looser wall-time
232+
# backstop still catches lost parallelism and lock or I/O waits.
233+
wall_ratio: float = planned.elapsed_seconds / compiled.elapsed_seconds
234+
cpu_ratio: float = planned.cpu_seconds / compiled.cpu_seconds
231235
_LOGGER.info(
232-
"plan %s wall=%.3fs cpu=%.3fs peak_rss_bytes=%d compile_wall=%.3fs ratio=%.2f",
236+
"plan %s wall=%.3fs cpu=%.3fs peak_rss_bytes=%d compile_wall=%.3fs compile_cpu=%.3fs "
237+
"ratio=%.2f cpu_ratio=%.2f",
233238
test_case.description,
234239
planned.elapsed_seconds,
235240
planned.cpu_seconds,
236241
planned.peak_rss_bytes,
237242
compiled.elapsed_seconds,
238-
ratio,
243+
compiled.cpu_seconds,
244+
wall_ratio,
245+
cpu_ratio,
239246
)
240247
assert len(cast(dict[str, list[object]], planned.payload)["models"]) == _MODEL_COUNT
241-
assert ratio < test_case.expected_max_compile_ratio
248+
assert cpu_ratio < test_case.expected_max_compile_cpu_ratio
249+
assert wall_ratio < test_case.expected_max_compile_wall_ratio
242250
assert planned.peak_rss_bytes < test_case.expected_max_rss_bytes
243251

244252

0 commit comments

Comments
 (0)