test(perf): scale the dense compile guards to the runner and judge warm and edit on medians

tests/e2e/src/sqlbuild/cli/commands/main/compile/helpers.py

@@ -18,10 +18,12 @@
 import zipfile
 from bisect import bisect_left
 from collections.abc import Callable, Iterator
+from concurrent.futures import ProcessPoolExecutor
 from contextlib import contextmanager, redirect_stdout
 from dataclasses import replace
 from functools import partial
 from io import StringIO
+from multiprocessing import get_context
 from pathlib import Path
 from types import FrameType, ModuleType
 from typing import Any, NamedTuple, cast
@@ -41,6 +43,7 @@
 from scripts.cold_compile_performance.main.semantic_compile_fingerprint import (
     semantic_compile_fingerprint,
 )
+from scripts.compile_performance_ratio._helpers.report import cpu_model
 from scripts.compiler_differential.constants import FAILURE_BASE_FILES
 from scripts.delta_compile_series.models import DeltaCompileOutcome
 from sqlbuild._native import COMPILE_REUSE_DISABLE_ENV_VAR as REUSE_DISABLE_ENV_VAR
@@ -916,49 +919,71 @@ def prepare_small_inspection_project(*, project_dir: Path, model_count: int) ->
     )
 
 
+# Each repeated edit writes a value the cache has not seen, so every edit recompiles the edited
+# model; the last writes the value the expected edit fingerprint and the oracle describe.
+_DENSE_EDIT_AMOUNTS: tuple[str, ...] = ("1001", "1002", "1000")
+_DENSE_WARM_RUNS: int = 3
+
+
+class DenseWarmEditMeasurements(NamedTuple):
+    """Cached cold, repeated warm and one-model edit compiles, and a from-scratch oracle."""
+
+    cold: FreshProcessCompileBenchmarkResult
+    warm: tuple[FreshProcessCompileBenchmarkResult, ...]
+    edit: tuple[FreshProcessCompileBenchmarkResult, ...]
+    oracle: FreshProcessCompileBenchmarkResult
+
+
 def run_dense_warm_edit_benchmark(
     *,
     project_dir: Path,
     edited_model_path: Path,
     expected_warm_max_seconds: float,
     expected_edit_max_seconds: float,
-) -> dict[str, FreshProcessCompileBenchmarkResult]:
-    """Measure cached cold, warm, and one-model edit plus a from-scratch oracle of the edit."""
+) -> DenseWarmEditMeasurements:
+    """Measure cached cold, three warm and three one-model edit compiles, then the edit's oracle."""
 
-    measurements: dict[str, FreshProcessCompileBenchmarkResult] = {
-        "cold": _run_fresh_process_compile_benchmark(
-            project_dir=project_dir,
-            label="dense-cold",
-            expected_max_wall_seconds=4 * expected_warm_max_seconds,
-            compile_args=(),
-        ),
-        "warm": _run_fresh_process_compile_benchmark(
+    cold: FreshProcessCompileBenchmarkResult = _run_fresh_process_compile_benchmark(
+        project_dir=project_dir,
+        label="dense-cold",
+        expected_max_wall_seconds=4 * expected_warm_max_seconds,
+        compile_args=(),
+    )
+    warm: tuple[FreshProcessCompileBenchmarkResult, ...] = tuple(
+        _run_fresh_process_compile_benchmark(
             project_dir=project_dir,
-            label="dense-warm",
+            label=f"dense-warm-{run}",
             expected_max_wall_seconds=expected_warm_max_seconds,
             compile_args=(),
-        ),
-    }
-    _replace_benchmark_text(
-        path=edited_model_path,
-        old="COALESCE(b.amount, 0) + CAST(b.id AS DOUBLE)",
-        new="COALESCE(b.amount, 1000) + CAST(b.id AS DOUBLE)",
-    )
-    measurements["edit"] = _run_fresh_process_compile_benchmark(
-        project_dir=project_dir,
-        label="dense-edit",
-        expected_max_wall_seconds=expected_edit_max_seconds,
-        compile_args=(),
+        )
+        for run in range(_DENSE_WARM_RUNS)
     )
+    edits: list[FreshProcessCompileBenchmarkResult] = []
+    previous: str = "0"
+    for amount in _DENSE_EDIT_AMOUNTS:
+        _replace_benchmark_text(
+            path=edited_model_path,
+            old=f"COALESCE(b.amount, {previous}) + CAST(b.id AS DOUBLE)",
+            new=f"COALESCE(b.amount, {amount}) + CAST(b.id AS DOUBLE)",
+        )
+        edits.append(
+            _run_fresh_process_compile_benchmark(
+                project_dir=project_dir,
+                label=f"dense-edit-{len(edits)}",
+                expected_max_wall_seconds=expected_edit_max_seconds,
+                compile_args=(),
+            )
+        )
+        previous = amount
     oracle_dir: Path = project_dir.with_name(f"{project_dir.name}_oracle")
     _ = shutil.copytree(project_dir, oracle_dir, ignore=shutil.ignore_patterns("target"))
-    measurements["oracle"] = _run_fresh_process_compile_benchmark(
+    oracle: FreshProcessCompileBenchmarkResult = _run_fresh_process_compile_benchmark(
         project_dir=oracle_dir,
         label="dense-oracle",
         expected_max_wall_seconds=4 * expected_edit_max_seconds,
         compile_args=("--no-cache",),
     )
-    return measurements
+    return DenseWarmEditMeasurements(cold=cold, warm=warm, edit=tuple(edits), oracle=oracle)
 
 
 class DeltaEditLoopMeasurements(NamedTuple):
@@ -1091,6 +1116,79 @@ def median_cpu_seconds(results: tuple[FreshProcessCompileBenchmarkResult, ...])
     return statistics.median(result.cpu_seconds for result in results)
 
 
+def median_wall_seconds(results: tuple[FreshProcessCompileBenchmarkResult, ...]) -> float:
+    """The median wall seconds of same-runner compiles."""
+
+    return statistics.median(result.elapsed_seconds for result in results)
+
+
+# The slowest fastest-repetition seen on the hosted runner classes the dense budgets fit as they
+# are. A measurably slower runner scales the budgets by its own fastest repetition over this; a
+# faster one keeps them unchanged.
+_RUNNER_CALIBRATION_REFERENCE_SECONDS: float = 0.55
+_RUNNER_CALIBRATION_CHECKSUM: int = 2910182
+_RUNNER_CALIBRATION_REPETITIONS: int = 7
+_RUNNER_CALIBRATION_VALUES: int = 600_000
+
+
+class RunnerCalibration(NamedTuple):
+    """How much slower this runner is than the budgets' reference runner, from fixed work."""
+
+    cpu_model: str
+    seconds: tuple[float, ...]
+    factor: float
+
+
+def _runner_calibration_workload() -> int:
+    """Generate, count, sort and format pseudo-random integers; return a checksum of the work."""
+
+    state: int = 1
+    values: list[int] = []
+    for _ in range(_RUNNER_CALIBRATION_VALUES):
+        state = (state * 6364136223846793005 + 1442695040888963407) % 2**64
+        values.append(state >> 33)
+    counts: dict[int, int] = {}
+    for value in values:
+        counts[value & 0xFFFF] = counts.get(value & 0xFFFF, 0) + 1
+    values.sort()
+    rows: list[tuple[int, str]] = [(value, str(value)) for value in values[::2]]
+    return sum(len(text) for _, text in rows) + len(counts)
+
+
+def _time_runner_calibration() -> tuple[tuple[float, ...], int]:
+    """Each repetition's seconds and the workload checksum, which pins the work done."""
+
+    seconds: list[float] = []
+    checksum: int = 0
+    for _ in range(_RUNNER_CALIBRATION_REPETITIONS):
+        started: float = time.perf_counter()
+        checksum = _runner_calibration_workload()
+        seconds.append(time.perf_counter() - started)
+    return tuple(seconds), checksum
+
+
+def measure_runner_calibration() -> RunnerCalibration:
+    """Time the fixed calibration workload in a fresh interpreter and derive the budget factor."""
+
+    # A fresh interpreter keeps the test process's heap and collector out of the timing.
+    with ProcessPoolExecutor(max_workers=1, mp_context=get_context("spawn")) as pool:
+        seconds, checksum = pool.submit(_time_runner_calibration).result(timeout=120)
+    assert checksum == _RUNNER_CALIBRATION_CHECKSUM
+    # The fastest repetition measures the machine; a busy moment only ever adds time to it.
+    calibration: RunnerCalibration = RunnerCalibration(
+        cpu_model=cpu_model(),
+        seconds=seconds,
+        factor=max(1.0, min(seconds) / _RUNNER_CALIBRATION_REFERENCE_SECONDS),
+    )
+    print(
+        f"runner calibration cpu={calibration.cpu_model!r} "
+        f"runs={[round(run, 3) for run in seconds]} fastest={min(seconds):.3f}s "
+        f"reference={_RUNNER_CALIBRATION_REFERENCE_SECONDS:.3f}s factor={calibration.factor:.3f}",
+        flush=True,
+    )
+    return calibration
+
+
 def _append_benchmark_edit(path: Path, label: str) -> None:
     path.write_text(
         path.read_text(encoding="utf-8") + f"\n-- one {label} edit\n",

tests/e2e/src/sqlbuild/cli/commands/main/compile/test_dense_compile_performance.py

@@ -29,12 +29,15 @@
 )
 from tests.e2e.src.sqlbuild.cli.commands.main.compile.helpers import (
     DeltaEditLoopMeasurements,
+    DenseWarmEditMeasurements,
     FreshProcessCompileBenchmarkResult,
     _run_fresh_process_compile_benchmark,
     analysed_model_count,
     fresh_process_compile_cache_metrics,
     measure_model_sql_bytes,
+    measure_runner_calibration,
     median_cpu_seconds,
+    median_wall_seconds,
     run_dense_delta_edit_loop_benchmark,
     run_dense_warm_edit_benchmark,
 )
@@ -86,6 +89,9 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_
     tmp_path: Path,
 ) -> None:
     require_compile_memory_scope()
+    # Fixed work that does not depend on SQLBuild scales the budget on a slower runner class.
+    wall_budget: float = test_case.expected_max_wall_seconds * measure_runner_calibration().factor
+    print(f"dense cold models={test_case.model_count} wall_budget={wall_budget:.2f}s", flush=True)
     project_dir: Path = tmp_path / "dense_orders"
     write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count)
     config: RulesConfig = load_rules_config(project_dir=project_dir)
@@ -107,7 +113,7 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_
     result: FreshProcessCompileBenchmarkResult = _run_fresh_process_compile_benchmark(
         project_dir=project_dir,
         label=f"dense-{test_case.model_count}",
-        expected_max_wall_seconds=test_case.expected_max_wall_seconds,
+        expected_max_wall_seconds=wall_budget,
         compile_args=("--no-cache",),
     )
     print(
@@ -141,7 +147,7 @@ def test_given_dense_project_when_compiling_cold_then_preserves_rules_semantics_
     assert timings["custom_rules_ms"] > 0
     assert timings["custom_rules_cpu_ms"] > 0
     assert result.semantic_fingerprint == test_case.expected_fingerprint
-    assert result.elapsed_seconds < test_case.expected_max_wall_seconds
+    assert result.elapsed_seconds < wall_budget
     assert result.peak_rss_bytes < test_case.expected_max_rss_bytes
 
 
@@ -180,18 +186,34 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches
     tmp_path: Path,
 ) -> None:
     require_compile_memory_scope()
+    # Fixed work that does not depend on SQLBuild scales the budgets on a slower runner class;
+    # the same-runner ratio jobs remain what catches a slower compiler.
+    factor: float = measure_runner_calibration().factor
+    warm_budget: float = test_case.expected_warm_max_seconds * factor
+    edit_budget: float = test_case.expected_edit_max_seconds * factor
+    print(
+        f"dense models={test_case.model_count} warm_budget={warm_budget:.2f}s "
+        f"edit_budget={edit_budget:.2f}s",
+        flush=True,
+    )
     project_dir: Path = tmp_path / "dense_orders"
     write_dense_compile_project(project_dir=project_dir, model_count=test_case.model_count)
     edited_model_path: Path = dense_model_path(
         project_dir=project_dir, index=test_case.edited_model_index
     )
-    measurements: dict[str, FreshProcessCompileBenchmarkResult] = run_dense_warm_edit_benchmark(
+    measurements: DenseWarmEditMeasurements = run_dense_warm_edit_benchmark(
         project_dir=project_dir,
         edited_model_path=edited_model_path,
-        expected_warm_max_seconds=test_case.expected_warm_max_seconds,
-        expected_edit_max_seconds=test_case.expected_edit_max_seconds,
+        expected_warm_max_seconds=warm_budget,
+        expected_edit_max_seconds=edit_budget,
+    )
+    labelled: tuple[tuple[str, FreshProcessCompileBenchmarkResult], ...] = (
+        ("cold", measurements.cold),
+        *(("warm", result) for result in measurements.warm),
+        *(("edit", result) for result in measurements.edit),
+        ("oracle", measurements.oracle),
     )
-    for label, result in measurements.items():
+    for label, result in labelled:
         print(
             f"dense {label} models={test_case.model_count} wall={result.elapsed_seconds:.2f}s "
             f"cpu={result.cpu_seconds:.2f}s fingerprint={result.semantic_fingerprint}",
@@ -200,30 +222,37 @@ def test_given_dense_project_when_compiling_warm_and_after_one_edit_then_matches
         assert result.payload["diagnostics"] == []
         assert result.payload["has_errors"] is False
         assert result.peak_rss_bytes < test_case.expected_max_rss_bytes
-    cold: FreshProcessCompileBenchmarkResult = measurements["cold"]
-    warm: FreshProcessCompileBenchmarkResult = measurements["warm"]
-    edit: FreshProcessCompileBenchmarkResult = measurements["edit"]
-    assert cold.semantic_fingerprint == test_case.expected_cold_fingerprint
-    assert warm.semantic_fingerprint == test_case.expected_cold_fingerprint
-    assert edit.semantic_fingerprint == test_case.expected_edit_fingerprint
-    assert measurements["oracle"].semantic_fingerprint == test_case.expected_edit_fingerprint
-    warm_timings: dict[str, int] = cast(dict[str, int], warm.payload["compile_timings"])
-    edit_timings: dict[str, int] = cast(dict[str, int], edit.payload["compile_timings"])
-    batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics(warm)
-    assert (batch_hits + entry_hits, misses, bypasses) == (test_case.model_count, 0, 0)
-    assert warm_timings["rule_cache_misses"] == 0
-    batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics(edit)
-    assert 0 < misses < test_case.model_count
-    assert batch_hits + entry_hits + misses == test_case.model_count
-    # The edited model misses once in built-in model rules and once in SQL lint; custom rules
-    # replay their recorded facts, and every other model's built-in findings stay cached.
-    assert edit_timings["rule_cache_misses"] == test_case.expected_edit_rule_cache_misses
+    assert measurements.cold.semantic_fingerprint == test_case.expected_cold_fingerprint
+    for warm in measurements.warm:
+        assert warm.semantic_fingerprint == test_case.expected_cold_fingerprint
+        warm_timings: dict[str, int] = cast(dict[str, int], warm.payload["compile_timings"])
+        batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics(warm)
+        assert (batch_hits + entry_hits, misses, bypasses) == (test_case.model_count, 0, 0)
+        assert warm_timings["rule_cache_misses"] == 0
+    for edit in measurements.edit:
+        edit_timings: dict[str, int] = cast(dict[str, int], edit.payload["compile_timings"])
+        batch_hits, entry_hits, misses, bypasses = fresh_process_compile_cache_metrics(edit)
+        assert 0 < misses < test_case.model_count
+        assert batch_hits + entry_hits + misses == test_case.model_count
+        # The edited model misses once in built-in model rules and once in SQL lint; custom
+        # rules replay their recorded facts, and every other model's built-in findings stay
+        # cached.
+        assert edit_timings["rule_cache_misses"] == test_case.expected_edit_rule_cache_misses
+    # Earlier edits write other values, so their output differs from both the cold and final edit.
+    for edit in measurements.edit[:-1]:
+        assert edit.semantic_fingerprint not in {
+            test_case.expected_cold_fingerprint,
+            test_case.expected_edit_fingerprint,
+        }
+    assert measurements.edit[-1].semantic_fingerprint == test_case.expected_edit_fingerprint
+    assert measurements.oracle.semantic_fingerprint == test_case.expected_edit_fingerprint
     oracle_timings: dict[str, int] = cast(
-        dict[str, int], measurements["oracle"].payload["compile_timings"]
+        dict[str, int], measurements.oracle.payload["compile_timings"]
     )
     assert oracle_timings["rule_cache_hits"] == 0
-    assert warm.elapsed_seconds < test_case.expected_warm_max_seconds
-    assert edit.elapsed_seconds < test_case.expected_edit_max_seconds
+    # Medians of same-job runs keep one slow runner moment from failing an unchanged compiler.
+    assert median_wall_seconds(measurements.warm) < warm_budget
+    assert median_wall_seconds(measurements.edit) < edit_budget
 
 
 if __name__ == "__main__":