test(compile): accept a delta compile's fresh analysis in the same-runner edit ratio

A delta compile analyses only the models it reruns, without the analysis
cache, so the ratio script's cache precondition rejected head's one-model
edit as an uncached compile. Each run now records how many models its
analysis counters account for. An edit run that analysed fewer models than
the project has is a delta, and passes when it analysed at least one model
afresh. Full compiles keep the existing checks, so a bypassing full compile
is still rejected.

Author: kvlonge

scripts/compile_performance_ratio/_helpers/measure.py

@@ -17,6 +17,7 @@
 from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project
 from scripts.compile_performance_ratio._helpers.edit import apply_one_model_edit
 from scripts.compile_performance_ratio.constants import (
+    ANALYSED_MODEL_COUNTERS,
     ANALYSIS_CACHE_BYPASSES,
     ANALYSIS_CACHE_MISSES,
     BASE_GENERATOR_ENTRY,
@@ -166,7 +167,7 @@ def compare_builds(
                     mode=mode,
                     compile_args=compile_args,
                 )
-                check_cache_use(run=run, mode=mode)
+                check_cache_use(run=run, mode=mode, models=models)
                 results[label].append(run)
         if mode != COLD_MODE:
             _, head_python, head_dir = builds[1]
@@ -213,13 +214,21 @@ def _comparison(
     )
 
 
-def check_cache_use(*, run: CompileRun, mode: str) -> None:
-    """Fail a bypassing run, a warm run with misses, or an edit run that analysed nothing."""
+def check_cache_use(*, run: CompileRun, mode: str, models: int) -> None:
+    """Fail a bypassing full compile, a warm run with misses, or an edit that analysed nothing."""
 
     misses: int | None = run.analysis_cache_misses
     if misses is None or mode == COLD_MODE:
         return
     bypasses: int = run.analysis_cache_bypasses or 0
+    delta: bool = run.analysed_models is not None and run.analysed_models < models
+    if mode == EDIT_MODE and delta:
+        if misses + bypasses == 0:
+            raise CompileComparisonError(
+                f"{run.label} one-model edit delta compile analysed no model afresh, so the "
+                "edit did not invalidate the edited model"
+            )
+        return
     if bypasses != 0:
         raise CompileComparisonError(
             f"{run.label} {mode} compile bypassed the analysis cache for {bypasses} models, so "
@@ -328,6 +337,11 @@ def _compile_once(
         timings_ms={name: value for name, value in numeric.items() if name in REPORTED_PHASES},
         analysis_cache_misses=numeric.get(ANALYSIS_CACHE_MISSES),
         analysis_cache_bypasses=numeric.get(ANALYSIS_CACHE_BYPASSES),
+        analysed_models=(
+            sum(numeric[name] for name in ANALYSED_MODEL_COUNTERS)
+            if all(name in numeric for name in ANALYSED_MODEL_COUNTERS)
+            else None
+        ),
         report=json.dumps(payload, sort_keys=True),
     )
 

scripts/compile_performance_ratio/constants.py

@@ -20,6 +20,12 @@
 }
 ANALYSIS_CACHE_MISSES: str = "analysis_cache_misses"
 ANALYSIS_CACHE_BYPASSES: str = "analysis_cache_bypasses"
+ANALYSED_MODEL_COUNTERS: tuple[str, ...] = (
+    "analysis_batch_cache_hits",
+    "analysis_entry_cache_hits",
+    ANALYSIS_CACHE_MISSES,
+    ANALYSIS_CACHE_BYPASSES,
+)
 MODELS_DIRECTORY: str = "models"
 MODEL_FILE_PATTERN: str = "*.sql"
 EDIT_COMMENT: str = "-- Benchmark edit {revision}.\n"

scripts/compile_performance_ratio/models.py

@@ -13,6 +13,8 @@ class CompileRun:
     timings_ms: dict[str, int]
     analysis_cache_misses: int | None = None
     analysis_cache_bypasses: int | None = None
+    analysed_models: int | None = None
+    """How many models the compile analysed; a delta compile analyses only the models it reran."""
     report: str = ""
     """The JSON report without its timings, which an uncached compile must reproduce."""
 

tests/unit/scripts/compile_performance_ratio/_helpers/_test_types.py

@@ -51,6 +51,7 @@ class CacheUseTestCase:
     description: str
     run: CompileRun
     mode: str
+    models: int = 3000
     expected_result: None = None
 
 
@@ -60,6 +61,7 @@ class CacheUseErrorTestCase:
     run: CompileRun
     mode: str
     expected_message: str
+    models: int = 3000
 
 
 @dataclass(frozen=True)

tests/unit/scripts/compile_performance_ratio/_helpers/test_measure.py

@@ -78,6 +78,20 @@ def test_given_compile_runs_when_taking_phase_medians_then_reports_only_complete
                 timings_ms={},
                 analysis_cache_misses=1,
                 analysis_cache_bypasses=0,
+                analysed_models=3000,
+            ),
+            mode=EDIT_MODE,
+        ),
+        CacheUseTestCase(
+            description="a delta edit analyses only the edited model, afresh",
+            run=CompileRun(
+                label="head",
+                wall_seconds=1.0,
+                cpu_seconds=1.0,
+                timings_ms={},
+                analysis_cache_misses=0,
+                analysis_cache_bypasses=1,
+                analysed_models=1,
             ),
             mode=EDIT_MODE,
         ),
@@ -110,7 +124,10 @@ def test_given_compile_runs_when_taking_phase_medians_then_reports_only_complete
 def test_given_observed_cache_use_when_checking_then_accepts_the_run(
     test_case: CacheUseTestCase,
 ) -> None:
-    assert check_cache_use(run=test_case.run, mode=test_case.mode) is test_case.expected_result
+    assert (
+        check_cache_use(run=test_case.run, mode=test_case.mode, models=test_case.models)
+        is test_case.expected_result
+    )
 
 
 @pytest.mark.parametrize(
@@ -138,10 +155,39 @@ def test_given_observed_cache_use_when_checking_then_accepts_the_run(
                 timings_ms={},
                 analysis_cache_misses=0,
                 analysis_cache_bypasses=3000,
+                analysed_models=3000,
             ),
             mode=EDIT_MODE,
             expected_message="bypassed the analysis cache for 3000 models",
         ),
+        CacheUseErrorTestCase(
+            description="a delta edit that analysed nothing afresh did not observe the edit",
+            run=CompileRun(
+                label="head",
+                wall_seconds=1.0,
+                cpu_seconds=1.0,
+                timings_ms={},
+                analysis_cache_misses=0,
+                analysis_cache_bypasses=0,
+                analysed_models=1,
+            ),
+            mode=EDIT_MODE,
+            expected_message="delta compile analysed no model afresh",
+        ),
+        CacheUseErrorTestCase(
+            description="a warm compile that bypassed the cache, even for few models, is rejected",
+            run=CompileRun(
+                label="head",
+                wall_seconds=1.0,
+                cpu_seconds=1.0,
+                timings_ms={},
+                analysis_cache_misses=0,
+                analysis_cache_bypasses=1,
+                analysed_models=1,
+            ),
+            mode=WARM_MODE,
+            expected_message="bypassed the analysis cache for 1 models",
+        ),
         CacheUseErrorTestCase(
             description="a warm compile that missed the cache is not an unchanged compile",
             run=CompileRun(
@@ -162,7 +208,7 @@ def test_given_unobserved_cache_use_when_checking_then_rejects_the_run(
     test_case: CacheUseErrorTestCase,
 ) -> None:
     with pytest.raises(CompileComparisonError, match=test_case.expected_message):
-        check_cache_use(run=test_case.run, mode=test_case.mode)
+        check_cache_use(run=test_case.run, mode=test_case.mode, models=test_case.models)
 
 
 @pytest.mark.parametrize(