ci: add an absolute floor to the same-runner compile ratio

scripts/compile_performance_ratio/_helpers/verdict.py

@@ -8,9 +8,13 @@
 
 
 def ratio_failures(
-    *, comparisons: tuple[CompileComparison, ...], modes: tuple[str, ...], max_ratio: float
+    *,
+    comparisons: tuple[CompileComparison, ...],
+    modes: tuple[str, ...],
+    max_ratio: float,
+    noise_floor_seconds: float,
 ) -> tuple[str, ...]:
-    """Describe every requested mode that is unmeasured or whose wall or CPU ratio is too high."""
+    """Describe every requested mode that is unmeasured or slower than ratio plus floor allow."""
 
     measured: dict[str, CompileComparison] = {
         comparison.mode: comparison for comparison in comparisons
@@ -35,7 +39,7 @@ def ratio_failures(
                 failures.append(
                     f"{prefix}: {metric} measurement missing (base {base:.2f} s, head {head:.2f} s)"
                 )
-            elif ratio > max_ratio:
+            elif head > base * max_ratio + noise_floor_seconds:
                 failures.append(f"{prefix}: {metric} ratio {ratio:.3f} exceeds {max_ratio:.2f}")
     return tuple(failures)
 

scripts/compile_performance_ratio/constants.py

@@ -4,6 +4,7 @@
 
 DEFAULT_RUNS: int = 3
 DEFAULT_MAX_RATIO: float = 1.10
+NOISE_FLOOR_SECONDS: float = 0.2
 DENSE_KIND: str = "dense"
 FRESH_KIND: str = "fresh"
 PROJECT_KINDS: tuple[str, ...] = (DENSE_KIND, FRESH_KIND)

scripts/compile_performance_ratio/main/compare.py

@@ -19,6 +19,7 @@
     COMPILE_MODES,
     DEFAULT_MAX_RATIO,
     DEFAULT_RUNS,
+    NOISE_FLOOR_SECONDS,
     PROJECT_KINDS,
 )
 from scripts.compile_performance_ratio.models import CompileComparison
@@ -59,7 +60,10 @@ def compare_compile_performance(argv: list[str] | None = None) -> int:
             modes=modes,
         )
     failures: tuple[str, ...] = ratio_failures(
-        comparisons=comparisons, modes=modes, max_ratio=args.max_ratio
+        comparisons=comparisons,
+        modes=modes,
+        max_ratio=args.max_ratio,
+        noise_floor_seconds=args.noise_floor_seconds,
     )
     markdown: str = comparison_markdown(
         comparisons=comparisons,
@@ -100,4 +104,10 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace:
     )
     parser.add_argument("--runs", type=int, default=DEFAULT_RUNS)
     parser.add_argument("--max-ratio", type=float, default=DEFAULT_MAX_RATIO)
+    parser.add_argument(
+        "--noise-floor-seconds",
+        type=float,
+        default=NOISE_FLOOR_SECONDS,
+        help="Seconds of slack added to the ratio allowance before a slowdown fails.",
+    )
     return parser.parse_args(argv)

tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py

@@ -81,6 +81,8 @@ def test_given_two_builds_when_comparing_compile_performance_then_enforces_ratio
             sys.executable,
             "--max-ratio",
             test_case.max_ratio,
+            "--noise-floor-seconds",
+            "0",
         ],
         cwd=_REPO_ROOT,
         capture_output=True,

tests/unit/scripts/compile_performance_ratio/_helpers/test_verdict.py

@@ -3,7 +3,7 @@
 import pytest
 
 from scripts.compile_performance_ratio._helpers.verdict import ratio_failures
-from scripts.compile_performance_ratio.constants import COMPILE_MODES
+from scripts.compile_performance_ratio.constants import COMPILE_MODES, NOISE_FLOOR_SECONDS
 from tests.unit.scripts.compile_performance_ratio._helpers._test_types import (
     RatioFailuresTestCase,
 )
@@ -52,6 +52,20 @@
             max_ratio=1.10,
             expected_failures=("dense 3000 edit: CPU ratio 1.200 exceeds 1.10",),
         ),
+        RatioFailuresTestCase(
+            description="a sub-second reused warm compile within the floor passes",
+            comparisons=(comparison(mode="warm", wall=(0.56, 0.65), cpu=(0.44, 0.52)),),
+            modes=("warm",),
+            max_ratio=1.10,
+            expected_failures=(),
+        ),
+        RatioFailuresTestCase(
+            description="a sub-second reused warm compile beyond the floor fails",
+            comparisons=(comparison(mode="warm", wall=(0.56, 0.85), cpu=(0.44, 0.44)),),
+            modes=("warm",),
+            max_ratio=1.10,
+            expected_failures=("dense 3000 warm: wall ratio 1.518 exceeds 1.10",),
+        ),
         RatioFailuresTestCase(
             description="a requested mode without a measurement fails",
             comparisons=(comparison(mode="cold", wall=(18.0, 18.0), cpu=(40.0, 40.0)),),
@@ -84,7 +98,10 @@ def test_given_mode_comparisons_when_judging_then_reports_each_failed_limit(
     test_case: RatioFailuresTestCase,
 ) -> None:
     failures: tuple[str, ...] = ratio_failures(
-        comparisons=test_case.comparisons, modes=test_case.modes, max_ratio=test_case.max_ratio
+        comparisons=test_case.comparisons,
+        modes=test_case.modes,
+        max_ratio=test_case.max_ratio,
+        noise_floor_seconds=NOISE_FLOOR_SECONDS,
     )
 
     assert failures == test_case.expected_failures