ci: give the release comparison its own one-model edit limit (#1040)

One-model edit compiles may run up to 1.25x the previous release until
native rendering is the default; cold and warm compiles keep 1.15x.

Author: kvlonge

.github/workflows/release-performance.yml

@@ -63,6 +63,8 @@ jobs:
         run: uv sync --locked --all-groups --all-extras
       - name: Build candidate wheel
         run: uv build --wheel --out-dir "${RUNNER_TEMP}/candidate"
+      # Temporary: one-model edit compiles use MAX_EDIT_COMPILE_TIME_RATIO (1.25x) because an edited
+      # model recompiles uncached; tighten it back to 1.15x once native rendering is the default.
       - name: Compare candidate and previous release on this runner
         env:
           BASELINE: ${{ inputs.baseline }}

scripts/release_performance/constants.py

@@ -12,6 +12,7 @@
 MAX_CPU_RATIO: float = 1.25
 MAX_RSS_RATIO: float = 1.25
 MAX_COMPILE_TIME_RATIO: float = 1.15
+MAX_EDIT_COMPILE_TIME_RATIO: float = 1.25
 MIN_WALL_REGRESSION_SECONDS: float = 0.5
 MIN_CPU_REGRESSION_SECONDS: float = 0.5
 MIN_RSS_REGRESSION_BYTES: int = 32 * 1024 * 1024
@@ -118,7 +119,7 @@
         project=INSPECTION_PROJECT,
         sqb_args=("compile", "--json"),
         edits_model=True,
-        max_time_ratio=MAX_COMPILE_TIME_RATIO,
+        max_time_ratio=MAX_EDIT_COMPILE_TIME_RATIO,
     ),
     BenchmarkCommand(
         name="dense compile (warm cache)",
@@ -131,7 +132,7 @@
         project=DENSE_PROJECT,
         sqb_args=("compile", "--json"),
         edits_model=True,
-        max_time_ratio=MAX_COMPILE_TIME_RATIO,
+        max_time_ratio=MAX_EDIT_COMPILE_TIME_RATIO,
     ),
     BenchmarkCommand(
         name="dense compile (no cache)",

tests/unit/scripts/release_performance/_helpers/_test_types.py

@@ -12,6 +12,14 @@ class MetricVerdictTestCase:
     expected_regressed: tuple[bool, bool, bool]
 
 
+@dataclass(frozen=True)
+class ConfiguredTimeLimitTestCase:
+    description: str
+    command: str
+    wall_ratio: float
+    expected_regressed: bool
+
+
 @dataclass(frozen=True)
 class SkipReasonTestCase:
     description: str

tests/unit/scripts/release_performance/_helpers/test_verdict.py

@@ -7,8 +7,10 @@
     regression_messages,
     skip_reason,
 )
+from scripts.release_performance.constants import BENCHMARK_COMMANDS
 from scripts.release_performance.models import BenchmarkCommand, MetricVerdict
 from tests.unit.scripts.release_performance._helpers._test_types import (
+    ConfiguredTimeLimitTestCase,
     MetricVerdictTestCase,
     SkipReasonTestCase,
 )
@@ -221,5 +223,64 @@ def test_given_command_minimum_version_when_checking_versions_then_explains_skip
     assert reason == test_case.expected_reason
 
 
+@pytest.mark.parametrize(
+    "test_case",
+    (
+        ConfiguredTimeLimitTestCase(
+            description="a dense one-model edit within the edit limit passes",
+            command="dense compile (one-model edit)",
+            wall_ratio=1.232,
+            expected_regressed=False,
+        ),
+        ConfiguredTimeLimitTestCase(
+            description="a one-model edit within the edit limit passes",
+            command="compile (one-model edit)",
+            wall_ratio=1.24,
+            expected_regressed=False,
+        ),
+        ConfiguredTimeLimitTestCase(
+            description="a dense one-model edit beyond the edit limit fails",
+            command="dense compile (one-model edit)",
+            wall_ratio=1.26,
+            expected_regressed=True,
+        ),
+        ConfiguredTimeLimitTestCase(
+            description="an unchanged warm compile keeps the tighter compile limit",
+            command="dense compile (warm cache)",
+            wall_ratio=1.232,
+            expected_regressed=True,
+        ),
+        ConfiguredTimeLimitTestCase(
+            description="a cold compile keeps the tighter compile limit",
+            command="dense compile (no cache)",
+            wall_ratio=1.16,
+            expected_regressed=True,
+        ),
+    ),
+    ids=lambda case: case.description,
+)
+def test_given_configured_command_when_judging_time_then_its_own_limit_applies(
+    test_case: ConfiguredTimeLimitTestCase,
+) -> None:
+    configured: BenchmarkCommand = {command.name: command for command in BENCHMARK_COMMANDS}[
+        test_case.command
+    ]
+    wall: float = 10.0 * test_case.wall_ratio
+
+    verdicts: tuple[MetricVerdict, ...] = metric_verdicts(
+        comparison=comparison(
+            name=configured.name,
+            baseline=((10.0, 10.0, 400),) * 3,
+            candidate=((wall, wall, 400),) * 3,
+            max_time_ratio=configured.max_time_ratio,
+        )
+    )
+
+    assert (verdicts[0].regressed, verdicts[1].regressed) == (
+        test_case.expected_regressed,
+        test_case.expected_regressed,
+    )
+
+
 if __name__ == "__main__":
     pytest.main([__file__, "-vv"])