diff --git a/.github/workflows/release-performance.yml b/.github/workflows/release-performance.yml index a565a03f62..1b86ea65f2 100644 --- a/.github/workflows/release-performance.yml +++ b/.github/workflows/release-performance.yml @@ -63,6 +63,8 @@ jobs: run: uv sync --locked --all-groups --all-extras - name: Build candidate wheel run: uv build --wheel --out-dir "${RUNNER_TEMP}/candidate" + # Temporary: one-model edit compiles use MAX_EDIT_COMPILE_TIME_RATIO (1.25x) because an edited + # model recompiles uncached; tighten it back to 1.15x once native rendering is the default. - name: Compare candidate and previous release on this runner env: BASELINE: ${{ inputs.baseline }} diff --git a/scripts/release_performance/constants.py b/scripts/release_performance/constants.py index 3f766fdcce..c3c7497e6f 100644 --- a/scripts/release_performance/constants.py +++ b/scripts/release_performance/constants.py @@ -12,6 +12,7 @@ MAX_CPU_RATIO: float = 1.25 MAX_RSS_RATIO: float = 1.25 MAX_COMPILE_TIME_RATIO: float = 1.15 +MAX_EDIT_COMPILE_TIME_RATIO: float = 1.25 MIN_WALL_REGRESSION_SECONDS: float = 0.5 MIN_CPU_REGRESSION_SECONDS: float = 0.5 MIN_RSS_REGRESSION_BYTES: int = 32 * 1024 * 1024 @@ -118,7 +119,7 @@ project=INSPECTION_PROJECT, sqb_args=("compile", "--json"), edits_model=True, - max_time_ratio=MAX_COMPILE_TIME_RATIO, + max_time_ratio=MAX_EDIT_COMPILE_TIME_RATIO, ), BenchmarkCommand( name="dense compile (warm cache)", @@ -131,7 +132,7 @@ project=DENSE_PROJECT, sqb_args=("compile", "--json"), edits_model=True, - max_time_ratio=MAX_COMPILE_TIME_RATIO, + max_time_ratio=MAX_EDIT_COMPILE_TIME_RATIO, ), BenchmarkCommand( name="dense compile (no cache)", diff --git a/tests/unit/scripts/release_performance/_helpers/_test_types.py b/tests/unit/scripts/release_performance/_helpers/_test_types.py index bc602ccbea..184c3eb6d2 100644 --- a/tests/unit/scripts/release_performance/_helpers/_test_types.py +++ b/tests/unit/scripts/release_performance/_helpers/_test_types.py @@ -12,6 +12,14 @@ class MetricVerdictTestCase: expected_regressed: tuple[bool, bool, bool] +@dataclass(frozen=True) +class ConfiguredTimeLimitTestCase: + description: str + command: str + wall_ratio: float + expected_regressed: bool + + @dataclass(frozen=True) class SkipReasonTestCase: description: str diff --git a/tests/unit/scripts/release_performance/_helpers/test_verdict.py b/tests/unit/scripts/release_performance/_helpers/test_verdict.py index b5b481a904..f8a6865933 100644 --- a/tests/unit/scripts/release_performance/_helpers/test_verdict.py +++ b/tests/unit/scripts/release_performance/_helpers/test_verdict.py @@ -7,8 +7,10 @@ regression_messages, skip_reason, ) +from scripts.release_performance.constants import BENCHMARK_COMMANDS from scripts.release_performance.models import BenchmarkCommand, MetricVerdict from tests.unit.scripts.release_performance._helpers._test_types import ( + ConfiguredTimeLimitTestCase, MetricVerdictTestCase, SkipReasonTestCase, ) @@ -221,5 +223,64 @@ def test_given_command_minimum_version_when_checking_versions_then_explains_skip assert reason == test_case.expected_reason +@pytest.mark.parametrize( + "test_case", + ( + ConfiguredTimeLimitTestCase( + description="a dense one-model edit within the edit limit passes", + command="dense compile (one-model edit)", + wall_ratio=1.232, + expected_regressed=False, + ), + ConfiguredTimeLimitTestCase( + description="a one-model edit within the edit limit passes", + command="compile (one-model edit)", + wall_ratio=1.24, + expected_regressed=False, + ), + ConfiguredTimeLimitTestCase( + description="a dense one-model edit beyond the edit limit fails", + command="dense compile (one-model edit)", + wall_ratio=1.26, + expected_regressed=True, + ), + ConfiguredTimeLimitTestCase( + description="an unchanged warm compile keeps the tighter compile limit", + command="dense compile (warm cache)", + wall_ratio=1.232, + expected_regressed=True, + ), + ConfiguredTimeLimitTestCase( + description="a cold compile keeps the tighter compile limit", + command="dense compile (no cache)", + wall_ratio=1.16, + expected_regressed=True, + ), + ), + ids=lambda case: case.description, +) +def test_given_configured_command_when_judging_time_then_its_own_limit_applies( + test_case: ConfiguredTimeLimitTestCase, +) -> None: + configured: BenchmarkCommand = {command.name: command for command in BENCHMARK_COMMANDS}[ + test_case.command + ] + wall: float = 10.0 * test_case.wall_ratio + + verdicts: tuple[MetricVerdict, ...] = metric_verdicts( + comparison=comparison( + name=configured.name, + baseline=((10.0, 10.0, 400),) * 3, + candidate=((wall, wall, 400),) * 3, + max_time_ratio=configured.max_time_ratio, + ) + ) + + assert (verdicts[0].regressed, verdicts[1].regressed) == ( + test_case.expected_regressed, + test_case.expected_regressed, + ) + + if __name__ == "__main__": pytest.main([__file__, "-vv"])