diff --git a/scripts/compile_performance_ratio/_helpers/verdict.py b/scripts/compile_performance_ratio/_helpers/verdict.py index 1caa0296b7..6a85f2f240 100644 --- a/scripts/compile_performance_ratio/_helpers/verdict.py +++ b/scripts/compile_performance_ratio/_helpers/verdict.py @@ -8,9 +8,13 @@ def ratio_failures( - *, comparisons: tuple[CompileComparison, ...], modes: tuple[str, ...], max_ratio: float + *, + comparisons: tuple[CompileComparison, ...], + modes: tuple[str, ...], + max_ratio: float, + noise_floor_seconds: float, ) -> tuple[str, ...]: - """Describe every requested mode that is unmeasured or whose wall or CPU ratio is too high.""" + """Describe every requested mode that is unmeasured or slower than ratio plus floor allow.""" measured: dict[str, CompileComparison] = { comparison.mode: comparison for comparison in comparisons @@ -35,7 +39,7 @@ def ratio_failures( failures.append( f"{prefix}: {metric} measurement missing (base {base:.2f} s, head {head:.2f} s)" ) - elif ratio > max_ratio: + elif head > base * max_ratio + noise_floor_seconds: failures.append(f"{prefix}: {metric} ratio {ratio:.3f} exceeds {max_ratio:.2f}") return tuple(failures) diff --git a/scripts/compile_performance_ratio/constants.py b/scripts/compile_performance_ratio/constants.py index 00b18fbd69..ca1301ec6a 100644 --- a/scripts/compile_performance_ratio/constants.py +++ b/scripts/compile_performance_ratio/constants.py @@ -4,6 +4,7 @@ DEFAULT_RUNS: int = 3 DEFAULT_MAX_RATIO: float = 1.10 +NOISE_FLOOR_SECONDS: float = 0.2 DENSE_KIND: str = "dense" FRESH_KIND: str = "fresh" PROJECT_KINDS: tuple[str, ...] = (DENSE_KIND, FRESH_KIND) diff --git a/scripts/compile_performance_ratio/main/compare.py b/scripts/compile_performance_ratio/main/compare.py index e8e29d1fed..237879c674 100644 --- a/scripts/compile_performance_ratio/main/compare.py +++ b/scripts/compile_performance_ratio/main/compare.py @@ -19,6 +19,7 @@ COMPILE_MODES, DEFAULT_MAX_RATIO, DEFAULT_RUNS, + NOISE_FLOOR_SECONDS, PROJECT_KINDS, ) from scripts.compile_performance_ratio.models import CompileComparison @@ -59,7 +60,10 @@ def compare_compile_performance(argv: list[str] | None = None) -> int: modes=modes, ) failures: tuple[str, ...] = ratio_failures( - comparisons=comparisons, modes=modes, max_ratio=args.max_ratio + comparisons=comparisons, + modes=modes, + max_ratio=args.max_ratio, + noise_floor_seconds=args.noise_floor_seconds, ) markdown: str = comparison_markdown( comparisons=comparisons, @@ -100,4 +104,10 @@ def _parse_args(argv: list[str] | None) -> argparse.Namespace: ) parser.add_argument("--runs", type=int, default=DEFAULT_RUNS) parser.add_argument("--max-ratio", type=float, default=DEFAULT_MAX_RATIO) + parser.add_argument( + "--noise-floor-seconds", + type=float, + default=NOISE_FLOOR_SECONDS, + help="Seconds of slack added to the ratio allowance before a slowdown fails.", + ) return parser.parse_args(argv) diff --git a/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py b/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py index 3d2d6708e6..fda715ddc8 100644 --- a/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py +++ b/tests/e2e/scripts/compile_performance_ratio/test_compare_compile_performance.py @@ -81,6 +81,8 @@ def test_given_two_builds_when_comparing_compile_performance_then_enforces_ratio sys.executable, "--max-ratio", test_case.max_ratio, + "--noise-floor-seconds", + "0", ], cwd=_REPO_ROOT, capture_output=True, diff --git a/tests/unit/scripts/compile_performance_ratio/_helpers/test_verdict.py b/tests/unit/scripts/compile_performance_ratio/_helpers/test_verdict.py index 189b2e651f..6da3ed0a93 100644 --- a/tests/unit/scripts/compile_performance_ratio/_helpers/test_verdict.py +++ b/tests/unit/scripts/compile_performance_ratio/_helpers/test_verdict.py @@ -3,7 +3,7 @@ import pytest from scripts.compile_performance_ratio._helpers.verdict import ratio_failures -from scripts.compile_performance_ratio.constants import COMPILE_MODES +from scripts.compile_performance_ratio.constants import COMPILE_MODES, NOISE_FLOOR_SECONDS from tests.unit.scripts.compile_performance_ratio._helpers._test_types import ( RatioFailuresTestCase, ) @@ -52,6 +52,20 @@ max_ratio=1.10, expected_failures=("dense 3000 edit: CPU ratio 1.200 exceeds 1.10",), ), + RatioFailuresTestCase( + description="a sub-second reused warm compile within the floor passes", + comparisons=(comparison(mode="warm", wall=(0.56, 0.65), cpu=(0.44, 0.52)),), + modes=("warm",), + max_ratio=1.10, + expected_failures=(), + ), + RatioFailuresTestCase( + description="a sub-second reused warm compile beyond the floor fails", + comparisons=(comparison(mode="warm", wall=(0.56, 0.85), cpu=(0.44, 0.44)),), + modes=("warm",), + max_ratio=1.10, + expected_failures=("dense 3000 warm: wall ratio 1.518 exceeds 1.10",), + ), RatioFailuresTestCase( description="a requested mode without a measurement fails", comparisons=(comparison(mode="cold", wall=(18.0, 18.0), cpu=(40.0, 40.0)),), @@ -84,7 +98,10 @@ def test_given_mode_comparisons_when_judging_then_reports_each_failed_limit( test_case: RatioFailuresTestCase, ) -> None: failures: tuple[str, ...] = ratio_failures( - comparisons=test_case.comparisons, modes=test_case.modes, max_ratio=test_case.max_ratio + comparisons=test_case.comparisons, + modes=test_case.modes, + max_ratio=test_case.max_ratio, + noise_floor_seconds=NOISE_FLOOR_SECONDS, ) assert failures == test_case.expected_failures