test: make the release-performance self-comparison E2E independent of runner noise

The E2E compares one build with itself using a single run, so asserting a
passed verdict depended on wall-time noise (a 1.31x wall ratio at 0.99x
CPU failed CI). It now asserts the report structure, every command row,
the per-side generator markers and the recorded evidence, and accepts
either verdict as long as the exit code matches it. The production 1.25x
limits are unchanged; the verdict unit tests gain fixed-timing cases for
a wall-only regression at CPU parity and a wall-only slowdown just under
the limit.

Fixes CHI-481

Author: kvlonge

tests/e2e/scripts/release_performance/_test_types.py

@@ -10,6 +10,6 @@ class ReleasePerformanceTestCase:
     description: str
     inspection_models: int
     build_models: int
-    expected_return_code: int
+    expected_outcomes: tuple[tuple[int, bool], ...]
     expected_fragments: tuple[str, ...]
     expected_marked_projects: tuple[str, ...]

tests/e2e/scripts/release_performance/test_compare_release_performance.py

@@ -24,16 +24,16 @@
     "test_case",
     (
         ReleasePerformanceTestCase(
-            description="the same build as candidate and baseline passes every command",
+            description="the same build as candidate and baseline reports every command",
             inspection_models=300,
             build_models=100,
-            expected_return_code=0,
+            expected_outcomes=((0, True), (1, False)),
             expected_fragments=(
                 "| `plan --json` |",
                 "| `build (empty warehouse)` |",
                 "| `lineage column trace` |",
                 "Benchmark projects are generated per side",
-                "**Result: passed.**",
+                "**Result: ",
             ),
             expected_marked_projects=(
                 "baseline/build",
@@ -96,10 +96,12 @@ def test_given_two_installations_when_comparing_release_performance_then_reports
         == test_case.expected_marked_projects
     )
     output: str = result.stdout + result.stderr
-    assert result.returncode == test_case.expected_return_code, output
     markdown: str = summary.read_text(encoding="utf-8")
     for fragment in test_case.expected_fragments:
         assert fragment in markdown
+    assert (result.returncode, "**Result: passed.**" in markdown) in (
+        test_case.expected_outcomes
+    ), output
     recorded: dict[str, object] = json.loads(evidence.read_text(encoding="utf-8"))
     assert [command["name"] for command in recorded["commands"]] == [
         command.name for command in BENCHMARK_COMMANDS

tests/unit/scripts/release_performance/_helpers/test_verdict.py

@@ -63,6 +63,24 @@
             ),
             expected_regressed=(False, True, False),
         ),
+        MetricVerdictTestCase(
+            description="a wall-only regression at CPU parity fails on wall alone",
+            comparison=comparison(
+                name="build (empty warehouse)",
+                baseline=((10.0, 6.4, 330),) * 3,
+                candidate=((13.14, 6.36, 330),) * 3,
+            ),
+            expected_regressed=(True, False, False),
+        ),
+        MetricVerdictTestCase(
+            description="a wall-only slowdown under the ratio limit at CPU parity passes",
+            comparison=comparison(
+                name="build (empty warehouse)",
+                baseline=((10.0, 6.4, 330),) * 3,
+                candidate=((12.4, 6.4, 330),) * 3,
+            ),
+            expected_regressed=(False, False, False),
+        ),
         MetricVerdictTestCase(
             description="one slow outlier does not move the median time",
             comparison=comparison(