ci: generate baseline benchmark projects with the baseline release generator (#921)

* ci: generate release benchmark projects with each version's own generator

The release performance comparison generated its benchmark projects with
the candidate's generator and compiled them with the previous release, so
any generator syntax newer than that release (such as FUNCTION
descriptions) failed the baseline before timing. The baseline now runs
projects written by its own generator: the release tag's source is
extracted with git archive (or taken from --baseline-source) and its
write_pristine_projects runs with this interpreter, with cwd and
PYTHONPATH set to that source, through the checkout generator runner
shared with the same-runner compile ratio job. The candidate keeps its
own generator. Generation failures stop the comparison with the
generator's stderr tail, and the report and evidence record the
baseline's generator.

* test: make the release-performance self-comparison E2E independent of runner noise

The E2E compares one build with itself using a single run, so asserting a
passed verdict depended on wall-time noise (a 1.31x wall ratio at 0.99x
CPU failed CI). It now asserts the report structure, every command row,
the per-side generator markers and the recorded evidence, and accepts
either verdict as long as the exit code matches it. The production 1.25x
limits are unchanged; the verdict unit tests gain fixed-timing cases for
a wall-only regression at CPU parity and a wall-only slowdown just under
the limit.

Fixes CHI-481

Author: kvlonge

scripts/compile_performance_ratio/_helpers/measure.py

@@ -55,18 +55,37 @@ def write_benchmark_project(*, kind: str, project_dir: Path, models: int) -> Non
     )
 
 
+def run_checkout_generator(
+    *,
+    checkout: Path,
+    python: Path,
+    entry: str,
+    arguments: tuple[str, ...],
+    environment: dict[str, str],
+) -> subprocess.CompletedProcess[str]:
+    """Run generator code from another checkout's source tree with this `python`."""
+
+    return subprocess.run(
+        [str(python), "-c", entry, *arguments],
+        cwd=checkout,
+        capture_output=True,
+        text=True,
+        env={**environment, PYTHONPATH_KEY: str(checkout)},
+        check=False,
+    )
+
+
 def write_base_benchmark_project(
     *, base_root: Path, python: Path, kind: str, project_dir: Path, models: int
 ) -> None:
     """Write the base project with the base checkout's own generator, run by `python`."""
 
-    completed: subprocess.CompletedProcess[str] = subprocess.run(
-        [str(python), "-c", BASE_GENERATOR_ENTRY, kind, str(project_dir), str(models)],
-        cwd=base_root,
-        capture_output=True,
-        text=True,
-        env={**_compile_environment(), PYTHONPATH_KEY: str(base_root)},
-        check=False,
+    completed: subprocess.CompletedProcess[str] = run_checkout_generator(
+        checkout=base_root,
+        python=python,
+        entry=BASE_GENERATOR_ENTRY,
+        arguments=(kind, str(project_dir), str(models)),
+        environment=_compile_environment(),
     )
     if completed.returncode != 0:
         raise CompileComparisonError(

scripts/release_performance/_helpers/benchmark.py

@@ -11,7 +11,9 @@
 from pathlib import Path
 from typing import cast
 
+from scripts.compile_performance_ratio._helpers.measure import run_checkout_generator
 from scripts.release_performance.constants import (
+    BASELINE_GENERATOR_ENTRY,
     BUILD_PROJECT,
     BUILD_WARMUPS,
     COMMAND_TIMEOUT_SECONDS,
@@ -20,6 +22,7 @@
     EXCLUDED_ENVIRONMENT_PREFIX,
     INSPECTION_PROJECT,
     INSPECTION_WARMUPS,
+    PRISTINE_DIRECTORY,
     STDERR_TAIL_CHARACTERS,
     TIME_FORMAT,
 )
@@ -42,14 +45,34 @@
 def write_pristine_projects(*, root: Path, inspection_models: int, build_models: int) -> Path:
     """Generate the inspection and build benchmarks once so every version reads the same files."""
 
-    pristine: Path = root / "pristine"
+    pristine: Path = root / PRISTINE_DIRECTORY
     write_inspection_benchmark_project(
         project_dir=pristine / INSPECTION_PROJECT, model_count=inspection_models
     )
     write_build_benchmark_project(project_dir=pristine / BUILD_PROJECT, model_count=build_models)
     return pristine
 
 
+def write_baseline_pristine_projects(
+    *, source: Path, root: Path, inspection_models: int, build_models: int
+) -> Path:
+    """Generate the baseline's benchmarks with the baseline source's own generator."""
+
+    completed: subprocess.CompletedProcess[str] = run_checkout_generator(
+        checkout=source,
+        python=Path(sys.executable).absolute(),
+        entry=BASELINE_GENERATOR_ENTRY,
+        arguments=(str(root), str(inspection_models), str(build_models)),
+        environment=_environment(),
+    )
+    if completed.returncode != 0:
+        raise ReleasePerformanceError(
+            f"Baseline benchmark generation with the generator in {source} failed with exit "
+            f"{completed.returncode}:\n{completed.stderr[-STDERR_TAIL_CHARACTERS:]}"
+        )
+    return root / PRISTINE_DIRECTORY
+
+
 def prepare_version_projects(
     *, version: InstalledVersion, pristine: Path, root: Path
 ) -> dict[str, Path]:

scripts/release_performance/_helpers/report.py

@@ -31,6 +31,9 @@ def comparison_markdown(*, comparison: ReleaseComparison) -> str:
         f"{comparison.runs} interleaved runs per version; wall and CPU are medians, "
         "peak RSS is the worst run.",
         "",
+        "Benchmark projects are generated per side: the baseline runs projects from its own "
+        f"generator ({comparison.baseline_generator}), the candidate from the candidate's.",
+        "",
         f"Limits: wall {_percent(MAX_WALL_RATIO)} (ignored under "
         f"{MIN_WALL_REGRESSION_SECONDS:g} s), CPU {_percent(MAX_CPU_RATIO)} (ignored under "
         f"{MIN_CPU_REGRESSION_SECONDS:g} s), peak RSS {_percent(MAX_RSS_RATIO)} (ignored under "

scripts/release_performance/_helpers/versions.py

@@ -3,12 +3,14 @@
 from __future__ import annotations
 
 import hashlib
+import io
 import json
 import platform
 import re
 import shutil
 import subprocess
 import sys
+import tarfile
 import time
 import urllib.request
 from pathlib import Path
@@ -22,6 +24,7 @@
     PYPI_SIMPLE_JSON,
     PYPI_SIMPLE_URL,
     RELEASE_TAG_PATTERN,
+    RELEASE_TAG_PREFIX,
     STDERR_TAIL_CHARACTERS,
     UNIVERSAL_WHEEL_MACHINE,
     WHEEL_MACHINE_ALIASES,
@@ -118,6 +121,28 @@ def tagged_versions(*, repo_dir: Path) -> tuple[str, ...]:
     )
 
 
+def release_source(*, version: str, repo_dir: Path, destination: Path) -> Path:
+    """Extract the source tree of release tag v<version> from repo_dir into destination."""
+
+    tag: str = RELEASE_TAG_PREFIX + version
+    completed: subprocess.CompletedProcess[bytes] = subprocess.run(
+        ["git", "-C", str(repo_dir), "archive", "--format=tar", tag],
+        capture_output=True,
+        check=False,
+    )
+    if completed.returncode != 0:
+        raise ReleasePerformanceError(
+            f"Release tag {tag} is not available in {repo_dir}, so the baseline's own benchmark "
+            "generator cannot be used; fetch the release tags or pass --baseline-source: "
+            + _tail(completed.stderr.decode("utf-8", errors="replace"))
+        )
+    shutil.rmtree(destination, ignore_errors=True)
+    destination.mkdir(parents=True)
+    with tarfile.open(fileobj=io.BytesIO(completed.stdout)) as archive:
+        archive.extractall(destination, filter="data")
+    return destination
+
+
 def resolve_baseline(*, candidate: str, repo_dir: Path) -> str:
     """Return the highest published or not-yet-published tagged release below the candidate."""
 

scripts/release_performance/_helpers/workflow.py

@@ -13,6 +13,7 @@
 from scripts.release_performance._helpers.benchmark import (
     compare_command,
     prepare_version_projects,
+    write_baseline_pristine_projects,
     write_pristine_projects,
 )
 from scripts.release_performance._helpers.report import (
@@ -30,13 +31,17 @@
     install_published,
     install_wheel,
     installed_version,
+    release_source,
     resolve_baseline,
 )
 from scripts.release_performance.constants import (
+    BASELINE_GENERATED_DIRECTORY,
     BASELINE_LABEL,
     BASELINE_PUBLICATION_WAIT_SECONDS,
+    BASELINE_SOURCE_DIRECTORY,
     BENCHMARK_COMMANDS,
     CANDIDATE_LABEL,
+    RELEASE_TAG_PREFIX,
     REPO_ROOT,
 )
 from scripts.release_performance.exceptions import ReleasePerformanceError
@@ -104,16 +109,40 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison:
         file=sys.stderr,
     )
     load_before: tuple[float, float, float] = os.getloadavg()
-    print("Generating benchmark projects", file=sys.stderr)
-    pristine: Path = write_pristine_projects(
-        root=root, inspection_models=options.inspection_models, build_models=options.build_models
+    print("Generating candidate benchmark projects", file=sys.stderr)
+    pristine: dict[str, Path] = {
+        CANDIDATE_LABEL: write_pristine_projects(
+            root=root,
+            inspection_models=options.inspection_models,
+            build_models=options.build_models,
+        )
+    }
+    baseline_generator: str = (
+        str(options.baseline_source)
+        if options.baseline_source is not None
+        else RELEASE_TAG_PREFIX + baseline.version
+    )
+    print(f"Generating baseline benchmark projects with {baseline_generator}", file=sys.stderr)
+    pristine[BASELINE_LABEL] = write_baseline_pristine_projects(
+        source=(
+            options.baseline_source.absolute()
+            if options.baseline_source is not None
+            else release_source(
+                version=baseline.version,
+                repo_dir=REPO_ROOT,
+                destination=root / BASELINE_SOURCE_DIRECTORY,
+            )
+        ),
+        root=root / BASELINE_GENERATED_DIRECTORY,
+        inspection_models=options.inspection_models,
+        build_models=options.build_models,
     )
     versions: tuple[InstalledVersion, InstalledVersion] = (baseline, candidate)
     projects: dict[str, dict[str, Path]] = {}
     for version in versions:
         print(f"Warming caches for {version.label} {version.version}", file=sys.stderr)
         projects[version.label] = prepare_version_projects(
-            version=version, pristine=pristine, root=root
+            version=version, pristine=pristine[version.label], root=root
         )
     measured: list[CommandComparison] = []
     skipped: list[SkippedCommand] = []
@@ -133,6 +162,7 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison:
     return ReleaseComparison(
         baseline_version=baseline.version,
         candidate_version=candidate.version,
+        baseline_generator=baseline_generator,
         runs=options.runs,
         runner=RunnerContext(
             cpu_model=cpu_model(),

scripts/release_performance/constants.py

@@ -42,6 +42,16 @@
 TIME_FORMAT: str = "%e %M %U %S"
 COMPILE_COMMAND: str = "compile"
 STDERR_TAIL_CHARACTERS: int = 2_000
+PRISTINE_DIRECTORY: str = "pristine"
+BASELINE_GENERATED_DIRECTORY: str = "baseline-generated"
+BASELINE_SOURCE_DIRECTORY: str = "baseline-source"
+RELEASE_TAG_PREFIX: str = "v"
+BASELINE_GENERATOR_ENTRY: str = (
+    "import sys; from pathlib import Path; "
+    "from scripts.release_performance._helpers.benchmark import write_pristine_projects; "
+    "write_pristine_projects("
+    "root=Path(sys.argv[1]), inspection_models=int(sys.argv[2]), build_models=int(sys.argv[3]))"
+)
 
 BENCHMARK_COMMANDS: tuple[BenchmarkCommand, ...] = (
     BenchmarkCommand(

scripts/release_performance/main/compare.py

@@ -39,6 +39,15 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions:
     parser.add_argument("--python", default=DEFAULT_PYTHON)
     parser.add_argument("--inspection-models", type=int, default=DEFAULT_INSPECTION_MODELS)
     parser.add_argument("--build-models", type=int, default=DEFAULT_BUILD_MODELS)
+    parser.add_argument(
+        "--baseline-source",
+        type=Path,
+        default=None,
+        help=(
+            "Source tree whose benchmark generator writes the baseline's projects. Defaults to "
+            "the baseline's release tag extracted from this repository."
+        ),
+    )
     parser.add_argument("--work-dir", type=Path, default=None)
     parser.add_argument("--output", type=Path, default=None)
     args: argparse.Namespace = parser.parse_args(argv)
@@ -53,4 +62,5 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions:
         build_models=args.build_models,
         work_dir=args.work_dir,
         output=args.output,
+        baseline_source=args.baseline_source,
     )

scripts/release_performance/models.py

@@ -68,6 +68,7 @@ class RunnerContext:
 class ReleaseComparison:
     baseline_version: str
     candidate_version: str
+    baseline_generator: str
     runs: int
     runner: RunnerContext
     commands: tuple[CommandComparison, ...]
@@ -84,3 +85,4 @@ class ComparisonOptions:
     build_models: int
     work_dir: Path | None
     output: Path | None
+    baseline_source: Path | None = None

tests/e2e/scripts/release_performance/_test_types.py

@@ -10,5 +10,6 @@ class ReleasePerformanceTestCase:
     description: str
     inspection_models: int
     build_models: int
-    expected_return_code: int
+    expected_outcomes: tuple[tuple[int, bool], ...]
     expected_fragments: tuple[str, ...]
+    expected_marked_projects: tuple[str, ...]

tests/e2e/scripts/release_performance/helpers.py

@@ -0,0 +1,35 @@
+"""Fixtures for release performance comparisons with per-side benchmark generators."""
+
+from __future__ import annotations
+
+import shutil
+from pathlib import Path
+
+BASELINE_MARKER: str = "BASELINE_MARKER"
+_MARKING_WRAPPER: str = """
+
+_unmarked_write_pristine_projects = write_pristine_projects
+
+
+def write_pristine_projects(*, root: Path, inspection_models: int, build_models: int) -> Path:
+    pristine: Path = _unmarked_write_pristine_projects(
+        root=root, inspection_models=inspection_models, build_models=build_models
+    )
+    for project in pristine.iterdir():
+        (project / "BASELINE_MARKER").write_text("baseline", encoding="utf-8")
+    return pristine
+"""
+
+
+def write_marked_baseline_source(*, repo_root: Path, destination: Path) -> Path:
+    """Copy this checkout's generators and mark every project the copy generates."""
+
+    for name in ("scripts", "tests"):
+        _ = shutil.copytree(
+            repo_root / name, destination / name, ignore=shutil.ignore_patterns("__pycache__")
+        )
+    benchmark: Path = destination / "scripts" / "release_performance" / "_helpers" / "benchmark.py"
+    _ = benchmark.write_text(
+        benchmark.read_text(encoding="utf-8") + _MARKING_WRAPPER, encoding="utf-8"
+    )
+    return destination