ci: generate release baseline benchmarks with the baseline environment

The release comparison ran the previous release's benchmark generator with
the head interpreter and only the baseline source on PYTHONPATH, so the
generator imported the head sqlbuild package and failed once head removed a
symbol the baseline still imports. Put the baseline environment's package
directories ahead of the head packages so sqlbuild and its native
dependencies resolve to the baseline install, and require the baseline
environment to use the same Python minor version.

Author: kvlonge

.github/workflows/release-performance.yml

@@ -22,9 +22,13 @@ on:
       - scripts/compare_release_performance.py
       - scripts/release_performance/**
       - scripts/compile_performance_ratio/_helpers/edit.py
+      - scripts/compile_performance_ratio/_helpers/measure.py
+      - scripts/compile_performance_ratio/_helpers/report.py
       - scripts/compile_performance_ratio/constants.py
       - scripts/compile_performance_ratio/exceptions.py
+      - scripts/compile_performance_ratio/models.py
       - scripts/cold_compile_performance/_helpers/dense_project.py
+      - scripts/cold_compile_performance/constants.py
 
 permissions:
   contents: read

scripts/compile_performance_ratio/_helpers/measure.py

@@ -69,15 +69,19 @@ def run_checkout_generator(
     entry: str,
     arguments: tuple[str, ...],
     environment: dict[str, str],
+    library_paths: tuple[Path, ...],
 ) -> subprocess.CompletedProcess[str]:
-    """Run generator code from another checkout's source tree with this `python`."""
+    """Run another checkout's generator with `python`, importing `library_paths` before its own."""
 
     return subprocess.run(
         [str(python), "-c", entry, *arguments],
         cwd=checkout,
         capture_output=True,
         text=True,
-        env={**environment, PYTHONPATH_KEY: str(checkout)},
+        env={
+            **environment,
+            PYTHONPATH_KEY: os.pathsep.join(str(path) for path in (checkout, *library_paths)),
+        },
         check=False,
     )
 
@@ -93,6 +97,7 @@ def write_base_benchmark_project(
         entry=BASE_GENERATOR_ENTRY,
         arguments=(kind, str(project_dir), str(models)),
         environment=_compile_environment(),
+        library_paths=(),
     )
     if completed.returncode != 0:
         raise CompileComparisonError(

scripts/release_performance/_helpers/benchmark.py

@@ -14,6 +14,7 @@
 from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project
 from scripts.compile_performance_ratio._helpers.edit import apply_one_model_edit
 from scripts.compile_performance_ratio._helpers.measure import run_checkout_generator
+from scripts.release_performance._helpers.versions import library_paths
 from scripts.release_performance.constants import (
     BASELINE_GENERATOR_ENTRY,
     BUILD_PROJECT,
@@ -60,12 +61,13 @@ def write_pristine_projects(*, root: Path, inspection_models: int, build_models:
 
 
 def write_baseline_pristine_projects(
-    *, source: Path, root: Path, inspection_models: int, build_models: int
+    *, source: Path, python: Path, root: Path, inspection_models: int, build_models: int
 ) -> Path:
-    """Generate the baseline's benchmarks with the baseline source's own generator."""
+    """Generate the baseline's benchmarks with its own generator and installed `sqlbuild`."""
 
     _run_baseline_generator(
         source=source,
+        python=python,
         entry=BASELINE_GENERATOR_ENTRY,
         arguments=(str(root), str(inspection_models), str(build_models)),
         description="benchmark",
@@ -79,11 +81,14 @@ def write_dense_project(*, pristine: Path, models: int) -> None:
     write_dense_compile_project(project_dir=pristine / DENSE_PROJECT, model_count=models)
 
 
-def write_baseline_dense_project(*, source: Path, pristine: Path, models: int) -> None:
-    """Generate the baseline's dense benchmark with the baseline source's own generator."""
+def write_baseline_dense_project(
+    *, source: Path, python: Path, pristine: Path, models: int
+) -> None:
+    """Generate the baseline's dense benchmark with its own generator and installed `sqlbuild`."""
 
     _run_baseline_generator(
         source=source,
+        python=python,
         entry=DENSE_GENERATOR_ENTRY,
         arguments=(str(pristine / DENSE_PROJECT), str(models)),
         description="dense benchmark",
@@ -166,14 +171,15 @@ def compare_command(
 
 
 def _run_baseline_generator(
-    *, source: Path, entry: str, arguments: tuple[str, ...], description: str
+    *, source: Path, python: Path, entry: str, arguments: tuple[str, ...], description: str
 ) -> None:
     completed: subprocess.CompletedProcess[str] = run_checkout_generator(
         checkout=source,
         python=Path(sys.executable).absolute(),
         entry=entry,
         arguments=arguments,
         environment=_environment(),
+        library_paths=library_paths(python=python),
     )
     if completed.returncode != 0:
         raise ReleasePerformanceError(

scripts/release_performance/_helpers/versions.py

@@ -36,6 +36,10 @@
 _DISTRIBUTION_VERSION: re.Pattern[str] = re.compile(
     rf"^{PACKAGE_NAME}-(\d+\.\d+\.\d+)(?:-.+\.whl|\.tar\.gz)$"
 )
+_LIBRARY_PATHS_PROBE: str = (
+    "import json, sys, sysconfig; print(json.dumps({'version': list(sys.version_info[:2]), "
+    "'paths': [sysconfig.get_path('purelib'), sysconfig.get_path('platlib')]}))"
+)
 
 
 def release_versions(*, index: dict[str, object]) -> tuple[str, ...]:
@@ -211,6 +215,27 @@ def installed_version(*, sqb: Path) -> str:
     return match.group(0)
 
 
+def library_paths(*, python: Path) -> tuple[Path, ...]:
+    """Return the package directories of `python`'s environment, which must match this Python."""
+
+    completed: subprocess.CompletedProcess[str] = subprocess.run(
+        [str(python), "-c", _LIBRARY_PATHS_PROBE], capture_output=True, text=True, check=False
+    )
+    if completed.returncode != 0:
+        raise ReleasePerformanceError(
+            f"Reading the package directories of {python} failed: {_tail(completed.stderr)}"
+        )
+    probe: dict[str, list[object]] = cast(dict[str, list[object]], json.loads(completed.stdout))
+    version: tuple[object, ...] = tuple(probe["version"])
+    if version != tuple(sys.version_info[:2]):
+        raise ReleasePerformanceError(
+            f"{python} runs Python {'.'.join(map(str, version))}, but its packages are imported "
+            f"by Python {sys.version_info.major}.{sys.version_info.minor}; pass --python "
+            f"{sys.version_info.major}.{sys.version_info.minor}"
+        )
+    return tuple(dict.fromkeys(Path(str(path)) for path in probe["paths"]))
+
+
 def existing_sqb(*, venv_dir: Path) -> Path:
     """Return the sqb entry point of an existing virtual environment."""
 

scripts/release_performance/_helpers/workflow.py

@@ -138,12 +138,16 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison:
     )
     pristine[BASELINE_LABEL] = write_baseline_pristine_projects(
         source=baseline_source,
+        python=baseline.python,
         root=root / BASELINE_GENERATED_DIRECTORY,
         inspection_models=options.inspection_models,
         build_models=options.build_models,
     )
     write_baseline_dense_project(
-        source=baseline_source, pristine=pristine[BASELINE_LABEL], models=options.dense_models
+        source=baseline_source,
+        python=baseline.python,
+        pristine=pristine[BASELINE_LABEL],
+        models=options.dense_models,
     )
     versions: tuple[InstalledVersion, InstalledVersion] = (baseline, candidate)
     projects: dict[str, dict[str, Path]] = {}
@@ -196,7 +200,9 @@ def _install(
         sqb = install_published(
             version=spec, venv_dir=venv_dir, python=python, wait_seconds=wait_seconds
         )
-    return InstalledVersion(label=label, version=installed_version(sqb=sqb), sqb=sqb)
+    return InstalledVersion(
+        label=label, version=installed_version(sqb=sqb), sqb=sqb, python=sqb.with_name("python")
+    )
 
 
 def _write_evidence(*, path: Path, comparison: ReleaseComparison) -> None:

scripts/release_performance/models.py

@@ -22,6 +22,7 @@ class InstalledVersion:
     label: str
     version: str
     sqb: Path
+    python: Path
 
 
 @dataclass(frozen=True)

tests/unit/scripts/release_performance/_helpers/helpers.py

@@ -1,6 +1,7 @@
 """Builders for release performance comparison test data."""
 
 import subprocess
+import sys
 from pathlib import Path
 
 from scripts.release_performance.models import CommandComparison, CommandSample
@@ -15,6 +16,18 @@ def write_pristine_projects(*, root: Path, inspection_models: int, build_models:
         (root / "pristine" / name / "BASELINE_MARKER").write_text(str(models))
     return root / "pristine"
 """
+BASELINE_ONLY_SYMBOL: str = "BASELINE_ONLY_RELEASE_SYMBOL"
+BASELINE_SQLBUILD_GENERATOR: str = f"""from pathlib import Path
+
+from sqlbuild import {BASELINE_ONLY_SYMBOL}
+
+
+def write_pristine_projects(*, root: Path, inspection_models: int, build_models: int) -> Path:
+    for name in ("inspection", "build"):
+        (root / "pristine" / name).mkdir(parents=True)
+        (root / "pristine" / name / "BASELINE_MARKER").write_text({BASELINE_ONLY_SYMBOL})
+    return root / "pristine"
+"""
 FAILING_GENERATOR: str = """def write_pristine_projects(*, root, inspection_models, build_models):
     raise RuntimeError("baseline generator exploded")
 """
@@ -25,6 +38,15 @@ def write_dense_compile_project(*, project_dir: Path, model_count: int) -> None:
     project_dir.mkdir(parents=True)
     (project_dir / "BASELINE_MARKER").write_text(str(model_count))
 """
+BASELINE_SQLBUILD_DENSE_GENERATOR: str = f"""from pathlib import Path
+
+from sqlbuild import {BASELINE_ONLY_SYMBOL}
+
+
+def write_dense_compile_project(*, project_dir: Path, model_count: int) -> None:
+    project_dir.mkdir(parents=True)
+    (project_dir / "BASELINE_MARKER").write_text({BASELINE_ONLY_SYMBOL})
+"""
 FAILING_DENSE_GENERATOR: str = """def write_dense_compile_project(*, project_dir, model_count):
     raise RuntimeError("baseline dense generator exploded")
 """
@@ -80,6 +102,27 @@ def write_baseline_dense_source(*, root: Path, generator: str) -> Path:
     return root
 
 
+def write_baseline_environment(*, root: Path) -> Path:
+    """Create a virtual environment whose sqlbuild defines a symbol this checkout lacks."""
+
+    _ = subprocess.run(
+        [sys.executable, "-m", "venv", "--without-pip", str(root)], check=True, capture_output=True
+    )
+    python: Path = root / "bin" / "python"
+    purelib: str = subprocess.run(
+        [str(python), "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    package: Path = Path(purelib) / "sqlbuild"
+    package.mkdir(parents=True)
+    _ = (package / "__init__.py").write_text(
+        f"{BASELINE_ONLY_SYMBOL} = {BASELINE_ONLY_SYMBOL!r}\n", encoding="utf-8"
+    )
+    return python
+
+
 def tagged_repository(*, root: Path, tag: str) -> Path:
     """Create a one-commit git repository with a tracked file and the given tag."""
 

tests/unit/scripts/release_performance/_helpers/test_benchmark.py

@@ -18,12 +18,16 @@
     ReleaseSourceTestCase,
 )
 from tests.unit.scripts.release_performance._helpers.helpers import (
+    BASELINE_ONLY_SYMBOL,
+    BASELINE_SQLBUILD_DENSE_GENERATOR,
+    BASELINE_SQLBUILD_GENERATOR,
     FAILING_DENSE_GENERATOR,
     FAILING_GENERATOR,
     MARKER_DENSE_GENERATOR,
     MARKER_GENERATOR,
     tagged_repository,
     write_baseline_dense_source,
+    write_baseline_environment,
     write_baseline_source,
 )
 
@@ -37,6 +41,12 @@
             expected_projects=("build", "inspection"),
             expected_error_fragments=(),
         ),
+        BaselineGeneratorTestCase(
+            description="the generator imports the baseline environment's sqlbuild, not this one",
+            generator=BASELINE_SQLBUILD_GENERATOR,
+            expected_projects=("build", "inspection"),
+            expected_error_fragments=(),
+        ),
         BaselineGeneratorTestCase(
             description="a failing baseline generator is reported with its stderr",
             generator=FAILING_GENERATOR,
@@ -58,7 +68,11 @@ def test_given_baseline_source_when_generating_then_uses_its_own_generator(
 
     try:
         _ = write_baseline_pristine_projects(
-            source=source, root=root, inspection_models=30, build_models=10
+            source=source,
+            python=write_baseline_environment(root=tmp_path / "venv"),
+            root=root,
+            inspection_models=30,
+            build_models=10,
         )
     except ReleasePerformanceError as raised:
         error = str(raised)
@@ -79,6 +93,12 @@ def test_given_baseline_source_when_generating_then_uses_its_own_generator(
             expected_markers=("40",),
             expected_error_fragments=(),
         ),
+        BaselineDenseGeneratorTestCase(
+            description="the dense generator imports the baseline environment's sqlbuild",
+            generator=BASELINE_SQLBUILD_DENSE_GENERATOR,
+            expected_markers=(BASELINE_ONLY_SYMBOL,),
+            expected_error_fragments=(),
+        ),
         BaselineDenseGeneratorTestCase(
             description="a failing baseline dense generator is reported with its stderr",
             generator=FAILING_DENSE_GENERATOR,
@@ -101,7 +121,12 @@ def test_given_baseline_source_when_generating_dense_project_then_uses_its_own_g
     error: str = ""
 
     try:
-        write_baseline_dense_project(source=source, pristine=pristine, models=40)
+        write_baseline_dense_project(
+            source=source,
+            python=write_baseline_environment(root=tmp_path / "venv"),
+            pristine=pristine,
+            models=40,
+        )
     except ReleasePerformanceError as raised:
         error = str(raised)