ci: generate release baseline benchmarks with the baseline environment

.github/workflows/release-performance.yml

@@ -22,9 +22,13 @@ on:
       - scripts/compare_release_performance.py
       - scripts/release_performance/**
       - scripts/compile_performance_ratio/_helpers/edit.py
+      - scripts/compile_performance_ratio/_helpers/measure.py
+      - scripts/compile_performance_ratio/_helpers/report.py
       - scripts/compile_performance_ratio/constants.py
       - scripts/compile_performance_ratio/exceptions.py
+      - scripts/compile_performance_ratio/models.py
       - scripts/cold_compile_performance/_helpers/dense_project.py
+      - scripts/cold_compile_performance/constants.py
 
 permissions:
   contents: read

scripts/compile_performance_ratio/_helpers/measure.py

@@ -69,15 +69,19 @@ def run_checkout_generator(
     entry: str,
     arguments: tuple[str, ...],
     environment: dict[str, str],
+    library_paths: tuple[Path, ...],
 ) -> subprocess.CompletedProcess[str]:
-    """Run generator code from another checkout's source tree with this `python`."""
+    """Run another checkout's generator with `python`, importing `library_paths` before its own."""
 
     return subprocess.run(
         [str(python), "-c", entry, *arguments],
         cwd=checkout,
         capture_output=True,
         text=True,
-        env={**environment, PYTHONPATH_KEY: str(checkout)},
+        env={
+            **environment,
+            PYTHONPATH_KEY: os.pathsep.join(str(path) for path in (checkout, *library_paths)),
+        },
         check=False,
     )
 
@@ -93,6 +97,7 @@ def write_base_benchmark_project(
         entry=BASE_GENERATOR_ENTRY,
         arguments=(kind, str(project_dir), str(models)),
         environment=_compile_environment(),
+        library_paths=(),
     )
     if completed.returncode != 0:
         raise CompileComparisonError(

scripts/release_performance/_helpers/benchmark.py

@@ -14,6 +14,7 @@
 from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project
 from scripts.compile_performance_ratio._helpers.edit import apply_one_model_edit
 from scripts.compile_performance_ratio._helpers.measure import run_checkout_generator
+from scripts.release_performance._helpers.versions import library_paths
 from scripts.release_performance.constants import (
     BASELINE_GENERATOR_ENTRY,
     BUILD_PROJECT,
@@ -60,12 +61,13 @@ def write_pristine_projects(*, root: Path, inspection_models: int, build_models:
 
 
 def write_baseline_pristine_projects(
-    *, source: Path, root: Path, inspection_models: int, build_models: int
+    *, source: Path, python: Path, root: Path, inspection_models: int, build_models: int
 ) -> Path:
-    """Generate the baseline's benchmarks with the baseline source's own generator."""
+    """Generate the baseline's benchmarks with its own generator and installed `sqlbuild`."""
 
     _run_baseline_generator(
         source=source,
+        python=python,
         entry=BASELINE_GENERATOR_ENTRY,
         arguments=(str(root), str(inspection_models), str(build_models)),
         description="benchmark",
@@ -79,11 +81,14 @@ def write_dense_project(*, pristine: Path, models: int) -> None:
     write_dense_compile_project(project_dir=pristine / DENSE_PROJECT, model_count=models)
 
 
-def write_baseline_dense_project(*, source: Path, pristine: Path, models: int) -> None:
-    """Generate the baseline's dense benchmark with the baseline source's own generator."""
+def write_baseline_dense_project(
+    *, source: Path, python: Path, pristine: Path, models: int
+) -> None:
+    """Generate the baseline's dense benchmark with its own generator and installed `sqlbuild`."""
 
     _run_baseline_generator(
         source=source,
+        python=python,
         entry=DENSE_GENERATOR_ENTRY,
         arguments=(str(pristine / DENSE_PROJECT), str(models)),
         description="dense benchmark",
@@ -166,14 +171,15 @@ def compare_command(
 
 
 def _run_baseline_generator(
-    *, source: Path, entry: str, arguments: tuple[str, ...], description: str
+    *, source: Path, python: Path, entry: str, arguments: tuple[str, ...], description: str
 ) -> None:
     completed: subprocess.CompletedProcess[str] = run_checkout_generator(
         checkout=source,
         python=Path(sys.executable).absolute(),
         entry=entry,
         arguments=arguments,
         environment=_environment(),
+        library_paths=library_paths(python=python),
     )
     if completed.returncode != 0:
         raise ReleasePerformanceError(

scripts/release_performance/_helpers/versions.py

@@ -36,6 +36,10 @@
 _DISTRIBUTION_VERSION: re.Pattern[str] = re.compile(
     rf"^{PACKAGE_NAME}-(\d+\.\d+\.\d+)(?:-.+\.whl|\.tar\.gz)$"
 )
+_LIBRARY_PATHS_PROBE: str = (
+    "import json, sys, sysconfig; print(json.dumps({'version': list(sys.version_info[:2]), "
+    "'paths': [sysconfig.get_path('purelib'), sysconfig.get_path('platlib')]}))"
+)
 
 
 def release_versions(*, index: dict[str, object]) -> tuple[str, ...]:
@@ -211,6 +215,27 @@ def installed_version(*, sqb: Path) -> str:
     return match.group(0)
 
 
+def library_paths(*, python: Path) -> tuple[Path, ...]:
+    """Return the package directories of `python`'s environment, which must match this Python."""
+
+    completed: subprocess.CompletedProcess[str] = subprocess.run(
+        [str(python), "-c", _LIBRARY_PATHS_PROBE], capture_output=True, text=True, check=False
+    )
+    if completed.returncode != 0:
+        raise ReleasePerformanceError(
+            f"Reading the package directories of {python} failed: {_tail(completed.stderr)}"
+        )
+    probe: dict[str, list[object]] = cast(dict[str, list[object]], json.loads(completed.stdout))
+    version: tuple[object, ...] = tuple(probe["version"])
+    if version != tuple(sys.version_info[:2]):
+        raise ReleasePerformanceError(
+            f"{python} runs Python {'.'.join(map(str, version))}, but its packages are imported "
+            f"by Python {sys.version_info.major}.{sys.version_info.minor}; pass --python "
+            f"{sys.version_info.major}.{sys.version_info.minor}"
+        )
+    return tuple(dict.fromkeys(Path(str(path)) for path in probe["paths"]))
+
+
 def existing_sqb(*, venv_dir: Path) -> Path:
     """Return the sqb entry point of an existing virtual environment."""
 

scripts/release_performance/_helpers/workflow.py

@@ -138,12 +138,16 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison:
     )
     pristine[BASELINE_LABEL] = write_baseline_pristine_projects(
         source=baseline_source,
+        python=baseline.python,
         root=root / BASELINE_GENERATED_DIRECTORY,
         inspection_models=options.inspection_models,
         build_models=options.build_models,
     )
     write_baseline_dense_project(
-        source=baseline_source, pristine=pristine[BASELINE_LABEL], models=options.dense_models
+        source=baseline_source,
+        python=baseline.python,
+        pristine=pristine[BASELINE_LABEL],
+        models=options.dense_models,
     )
     versions: tuple[InstalledVersion, InstalledVersion] = (baseline, candidate)
     projects: dict[str, dict[str, Path]] = {}
@@ -196,7 +200,9 @@ def _install(
         sqb = install_published(
             version=spec, venv_dir=venv_dir, python=python, wait_seconds=wait_seconds
         )
-    return InstalledVersion(label=label, version=installed_version(sqb=sqb), sqb=sqb)
+    return InstalledVersion(
+        label=label, version=installed_version(sqb=sqb), sqb=sqb, python=sqb.with_name("python")
+    )
 
 
 def _write_evidence(*, path: Path, comparison: ReleaseComparison) -> None:

scripts/release_performance/models.py

@@ -22,6 +22,7 @@ class InstalledVersion:
     label: str
     version: str
     sqb: Path
+    python: Path
 
 
 @dataclass(frozen=True)

tests/unit/scripts/release_performance/_helpers/helpers.py

@@ -1,6 +1,7 @@
 """Builders for release performance comparison test data."""
 
 import subprocess
+import sys
 from pathlib import Path
 
 from scripts.release_performance.models import CommandComparison, CommandSample
@@ -15,6 +16,18 @@ def write_pristine_projects(*, root: Path, inspection_models: int, build_models:
         (root / "pristine" / name / "BASELINE_MARKER").write_text(str(models))
     return root / "pristine"
 """
+BASELINE_ONLY_SYMBOL: str = "BASELINE_ONLY_RELEASE_SYMBOL"
+BASELINE_SQLBUILD_GENERATOR: str = f"""from pathlib import Path
+
+from sqlbuild import {BASELINE_ONLY_SYMBOL}
+
+
+def write_pristine_projects(*, root: Path, inspection_models: int, build_models: int) -> Path:
+    for name in ("inspection", "build"):
+        (root / "pristine" / name).mkdir(parents=True)
+        (root / "pristine" / name / "BASELINE_MARKER").write_text({BASELINE_ONLY_SYMBOL})
+    return root / "pristine"
+"""
 FAILING_GENERATOR: str = """def write_pristine_projects(*, root, inspection_models, build_models):
     raise RuntimeError("baseline generator exploded")
 """
@@ -25,6 +38,15 @@ def write_dense_compile_project(*, project_dir: Path, model_count: int) -> None:
     project_dir.mkdir(parents=True)
     (project_dir / "BASELINE_MARKER").write_text(str(model_count))
 """
+BASELINE_SQLBUILD_DENSE_GENERATOR: str = f"""from pathlib import Path
+
+from sqlbuild import {BASELINE_ONLY_SYMBOL}
+
+
+def write_dense_compile_project(*, project_dir: Path, model_count: int) -> None:
+    project_dir.mkdir(parents=True)
+    (project_dir / "BASELINE_MARKER").write_text({BASELINE_ONLY_SYMBOL})
+"""
 FAILING_DENSE_GENERATOR: str = """def write_dense_compile_project(*, project_dir, model_count):
     raise RuntimeError("baseline dense generator exploded")
 """
@@ -80,6 +102,27 @@ def write_baseline_dense_source(*, root: Path, generator: str) -> Path:
     return root
 
 
+def write_baseline_environment(*, root: Path) -> Path:
+    """Create a virtual environment whose sqlbuild defines a symbol this checkout lacks."""
+
+    _ = subprocess.run(
+        [sys.executable, "-m", "venv", "--without-pip", str(root)], check=True, capture_output=True
+    )
+    python: Path = root / "bin" / "python"
+    purelib: str = subprocess.run(
+        [str(python), "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"],
+        check=True,
+        capture_output=True,
+        text=True,
+    ).stdout.strip()
+    package: Path = Path(purelib) / "sqlbuild"
+    package.mkdir(parents=True)
+    _ = (package / "__init__.py").write_text(
+        f"{BASELINE_ONLY_SYMBOL} = {BASELINE_ONLY_SYMBOL!r}\n", encoding="utf-8"
+    )
+    return python
+
+
 def tagged_repository(*, root: Path, tag: str) -> Path:
     """Create a one-commit git repository with a tracked file and the given tag."""
 

tests/unit/scripts/release_performance/_helpers/test_benchmark.py

@@ -18,12 +18,16 @@
     ReleaseSourceTestCase,
 )
 from tests.unit.scripts.release_performance._helpers.helpers import (
+    BASELINE_ONLY_SYMBOL,
+    BASELINE_SQLBUILD_DENSE_GENERATOR,
+    BASELINE_SQLBUILD_GENERATOR,
     FAILING_DENSE_GENERATOR,
     FAILING_GENERATOR,
     MARKER_DENSE_GENERATOR,
     MARKER_GENERATOR,
     tagged_repository,
     write_baseline_dense_source,
+    write_baseline_environment,
     write_baseline_source,
 )
 
@@ -37,6 +41,12 @@
             expected_projects=("build", "inspection"),
             expected_error_fragments=(),
         ),
+        BaselineGeneratorTestCase(
+            description="the generator imports the baseline environment's sqlbuild, not this one",
+            generator=BASELINE_SQLBUILD_GENERATOR,
+            expected_projects=("build", "inspection"),
+            expected_error_fragments=(),
+        ),
         BaselineGeneratorTestCase(
             description="a failing baseline generator is reported with its stderr",
             generator=FAILING_GENERATOR,
@@ -58,7 +68,11 @@ def test_given_baseline_source_when_generating_then_uses_its_own_generator(
 
     try:
         _ = write_baseline_pristine_projects(
-            source=source, root=root, inspection_models=30, build_models=10
+            source=source,
+            python=write_baseline_environment(root=tmp_path / "venv"),
+            root=root,
+            inspection_models=30,
+            build_models=10,
         )
     except ReleasePerformanceError as raised:
         error = str(raised)
@@ -79,6 +93,12 @@ def test_given_baseline_source_when_generating_then_uses_its_own_generator(
             expected_markers=("40",),
             expected_error_fragments=(),
         ),
+        BaselineDenseGeneratorTestCase(
+            description="the dense generator imports the baseline environment's sqlbuild",
+            generator=BASELINE_SQLBUILD_DENSE_GENERATOR,
+            expected_markers=(BASELINE_ONLY_SYMBOL,),
+            expected_error_fragments=(),
+        ),
         BaselineDenseGeneratorTestCase(
             description="a failing baseline dense generator is reported with its stderr",
             generator=FAILING_DENSE_GENERATOR,
@@ -101,7 +121,12 @@ def test_given_baseline_source_when_generating_dense_project_then_uses_its_own_g
     error: str = ""
 
     try:
-        write_baseline_dense_project(source=source, pristine=pristine, models=40)
+        write_baseline_dense_project(
+            source=source,
+            python=write_baseline_environment(root=tmp_path / "venv"),
+            pristine=pristine,
+            models=40,
+        )
     except ReleasePerformanceError as raised:
         error = str(raised)