From d25f0deef377552557565d2da32e31e7bf93d878 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 8 Oct 2026 05:31:48 +0100 Subject: [PATCH] ci: generate release baseline benchmarks with the baseline environment The release comparison ran the previous release's benchmark generator with the head interpreter and only the baseline source on PYTHONPATH, so the generator imported the head sqlbuild package and failed once head removed a symbol the baseline still imports. Put the baseline environment's package directories ahead of the head packages so sqlbuild and its native dependencies resolve to the baseline install, and require the baseline environment to use the same Python minor version. --- .github/workflows/release-performance.yml | 4 ++ .../_helpers/measure.py | 9 +++- .../release_performance/_helpers/benchmark.py | 16 ++++--- .../release_performance/_helpers/versions.py | 25 +++++++++++ .../release_performance/_helpers/workflow.py | 10 ++++- scripts/release_performance/models.py | 1 + .../release_performance/_helpers/helpers.py | 43 +++++++++++++++++++ .../_helpers/test_benchmark.py | 29 ++++++++++++- 8 files changed, 126 insertions(+), 11 deletions(-) diff --git a/.github/workflows/release-performance.yml b/.github/workflows/release-performance.yml index 87637a9459..a565a03f62 100644 --- a/.github/workflows/release-performance.yml +++ b/.github/workflows/release-performance.yml @@ -22,9 +22,13 @@ on: - scripts/compare_release_performance.py - scripts/release_performance/** - scripts/compile_performance_ratio/_helpers/edit.py + - scripts/compile_performance_ratio/_helpers/measure.py + - scripts/compile_performance_ratio/_helpers/report.py - scripts/compile_performance_ratio/constants.py - scripts/compile_performance_ratio/exceptions.py + - scripts/compile_performance_ratio/models.py - scripts/cold_compile_performance/_helpers/dense_project.py + - scripts/cold_compile_performance/constants.py permissions: contents: read diff --git a/scripts/compile_performance_ratio/_helpers/measure.py b/scripts/compile_performance_ratio/_helpers/measure.py index 45123ec561..9931aefcc4 100644 --- a/scripts/compile_performance_ratio/_helpers/measure.py +++ b/scripts/compile_performance_ratio/_helpers/measure.py @@ -69,15 +69,19 @@ def run_checkout_generator( entry: str, arguments: tuple[str, ...], environment: dict[str, str], + library_paths: tuple[Path, ...], ) -> subprocess.CompletedProcess[str]: - """Run generator code from another checkout's source tree with this `python`.""" + """Run another checkout's generator with `python`, importing `library_paths` before its own.""" return subprocess.run( [str(python), "-c", entry, *arguments], cwd=checkout, capture_output=True, text=True, - env={**environment, PYTHONPATH_KEY: str(checkout)}, + env={ + **environment, + PYTHONPATH_KEY: os.pathsep.join(str(path) for path in (checkout, *library_paths)), + }, check=False, ) @@ -93,6 +97,7 @@ def write_base_benchmark_project( entry=BASE_GENERATOR_ENTRY, arguments=(kind, str(project_dir), str(models)), environment=_compile_environment(), + library_paths=(), ) if completed.returncode != 0: raise CompileComparisonError( diff --git a/scripts/release_performance/_helpers/benchmark.py b/scripts/release_performance/_helpers/benchmark.py index 5812031eb3..c95a70865e 100644 --- a/scripts/release_performance/_helpers/benchmark.py +++ b/scripts/release_performance/_helpers/benchmark.py @@ -14,6 +14,7 @@ from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project from scripts.compile_performance_ratio._helpers.edit import apply_one_model_edit from scripts.compile_performance_ratio._helpers.measure import run_checkout_generator +from scripts.release_performance._helpers.versions import library_paths from scripts.release_performance.constants import ( BASELINE_GENERATOR_ENTRY, BUILD_PROJECT, @@ -60,12 +61,13 @@ def write_pristine_projects(*, root: Path, inspection_models: int, build_models: def write_baseline_pristine_projects( - *, source: Path, root: Path, inspection_models: int, build_models: int + *, source: Path, python: Path, root: Path, inspection_models: int, build_models: int ) -> Path: - """Generate the baseline's benchmarks with the baseline source's own generator.""" + """Generate the baseline's benchmarks with its own generator and installed `sqlbuild`.""" _run_baseline_generator( source=source, + python=python, entry=BASELINE_GENERATOR_ENTRY, arguments=(str(root), str(inspection_models), str(build_models)), description="benchmark", @@ -79,11 +81,14 @@ def write_dense_project(*, pristine: Path, models: int) -> None: write_dense_compile_project(project_dir=pristine / DENSE_PROJECT, model_count=models) -def write_baseline_dense_project(*, source: Path, pristine: Path, models: int) -> None: - """Generate the baseline's dense benchmark with the baseline source's own generator.""" +def write_baseline_dense_project( + *, source: Path, python: Path, pristine: Path, models: int +) -> None: + """Generate the baseline's dense benchmark with its own generator and installed `sqlbuild`.""" _run_baseline_generator( source=source, + python=python, entry=DENSE_GENERATOR_ENTRY, arguments=(str(pristine / DENSE_PROJECT), str(models)), description="dense benchmark", @@ -166,7 +171,7 @@ def compare_command( def _run_baseline_generator( - *, source: Path, entry: str, arguments: tuple[str, ...], description: str + *, source: Path, python: Path, entry: str, arguments: tuple[str, ...], description: str ) -> None: completed: subprocess.CompletedProcess[str] = run_checkout_generator( checkout=source, @@ -174,6 +179,7 @@ def _run_baseline_generator( entry=entry, arguments=arguments, environment=_environment(), + library_paths=library_paths(python=python), ) if completed.returncode != 0: raise ReleasePerformanceError( diff --git a/scripts/release_performance/_helpers/versions.py b/scripts/release_performance/_helpers/versions.py index 25b5a8b0bb..2c9ddeeefe 100644 --- a/scripts/release_performance/_helpers/versions.py +++ b/scripts/release_performance/_helpers/versions.py @@ -36,6 +36,10 @@ _DISTRIBUTION_VERSION: re.Pattern[str] = re.compile( rf"^{PACKAGE_NAME}-(\d+\.\d+\.\d+)(?:-.+\.whl|\.tar\.gz)$" ) +_LIBRARY_PATHS_PROBE: str = ( + "import json, sys, sysconfig; print(json.dumps({'version': list(sys.version_info[:2]), " + "'paths': [sysconfig.get_path('purelib'), sysconfig.get_path('platlib')]}))" +) def release_versions(*, index: dict[str, object]) -> tuple[str, ...]: @@ -211,6 +215,27 @@ def installed_version(*, sqb: Path) -> str: return match.group(0) +def library_paths(*, python: Path) -> tuple[Path, ...]: + """Return the package directories of `python`'s environment, which must match this Python.""" + + completed: subprocess.CompletedProcess[str] = subprocess.run( + [str(python), "-c", _LIBRARY_PATHS_PROBE], capture_output=True, text=True, check=False + ) + if completed.returncode != 0: + raise ReleasePerformanceError( + f"Reading the package directories of {python} failed: {_tail(completed.stderr)}" + ) + probe: dict[str, list[object]] = cast(dict[str, list[object]], json.loads(completed.stdout)) + version: tuple[object, ...] = tuple(probe["version"]) + if version != tuple(sys.version_info[:2]): + raise ReleasePerformanceError( + f"{python} runs Python {'.'.join(map(str, version))}, but its packages are imported " + f"by Python {sys.version_info.major}.{sys.version_info.minor}; pass --python " + f"{sys.version_info.major}.{sys.version_info.minor}" + ) + return tuple(dict.fromkeys(Path(str(path)) for path in probe["paths"])) + + def existing_sqb(*, venv_dir: Path) -> Path: """Return the sqb entry point of an existing virtual environment.""" diff --git a/scripts/release_performance/_helpers/workflow.py b/scripts/release_performance/_helpers/workflow.py index 92f7ec7f1d..eac8ba6776 100644 --- a/scripts/release_performance/_helpers/workflow.py +++ b/scripts/release_performance/_helpers/workflow.py @@ -138,12 +138,16 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: ) pristine[BASELINE_LABEL] = write_baseline_pristine_projects( source=baseline_source, + python=baseline.python, root=root / BASELINE_GENERATED_DIRECTORY, inspection_models=options.inspection_models, build_models=options.build_models, ) write_baseline_dense_project( - source=baseline_source, pristine=pristine[BASELINE_LABEL], models=options.dense_models + source=baseline_source, + python=baseline.python, + pristine=pristine[BASELINE_LABEL], + models=options.dense_models, ) versions: tuple[InstalledVersion, InstalledVersion] = (baseline, candidate) projects: dict[str, dict[str, Path]] = {} @@ -196,7 +200,9 @@ def _install( sqb = install_published( version=spec, venv_dir=venv_dir, python=python, wait_seconds=wait_seconds ) - return InstalledVersion(label=label, version=installed_version(sqb=sqb), sqb=sqb) + return InstalledVersion( + label=label, version=installed_version(sqb=sqb), sqb=sqb, python=sqb.with_name("python") + ) def _write_evidence(*, path: Path, comparison: ReleaseComparison) -> None: diff --git a/scripts/release_performance/models.py b/scripts/release_performance/models.py index fa3af73d32..fd8bac0454 100644 --- a/scripts/release_performance/models.py +++ b/scripts/release_performance/models.py @@ -22,6 +22,7 @@ class InstalledVersion: label: str version: str sqb: Path + python: Path @dataclass(frozen=True) diff --git a/tests/unit/scripts/release_performance/_helpers/helpers.py b/tests/unit/scripts/release_performance/_helpers/helpers.py index 347c353e3c..6ad21f7280 100644 --- a/tests/unit/scripts/release_performance/_helpers/helpers.py +++ b/tests/unit/scripts/release_performance/_helpers/helpers.py @@ -1,6 +1,7 @@ """Builders for release performance comparison test data.""" import subprocess +import sys from pathlib import Path from scripts.release_performance.models import CommandComparison, CommandSample @@ -15,6 +16,18 @@ def write_pristine_projects(*, root: Path, inspection_models: int, build_models: (root / "pristine" / name / "BASELINE_MARKER").write_text(str(models)) return root / "pristine" """ +BASELINE_ONLY_SYMBOL: str = "BASELINE_ONLY_RELEASE_SYMBOL" +BASELINE_SQLBUILD_GENERATOR: str = f"""from pathlib import Path + +from sqlbuild import {BASELINE_ONLY_SYMBOL} + + +def write_pristine_projects(*, root: Path, inspection_models: int, build_models: int) -> Path: + for name in ("inspection", "build"): + (root / "pristine" / name).mkdir(parents=True) + (root / "pristine" / name / "BASELINE_MARKER").write_text({BASELINE_ONLY_SYMBOL}) + return root / "pristine" +""" FAILING_GENERATOR: str = """def write_pristine_projects(*, root, inspection_models, build_models): raise RuntimeError("baseline generator exploded") """ @@ -25,6 +38,15 @@ def write_dense_compile_project(*, project_dir: Path, model_count: int) -> None: project_dir.mkdir(parents=True) (project_dir / "BASELINE_MARKER").write_text(str(model_count)) """ +BASELINE_SQLBUILD_DENSE_GENERATOR: str = f"""from pathlib import Path + +from sqlbuild import {BASELINE_ONLY_SYMBOL} + + +def write_dense_compile_project(*, project_dir: Path, model_count: int) -> None: + project_dir.mkdir(parents=True) + (project_dir / "BASELINE_MARKER").write_text({BASELINE_ONLY_SYMBOL}) +""" FAILING_DENSE_GENERATOR: str = """def write_dense_compile_project(*, project_dir, model_count): raise RuntimeError("baseline dense generator exploded") """ @@ -80,6 +102,27 @@ def write_baseline_dense_source(*, root: Path, generator: str) -> Path: return root +def write_baseline_environment(*, root: Path) -> Path: + """Create a virtual environment whose sqlbuild defines a symbol this checkout lacks.""" + + _ = subprocess.run( + [sys.executable, "-m", "venv", "--without-pip", str(root)], check=True, capture_output=True + ) + python: Path = root / "bin" / "python" + purelib: str = subprocess.run( + [str(python), "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + package: Path = Path(purelib) / "sqlbuild" + package.mkdir(parents=True) + _ = (package / "__init__.py").write_text( + f"{BASELINE_ONLY_SYMBOL} = {BASELINE_ONLY_SYMBOL!r}\n", encoding="utf-8" + ) + return python + + def tagged_repository(*, root: Path, tag: str) -> Path: """Create a one-commit git repository with a tracked file and the given tag.""" diff --git a/tests/unit/scripts/release_performance/_helpers/test_benchmark.py b/tests/unit/scripts/release_performance/_helpers/test_benchmark.py index 1458f4e231..8bb9bbf53b 100644 --- a/tests/unit/scripts/release_performance/_helpers/test_benchmark.py +++ b/tests/unit/scripts/release_performance/_helpers/test_benchmark.py @@ -18,12 +18,16 @@ ReleaseSourceTestCase, ) from tests.unit.scripts.release_performance._helpers.helpers import ( + BASELINE_ONLY_SYMBOL, + BASELINE_SQLBUILD_DENSE_GENERATOR, + BASELINE_SQLBUILD_GENERATOR, FAILING_DENSE_GENERATOR, FAILING_GENERATOR, MARKER_DENSE_GENERATOR, MARKER_GENERATOR, tagged_repository, write_baseline_dense_source, + write_baseline_environment, write_baseline_source, ) @@ -37,6 +41,12 @@ expected_projects=("build", "inspection"), expected_error_fragments=(), ), + BaselineGeneratorTestCase( + description="the generator imports the baseline environment's sqlbuild, not this one", + generator=BASELINE_SQLBUILD_GENERATOR, + expected_projects=("build", "inspection"), + expected_error_fragments=(), + ), BaselineGeneratorTestCase( description="a failing baseline generator is reported with its stderr", generator=FAILING_GENERATOR, @@ -58,7 +68,11 @@ def test_given_baseline_source_when_generating_then_uses_its_own_generator( try: _ = write_baseline_pristine_projects( - source=source, root=root, inspection_models=30, build_models=10 + source=source, + python=write_baseline_environment(root=tmp_path / "venv"), + root=root, + inspection_models=30, + build_models=10, ) except ReleasePerformanceError as raised: error = str(raised) @@ -79,6 +93,12 @@ def test_given_baseline_source_when_generating_then_uses_its_own_generator( expected_markers=("40",), expected_error_fragments=(), ), + BaselineDenseGeneratorTestCase( + description="the dense generator imports the baseline environment's sqlbuild", + generator=BASELINE_SQLBUILD_DENSE_GENERATOR, + expected_markers=(BASELINE_ONLY_SYMBOL,), + expected_error_fragments=(), + ), BaselineDenseGeneratorTestCase( description="a failing baseline dense generator is reported with its stderr", generator=FAILING_DENSE_GENERATOR, @@ -101,7 +121,12 @@ def test_given_baseline_source_when_generating_dense_project_then_uses_its_own_g error: str = "" try: - write_baseline_dense_project(source=source, pristine=pristine, models=40) + write_baseline_dense_project( + source=source, + python=write_baseline_environment(root=tmp_path / "venv"), + pristine=pristine, + models=40, + ) except ReleasePerformanceError as raised: error = str(raised)