diff --git a/.github/workflows/release-performance.yml b/.github/workflows/release-performance.yml index 87637a9459..a565a03f62 100644 --- a/.github/workflows/release-performance.yml +++ b/.github/workflows/release-performance.yml @@ -22,9 +22,13 @@ on: - scripts/compare_release_performance.py - scripts/release_performance/** - scripts/compile_performance_ratio/_helpers/edit.py + - scripts/compile_performance_ratio/_helpers/measure.py + - scripts/compile_performance_ratio/_helpers/report.py - scripts/compile_performance_ratio/constants.py - scripts/compile_performance_ratio/exceptions.py + - scripts/compile_performance_ratio/models.py - scripts/cold_compile_performance/_helpers/dense_project.py + - scripts/cold_compile_performance/constants.py permissions: contents: read diff --git a/scripts/compile_performance_ratio/_helpers/measure.py b/scripts/compile_performance_ratio/_helpers/measure.py index 45123ec561..9931aefcc4 100644 --- a/scripts/compile_performance_ratio/_helpers/measure.py +++ b/scripts/compile_performance_ratio/_helpers/measure.py @@ -69,15 +69,19 @@ def run_checkout_generator( entry: str, arguments: tuple[str, ...], environment: dict[str, str], + library_paths: tuple[Path, ...], ) -> subprocess.CompletedProcess[str]: - """Run generator code from another checkout's source tree with this `python`.""" + """Run another checkout's generator with `python`, importing `library_paths` before its own.""" return subprocess.run( [str(python), "-c", entry, *arguments], cwd=checkout, capture_output=True, text=True, - env={**environment, PYTHONPATH_KEY: str(checkout)}, + env={ + **environment, + PYTHONPATH_KEY: os.pathsep.join(str(path) for path in (checkout, *library_paths)), + }, check=False, ) @@ -93,6 +97,7 @@ def write_base_benchmark_project( entry=BASE_GENERATOR_ENTRY, arguments=(kind, str(project_dir), str(models)), environment=_compile_environment(), + library_paths=(), ) if completed.returncode != 0: raise CompileComparisonError( diff --git a/scripts/release_performance/_helpers/benchmark.py b/scripts/release_performance/_helpers/benchmark.py index 5812031eb3..c95a70865e 100644 --- a/scripts/release_performance/_helpers/benchmark.py +++ b/scripts/release_performance/_helpers/benchmark.py @@ -14,6 +14,7 @@ from scripts.cold_compile_performance._helpers.dense_project import write_dense_compile_project from scripts.compile_performance_ratio._helpers.edit import apply_one_model_edit from scripts.compile_performance_ratio._helpers.measure import run_checkout_generator +from scripts.release_performance._helpers.versions import library_paths from scripts.release_performance.constants import ( BASELINE_GENERATOR_ENTRY, BUILD_PROJECT, @@ -60,12 +61,13 @@ def write_pristine_projects(*, root: Path, inspection_models: int, build_models: def write_baseline_pristine_projects( - *, source: Path, root: Path, inspection_models: int, build_models: int + *, source: Path, python: Path, root: Path, inspection_models: int, build_models: int ) -> Path: - """Generate the baseline's benchmarks with the baseline source's own generator.""" + """Generate the baseline's benchmarks with its own generator and installed `sqlbuild`.""" _run_baseline_generator( source=source, + python=python, entry=BASELINE_GENERATOR_ENTRY, arguments=(str(root), str(inspection_models), str(build_models)), description="benchmark", @@ -79,11 +81,14 @@ def write_dense_project(*, pristine: Path, models: int) -> None: write_dense_compile_project(project_dir=pristine / DENSE_PROJECT, model_count=models) -def write_baseline_dense_project(*, source: Path, pristine: Path, models: int) -> None: - """Generate the baseline's dense benchmark with the baseline source's own generator.""" +def write_baseline_dense_project( + *, source: Path, python: Path, pristine: Path, models: int +) -> None: + """Generate the baseline's dense benchmark with its own generator and installed `sqlbuild`.""" _run_baseline_generator( source=source, + python=python, entry=DENSE_GENERATOR_ENTRY, arguments=(str(pristine / DENSE_PROJECT), str(models)), description="dense benchmark", @@ -166,7 +171,7 @@ def compare_command( def _run_baseline_generator( - *, source: Path, entry: str, arguments: tuple[str, ...], description: str + *, source: Path, python: Path, entry: str, arguments: tuple[str, ...], description: str ) -> None: completed: subprocess.CompletedProcess[str] = run_checkout_generator( checkout=source, @@ -174,6 +179,7 @@ def _run_baseline_generator( entry=entry, arguments=arguments, environment=_environment(), + library_paths=library_paths(python=python), ) if completed.returncode != 0: raise ReleasePerformanceError( diff --git a/scripts/release_performance/_helpers/versions.py b/scripts/release_performance/_helpers/versions.py index 25b5a8b0bb..2c9ddeeefe 100644 --- a/scripts/release_performance/_helpers/versions.py +++ b/scripts/release_performance/_helpers/versions.py @@ -36,6 +36,10 @@ _DISTRIBUTION_VERSION: re.Pattern[str] = re.compile( rf"^{PACKAGE_NAME}-(\d+\.\d+\.\d+)(?:-.+\.whl|\.tar\.gz)$" ) +_LIBRARY_PATHS_PROBE: str = ( + "import json, sys, sysconfig; print(json.dumps({'version': list(sys.version_info[:2]), " + "'paths': [sysconfig.get_path('purelib'), sysconfig.get_path('platlib')]}))" +) def release_versions(*, index: dict[str, object]) -> tuple[str, ...]: @@ -211,6 +215,27 @@ def installed_version(*, sqb: Path) -> str: return match.group(0) +def library_paths(*, python: Path) -> tuple[Path, ...]: + """Return the package directories of `python`'s environment, which must match this Python.""" + + completed: subprocess.CompletedProcess[str] = subprocess.run( + [str(python), "-c", _LIBRARY_PATHS_PROBE], capture_output=True, text=True, check=False + ) + if completed.returncode != 0: + raise ReleasePerformanceError( + f"Reading the package directories of {python} failed: {_tail(completed.stderr)}" + ) + probe: dict[str, list[object]] = cast(dict[str, list[object]], json.loads(completed.stdout)) + version: tuple[object, ...] = tuple(probe["version"]) + if version != tuple(sys.version_info[:2]): + raise ReleasePerformanceError( + f"{python} runs Python {'.'.join(map(str, version))}, but its packages are imported " + f"by Python {sys.version_info.major}.{sys.version_info.minor}; pass --python " + f"{sys.version_info.major}.{sys.version_info.minor}" + ) + return tuple(dict.fromkeys(Path(str(path)) for path in probe["paths"])) + + def existing_sqb(*, venv_dir: Path) -> Path: """Return the sqb entry point of an existing virtual environment.""" diff --git a/scripts/release_performance/_helpers/workflow.py b/scripts/release_performance/_helpers/workflow.py index 92f7ec7f1d..eac8ba6776 100644 --- a/scripts/release_performance/_helpers/workflow.py +++ b/scripts/release_performance/_helpers/workflow.py @@ -138,12 +138,16 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: ) pristine[BASELINE_LABEL] = write_baseline_pristine_projects( source=baseline_source, + python=baseline.python, root=root / BASELINE_GENERATED_DIRECTORY, inspection_models=options.inspection_models, build_models=options.build_models, ) write_baseline_dense_project( - source=baseline_source, pristine=pristine[BASELINE_LABEL], models=options.dense_models + source=baseline_source, + python=baseline.python, + pristine=pristine[BASELINE_LABEL], + models=options.dense_models, ) versions: tuple[InstalledVersion, InstalledVersion] = (baseline, candidate) projects: dict[str, dict[str, Path]] = {} @@ -196,7 +200,9 @@ def _install( sqb = install_published( version=spec, venv_dir=venv_dir, python=python, wait_seconds=wait_seconds ) - return InstalledVersion(label=label, version=installed_version(sqb=sqb), sqb=sqb) + return InstalledVersion( + label=label, version=installed_version(sqb=sqb), sqb=sqb, python=sqb.with_name("python") + ) def _write_evidence(*, path: Path, comparison: ReleaseComparison) -> None: diff --git a/scripts/release_performance/models.py b/scripts/release_performance/models.py index fa3af73d32..fd8bac0454 100644 --- a/scripts/release_performance/models.py +++ b/scripts/release_performance/models.py @@ -22,6 +22,7 @@ class InstalledVersion: label: str version: str sqb: Path + python: Path @dataclass(frozen=True) diff --git a/tests/unit/scripts/release_performance/_helpers/helpers.py b/tests/unit/scripts/release_performance/_helpers/helpers.py index 347c353e3c..6ad21f7280 100644 --- a/tests/unit/scripts/release_performance/_helpers/helpers.py +++ b/tests/unit/scripts/release_performance/_helpers/helpers.py @@ -1,6 +1,7 @@ """Builders for release performance comparison test data.""" import subprocess +import sys from pathlib import Path from scripts.release_performance.models import CommandComparison, CommandSample @@ -15,6 +16,18 @@ def write_pristine_projects(*, root: Path, inspection_models: int, build_models: (root / "pristine" / name / "BASELINE_MARKER").write_text(str(models)) return root / "pristine" """ +BASELINE_ONLY_SYMBOL: str = "BASELINE_ONLY_RELEASE_SYMBOL" +BASELINE_SQLBUILD_GENERATOR: str = f"""from pathlib import Path + +from sqlbuild import {BASELINE_ONLY_SYMBOL} + + +def write_pristine_projects(*, root: Path, inspection_models: int, build_models: int) -> Path: + for name in ("inspection", "build"): + (root / "pristine" / name).mkdir(parents=True) + (root / "pristine" / name / "BASELINE_MARKER").write_text({BASELINE_ONLY_SYMBOL}) + return root / "pristine" +""" FAILING_GENERATOR: str = """def write_pristine_projects(*, root, inspection_models, build_models): raise RuntimeError("baseline generator exploded") """ @@ -25,6 +38,15 @@ def write_dense_compile_project(*, project_dir: Path, model_count: int) -> None: project_dir.mkdir(parents=True) (project_dir / "BASELINE_MARKER").write_text(str(model_count)) """ +BASELINE_SQLBUILD_DENSE_GENERATOR: str = f"""from pathlib import Path + +from sqlbuild import {BASELINE_ONLY_SYMBOL} + + +def write_dense_compile_project(*, project_dir: Path, model_count: int) -> None: + project_dir.mkdir(parents=True) + (project_dir / "BASELINE_MARKER").write_text({BASELINE_ONLY_SYMBOL}) +""" FAILING_DENSE_GENERATOR: str = """def write_dense_compile_project(*, project_dir, model_count): raise RuntimeError("baseline dense generator exploded") """ @@ -80,6 +102,27 @@ def write_baseline_dense_source(*, root: Path, generator: str) -> Path: return root +def write_baseline_environment(*, root: Path) -> Path: + """Create a virtual environment whose sqlbuild defines a symbol this checkout lacks.""" + + _ = subprocess.run( + [sys.executable, "-m", "venv", "--without-pip", str(root)], check=True, capture_output=True + ) + python: Path = root / "bin" / "python" + purelib: str = subprocess.run( + [str(python), "-c", "import sysconfig; print(sysconfig.get_path('purelib'))"], + check=True, + capture_output=True, + text=True, + ).stdout.strip() + package: Path = Path(purelib) / "sqlbuild" + package.mkdir(parents=True) + _ = (package / "__init__.py").write_text( + f"{BASELINE_ONLY_SYMBOL} = {BASELINE_ONLY_SYMBOL!r}\n", encoding="utf-8" + ) + return python + + def tagged_repository(*, root: Path, tag: str) -> Path: """Create a one-commit git repository with a tracked file and the given tag.""" diff --git a/tests/unit/scripts/release_performance/_helpers/test_benchmark.py b/tests/unit/scripts/release_performance/_helpers/test_benchmark.py index 1458f4e231..8bb9bbf53b 100644 --- a/tests/unit/scripts/release_performance/_helpers/test_benchmark.py +++ b/tests/unit/scripts/release_performance/_helpers/test_benchmark.py @@ -18,12 +18,16 @@ ReleaseSourceTestCase, ) from tests.unit.scripts.release_performance._helpers.helpers import ( + BASELINE_ONLY_SYMBOL, + BASELINE_SQLBUILD_DENSE_GENERATOR, + BASELINE_SQLBUILD_GENERATOR, FAILING_DENSE_GENERATOR, FAILING_GENERATOR, MARKER_DENSE_GENERATOR, MARKER_GENERATOR, tagged_repository, write_baseline_dense_source, + write_baseline_environment, write_baseline_source, ) @@ -37,6 +41,12 @@ expected_projects=("build", "inspection"), expected_error_fragments=(), ), + BaselineGeneratorTestCase( + description="the generator imports the baseline environment's sqlbuild, not this one", + generator=BASELINE_SQLBUILD_GENERATOR, + expected_projects=("build", "inspection"), + expected_error_fragments=(), + ), BaselineGeneratorTestCase( description="a failing baseline generator is reported with its stderr", generator=FAILING_GENERATOR, @@ -58,7 +68,11 @@ def test_given_baseline_source_when_generating_then_uses_its_own_generator( try: _ = write_baseline_pristine_projects( - source=source, root=root, inspection_models=30, build_models=10 + source=source, + python=write_baseline_environment(root=tmp_path / "venv"), + root=root, + inspection_models=30, + build_models=10, ) except ReleasePerformanceError as raised: error = str(raised) @@ -79,6 +93,12 @@ def test_given_baseline_source_when_generating_then_uses_its_own_generator( expected_markers=("40",), expected_error_fragments=(), ), + BaselineDenseGeneratorTestCase( + description="the dense generator imports the baseline environment's sqlbuild", + generator=BASELINE_SQLBUILD_DENSE_GENERATOR, + expected_markers=(BASELINE_ONLY_SYMBOL,), + expected_error_fragments=(), + ), BaselineDenseGeneratorTestCase( description="a failing baseline dense generator is reported with its stderr", generator=FAILING_DENSE_GENERATOR, @@ -101,7 +121,12 @@ def test_given_baseline_source_when_generating_dense_project_then_uses_its_own_g error: str = "" try: - write_baseline_dense_project(source=source, pristine=pristine, models=40) + write_baseline_dense_project( + source=source, + python=write_baseline_environment(root=tmp_path / "venv"), + pristine=pristine, + models=40, + ) except ReleasePerformanceError as raised: error = str(raised)