From fd617dcfcd1fc7b1607616ea555f2fbfd30fbb43 Mon Sep 17 00:00:00 2001 From: Kevin Longe <273747162+kvlonge@users.noreply.github.com> Date: Thu, 5 Nov 2026 23:40:05 +0000 Subject: [PATCH] ci: compare release performance with a release PyPI serves Main commits carrying a version that is tagged but not yet on PyPI made the job wait for that release and then fail without comparing anything. The baseline is now the newest release PyPI actually serves: a main commit uses its own release once published and the newest earlier one until then, and a release commit never compares with itself. --- .github/workflows/release-performance.yml | 27 ++++++------ .../release_performance/_helpers/versions.py | 42 ++++-------------- .../release_performance/_helpers/workflow.py | 2 +- scripts/release_performance/constants.py | 1 - scripts/release_performance/main/compare.py | 11 ++++- scripts/release_performance/models.py | 1 + .../_helpers/_test_types.py | 2 +- .../_helpers/test_versions.py | 43 +++++++++++++------ 8 files changed, 65 insertions(+), 64 deletions(-) diff --git a/.github/workflows/release-performance.yml b/.github/workflows/release-performance.yml index ba90a40606..c9ddef80e3 100644 --- a/.github/workflows/release-performance.yml +++ b/.github/workflows/release-performance.yml @@ -81,19 +81,6 @@ jobs: BASELINE: ${{ inputs.baseline }} EVENT_NAME: ${{ github.event_name }} run: | - # A main commit that is not itself a release still carries the latest release's version, - # so it is compared with that release rather than the one before it. A release commit - # bumps the version and is compared with the release before it. - if [[ "${EVENT_NAME}" == "push" && -z "${BASELINE}" ]]; then - project_version() { - python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["project"]["version"])' - } - version="$(project_version < pyproject.toml)" - previous="$(git show HEAD^:pyproject.toml | project_version)" || previous="" - if [[ "${version}" == "${previous}" ]]; then - BASELINE="${version}" - fi - fi wheels=("${RUNNER_TEMP}"/candidate/sqlbuild-*.whl) evidence="${RUNNER_TEMP}/release-performance" mkdir -p "${evidence}" @@ -105,6 +92,20 @@ jobs: if [[ -n "${BASELINE}" ]]; then args+=(--baseline "${BASELINE}") fi + # A main commit that is not itself a release still carries the latest release's version, + # so it is compared with that release once PyPI serves it, and with the newest published + # release before it until then. A release commit bumps the version and is compared with + # the newest published release before it. + if [[ "${EVENT_NAME}" == "push" ]]; then + project_version() { + python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["project"]["version"])' + } + version="$(project_version < pyproject.toml)" + previous="$(git show HEAD^:pyproject.toml | project_version)" || previous="" + if [[ "${version}" == "${previous}" ]]; then + args+=(--same-version-baseline) + fi + fi # The comparison's own exit status, not tee's, decides the outcome. set +e uv run python -m scripts.compare_release_performance "${args[@]}" \ diff --git a/scripts/release_performance/_helpers/versions.py b/scripts/release_performance/_helpers/versions.py index 5d66ce8cbe..946f42440b 100644 --- a/scripts/release_performance/_helpers/versions.py +++ b/scripts/release_performance/_helpers/versions.py @@ -23,7 +23,6 @@ PYPI_REQUEST_TIMEOUT_SECONDS, PYPI_SIMPLE_JSON, PYPI_SIMPLE_URL, - RELEASE_TAG_PATTERN, RELEASE_TAG_PREFIX, STDERR_TAIL_CHARACTERS, UNIVERSAL_WHEEL_MACHINE, @@ -48,16 +47,13 @@ def release_versions(*, index: dict[str, object]) -> tuple[str, ...]: return _index_versions(index=index, include_yanked=False) -def choose_baseline( - *, candidate: str, index: dict[str, object], tags: tuple[str, ...] -) -> str | None: - """Return the highest installable or still-publishing release below the candidate.""" +def choose_baseline(*, candidate: str, index: dict[str, object], same_version: bool) -> str | None: + """Return the highest release on PyPI below the candidate, or its own version when allowed.""" - listed: frozenset[str] = frozenset(_index_versions(index=index, include_yanked=True)) - publishing: tuple[str, ...] = tuple(tag for tag in tags if tag not in listed) - return previous_version( - candidate=candidate, versions=(*release_versions(index=index), *publishing) - ) + published: tuple[str, ...] = release_versions(index=index) + if same_version and candidate in published: + return candidate + return previous_version(candidate=candidate, versions=published) def previous_version(*, candidate: str, versions: tuple[str, ...]) -> str | None: @@ -107,24 +103,6 @@ def fetch_simple_index() -> dict[str, object]: return cast(dict[str, object], json.loads(response.read())) -def tagged_versions(*, repo_dir: Path) -> tuple[str, ...]: - """Return the X.Y.Z versions of the release tags in repo_dir, or none outside a clone.""" - - completed: subprocess.CompletedProcess[str] = subprocess.run( - ["git", "-C", str(repo_dir), "tag", "--list", RELEASE_TAG_PATTERN], - capture_output=True, - text=True, - check=False, - ) - if completed.returncode != 0: - return () - return tuple( - tag.removeprefix("v") - for tag in completed.stdout.split() - if _RELEASE_VERSION.fullmatch(tag.removeprefix("v")) - ) - - def release_source(*, version: str, repo_dir: Path, destination: Path) -> Path: """Extract the source tree of release tag v from repo_dir into destination.""" @@ -147,13 +125,11 @@ def release_source(*, version: str, repo_dir: Path, destination: Path) -> Path: return destination -def resolve_baseline(*, candidate: str, repo_dir: Path) -> str: - """Return the highest published or not-yet-published tagged release below the candidate.""" +def resolve_baseline(*, candidate: str, same_version: bool) -> str: + """Return the baseline release that PyPI actually serves for the candidate.""" baseline: str | None = choose_baseline( - candidate=candidate, - index=fetch_simple_index(), - tags=tagged_versions(repo_dir=repo_dir), + candidate=candidate, index=fetch_simple_index(), same_version=same_version ) if baseline is None: raise ReleasePerformanceError(f"No published sqlbuild release precedes {candidate}") diff --git a/scripts/release_performance/_helpers/workflow.py b/scripts/release_performance/_helpers/workflow.py index eac8ba6776..b05be29949 100644 --- a/scripts/release_performance/_helpers/workflow.py +++ b/scripts/release_performance/_helpers/workflow.py @@ -97,7 +97,7 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: wait_seconds=0.0, ) baseline_spec: str = options.baseline or resolve_baseline( - candidate=candidate.version, repo_dir=REPO_ROOT + candidate=candidate.version, same_version=options.same_version_baseline ) print(f"Installing baseline {baseline_spec}", file=sys.stderr) baseline: InstalledVersion = _install( diff --git a/scripts/release_performance/constants.py b/scripts/release_performance/constants.py index 9efc600af8..6656111a0a 100644 --- a/scripts/release_performance/constants.py +++ b/scripts/release_performance/constants.py @@ -30,7 +30,6 @@ PYPI_REQUEST_TIMEOUT_SECONDS: float = 60.0 BASELINE_PUBLICATION_WAIT_SECONDS: float = 1_800.0 BASELINE_PUBLICATION_POLL_SECONDS: float = 30.0 -RELEASE_TAG_PATTERN: str = "v*" WHEEL_PLATFORM_FAMILIES: dict[str, str] = {"linux": "linux", "darwin": "macosx", "win32": "win"} WHEEL_MACHINE_ALIASES: dict[str, str] = {"amd64": "x86_64", "arm64": "aarch64"} UNIVERSAL_WHEEL_MACHINE: str = "universal2" diff --git a/scripts/release_performance/main/compare.py b/scripts/release_performance/main/compare.py index 7592549490..9ac280a1eb 100644 --- a/scripts/release_performance/main/compare.py +++ b/scripts/release_performance/main/compare.py @@ -34,7 +34,15 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions: parser.add_argument( "--baseline", default=None, - help="Defaults to the highest published or release-tagged version below the candidate.", + help="Defaults to the highest version on PyPI below the candidate.", + ) + parser.add_argument( + "--same-version-baseline", + action="store_true", + help=( + "The candidate is an unreleased commit carrying the latest release's version, so " + "that release is the baseline once it is on PyPI." + ), ) parser.add_argument("--runs", type=int, default=DEFAULT_RUNS) parser.add_argument("--python", default=DEFAULT_PYTHON) @@ -66,4 +74,5 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions: work_dir=args.work_dir, output=args.output, baseline_source=args.baseline_source, + same_version_baseline=args.same_version_baseline, ) diff --git a/scripts/release_performance/models.py b/scripts/release_performance/models.py index 7d1ae2a418..dff69c26da 100644 --- a/scripts/release_performance/models.py +++ b/scripts/release_performance/models.py @@ -94,3 +94,4 @@ class ComparisonOptions: work_dir: Path | None output: Path | None baseline_source: Path | None = None + same_version_baseline: bool = False diff --git a/tests/unit/scripts/release_performance/_helpers/_test_types.py b/tests/unit/scripts/release_performance/_helpers/_test_types.py index e9128dfd49..6a4994d5ed 100644 --- a/tests/unit/scripts/release_performance/_helpers/_test_types.py +++ b/tests/unit/scripts/release_performance/_helpers/_test_types.py @@ -73,7 +73,7 @@ class ChooseBaselineTestCase: description: str candidate: str files: tuple[dict[str, object], ...] - tags: tuple[str, ...] + same_version: bool expected_baseline: str | None diff --git a/tests/unit/scripts/release_performance/_helpers/test_versions.py b/tests/unit/scripts/release_performance/_helpers/test_versions.py index 0aae6d9e73..77b488f1c1 100644 --- a/tests/unit/scripts/release_performance/_helpers/test_versions.py +++ b/tests/unit/scripts/release_performance/_helpers/test_versions.py @@ -134,47 +134,62 @@ def test_given_release_files_when_selecting_wheel_urls_then_matches_runner_platf "test_case", ( ChooseBaselineTestCase( - description="a tagged release that is still publishing is the baseline", + description="a main commit whose own release is not on PyPI uses the newest one that is", candidate="0.126.3", - files=(release_file(filename="sqlbuild-0.126.1.tar.gz"),), - tags=("0.126.1", "0.126.2"), + files=( + release_file(filename="sqlbuild-0.126.1.tar.gz"), + release_file(filename="sqlbuild-0.126.2.tar.gz"), + ), + same_version=True, expected_baseline="0.126.2", ), ChooseBaselineTestCase( - description="a yanked release is skipped even though its tag exists", - candidate="0.126.3", + description="a main commit uses its own release once PyPI serves it", + candidate="0.126.2", files=( release_file(filename="sqlbuild-0.126.1.tar.gz"), - {"filename": "sqlbuild-0.126.2.tar.gz", "yanked": "broken build", "url": ""}, + release_file(filename="sqlbuild-0.126.2.tar.gz"), + ), + same_version=True, + expected_baseline="0.126.2", + ), + ChooseBaselineTestCase( + description="a release commit never compares with itself after it publishes", + candidate="0.126.2", + files=( + release_file(filename="sqlbuild-0.126.1.tar.gz"), + release_file(filename="sqlbuild-0.126.2.tar.gz"), ), - tags=("0.126.1", "0.126.2"), + same_version=False, expected_baseline="0.126.1", ), ChooseBaselineTestCase( - description="a published release without a local tag is still a baseline", + description="a yanked release is skipped", candidate="0.126.3", files=( release_file(filename="sqlbuild-0.126.1.tar.gz"), - release_file(filename="sqlbuild-0.126.2.tar.gz"), + {"filename": "sqlbuild-0.126.2.tar.gz", "yanked": "broken build", "url": ""}, ), - tags=(), - expected_baseline="0.126.2", + same_version=False, + expected_baseline="0.126.1", ), ChooseBaselineTestCase( description="nothing precedes the first release", candidate="0.1.0", files=(release_file(filename="sqlbuild-0.1.0.tar.gz"),), - tags=("0.1.0",), + same_version=False, expected_baseline=None, ), ), ids=lambda case: case.description, ) -def test_given_index_and_release_tags_when_choosing_baseline_then_skips_yanked_releases( +def test_given_published_releases_when_choosing_baseline_then_uses_one_pypi_serves( test_case: ChooseBaselineTestCase, ) -> None: baseline: str | None = choose_baseline( - candidate=test_case.candidate, index={"files": list(test_case.files)}, tags=test_case.tags + candidate=test_case.candidate, + index={"files": list(test_case.files)}, + same_version=test_case.same_version, ) assert baseline == test_case.expected_baseline