ci: compare release performance with a release PyPI serves

.github/workflows/release-performance.yml

@@ -81,19 +81,6 @@ jobs:
           BASELINE: ${{ inputs.baseline }}
           EVENT_NAME: ${{ github.event_name }}
         run: |
-          # A main commit that is not itself a release still carries the latest release's version,
-          # so it is compared with that release rather than the one before it. A release commit
-          # bumps the version and is compared with the release before it.
-          if [[ "${EVENT_NAME}" == "push" && -z "${BASELINE}" ]]; then
-            project_version() {
-              python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["project"]["version"])'
-            }
-            version="$(project_version < pyproject.toml)"
-            previous="$(git show HEAD^:pyproject.toml | project_version)" || previous=""
-            if [[ "${version}" == "${previous}" ]]; then
-              BASELINE="${version}"
-            fi
-          fi
           wheels=("${RUNNER_TEMP}"/candidate/sqlbuild-*.whl)
           evidence="${RUNNER_TEMP}/release-performance"
           mkdir -p "${evidence}"
@@ -105,6 +92,20 @@ jobs:
           if [[ -n "${BASELINE}" ]]; then
             args+=(--baseline "${BASELINE}")
           fi
+          # A main commit that is not itself a release still carries the latest release's version,
+          # so it is compared with that release once PyPI serves it, and with the newest published
+          # release before it until then. A release commit bumps the version and is compared with
+          # the newest published release before it.
+          if [[ "${EVENT_NAME}" == "push" ]]; then
+            project_version() {
+              python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["project"]["version"])'
+            }
+            version="$(project_version < pyproject.toml)"
+            previous="$(git show HEAD^:pyproject.toml | project_version)" || previous=""
+            if [[ "${version}" == "${previous}" ]]; then
+              args+=(--same-version-baseline)
+            fi
+          fi
           # The comparison's own exit status, not tee's, decides the outcome.
           set +e
           uv run python -m scripts.compare_release_performance "${args[@]}" \

scripts/release_performance/_helpers/versions.py

@@ -23,7 +23,6 @@
     PYPI_REQUEST_TIMEOUT_SECONDS,
     PYPI_SIMPLE_JSON,
     PYPI_SIMPLE_URL,
-    RELEASE_TAG_PATTERN,
     RELEASE_TAG_PREFIX,
     STDERR_TAIL_CHARACTERS,
     UNIVERSAL_WHEEL_MACHINE,
@@ -48,16 +47,13 @@ def release_versions(*, index: dict[str, object]) -> tuple[str, ...]:
     return _index_versions(index=index, include_yanked=False)
 
 
-def choose_baseline(
-    *, candidate: str, index: dict[str, object], tags: tuple[str, ...]
-) -> str | None:
-    """Return the highest installable or still-publishing release below the candidate."""
+def choose_baseline(*, candidate: str, index: dict[str, object], same_version: bool) -> str | None:
+    """Return the highest release on PyPI below the candidate, or its own version when allowed."""
 
-    listed: frozenset[str] = frozenset(_index_versions(index=index, include_yanked=True))
-    publishing: tuple[str, ...] = tuple(tag for tag in tags if tag not in listed)
-    return previous_version(
-        candidate=candidate, versions=(*release_versions(index=index), *publishing)
-    )
+    published: tuple[str, ...] = release_versions(index=index)
+    if same_version and candidate in published:
+        return candidate
+    return previous_version(candidate=candidate, versions=published)
 
 
 def previous_version(*, candidate: str, versions: tuple[str, ...]) -> str | None:
@@ -107,24 +103,6 @@ def fetch_simple_index() -> dict[str, object]:
         return cast(dict[str, object], json.loads(response.read()))
 
 
-def tagged_versions(*, repo_dir: Path) -> tuple[str, ...]:
-    """Return the X.Y.Z versions of the release tags in repo_dir, or none outside a clone."""
-
-    completed: subprocess.CompletedProcess[str] = subprocess.run(
-        ["git", "-C", str(repo_dir), "tag", "--list", RELEASE_TAG_PATTERN],
-        capture_output=True,
-        text=True,
-        check=False,
-    )
-    if completed.returncode != 0:
-        return ()
-    return tuple(
-        tag.removeprefix("v")
-        for tag in completed.stdout.split()
-        if _RELEASE_VERSION.fullmatch(tag.removeprefix("v"))
-    )
-
-
 def release_source(*, version: str, repo_dir: Path, destination: Path) -> Path:
     """Extract the source tree of release tag v<version> from repo_dir into destination."""
 
@@ -147,13 +125,11 @@ def release_source(*, version: str, repo_dir: Path, destination: Path) -> Path:
     return destination
 
 
-def resolve_baseline(*, candidate: str, repo_dir: Path) -> str:
-    """Return the highest published or not-yet-published tagged release below the candidate."""
+def resolve_baseline(*, candidate: str, same_version: bool) -> str:
+    """Return the baseline release that PyPI actually serves for the candidate."""
 
     baseline: str | None = choose_baseline(
-        candidate=candidate,
-        index=fetch_simple_index(),
-        tags=tagged_versions(repo_dir=repo_dir),
+        candidate=candidate, index=fetch_simple_index(), same_version=same_version
     )
     if baseline is None:
         raise ReleasePerformanceError(f"No published sqlbuild release precedes {candidate}")

scripts/release_performance/_helpers/workflow.py

@@ -97,7 +97,7 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison:
         wait_seconds=0.0,
     )
     baseline_spec: str = options.baseline or resolve_baseline(
-        candidate=candidate.version, repo_dir=REPO_ROOT
+        candidate=candidate.version, same_version=options.same_version_baseline
     )
     print(f"Installing baseline {baseline_spec}", file=sys.stderr)
     baseline: InstalledVersion = _install(

scripts/release_performance/constants.py

@@ -30,7 +30,6 @@
 PYPI_REQUEST_TIMEOUT_SECONDS: float = 60.0
 BASELINE_PUBLICATION_WAIT_SECONDS: float = 1_800.0
 BASELINE_PUBLICATION_POLL_SECONDS: float = 30.0
-RELEASE_TAG_PATTERN: str = "v*"
 WHEEL_PLATFORM_FAMILIES: dict[str, str] = {"linux": "linux", "darwin": "macosx", "win32": "win"}
 WHEEL_MACHINE_ALIASES: dict[str, str] = {"amd64": "x86_64", "arm64": "aarch64"}
 UNIVERSAL_WHEEL_MACHINE: str = "universal2"

scripts/release_performance/main/compare.py

@@ -34,7 +34,15 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions:
     parser.add_argument(
         "--baseline",
         default=None,
-        help="Defaults to the highest published or release-tagged version below the candidate.",
+        help="Defaults to the highest version on PyPI below the candidate.",
+    )
+    parser.add_argument(
+        "--same-version-baseline",
+        action="store_true",
+        help=(
+            "The candidate is an unreleased commit carrying the latest release's version, so "
+            "that release is the baseline once it is on PyPI."
+        ),
     )
     parser.add_argument("--runs", type=int, default=DEFAULT_RUNS)
     parser.add_argument("--python", default=DEFAULT_PYTHON)
@@ -66,4 +74,5 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions:
         work_dir=args.work_dir,
         output=args.output,
         baseline_source=args.baseline_source,
+        same_version_baseline=args.same_version_baseline,
     )

scripts/release_performance/models.py

@@ -94,3 +94,4 @@ class ComparisonOptions:
     work_dir: Path | None
     output: Path | None
     baseline_source: Path | None = None
+    same_version_baseline: bool = False

tests/unit/scripts/release_performance/_helpers/_test_types.py

@@ -73,7 +73,7 @@ class ChooseBaselineTestCase:
     description: str
     candidate: str
     files: tuple[dict[str, object], ...]
-    tags: tuple[str, ...]
+    same_version: bool
     expected_baseline: str | None
 
 

tests/unit/scripts/release_performance/_helpers/test_versions.py

@@ -134,47 +134,62 @@ def test_given_release_files_when_selecting_wheel_urls_then_matches_runner_platf
     "test_case",
     (
         ChooseBaselineTestCase(
-            description="a tagged release that is still publishing is the baseline",
+            description="a main commit whose own release is not on PyPI uses the newest one that is",
             candidate="0.126.3",
-            files=(release_file(filename="sqlbuild-0.126.1.tar.gz"),),
-            tags=("0.126.1", "0.126.2"),
+            files=(
+                release_file(filename="sqlbuild-0.126.1.tar.gz"),
+                release_file(filename="sqlbuild-0.126.2.tar.gz"),
+            ),
+            same_version=True,
             expected_baseline="0.126.2",
         ),
         ChooseBaselineTestCase(
-            description="a yanked release is skipped even though its tag exists",
-            candidate="0.126.3",
+            description="a main commit uses its own release once PyPI serves it",
+            candidate="0.126.2",
             files=(
                 release_file(filename="sqlbuild-0.126.1.tar.gz"),
-                {"filename": "sqlbuild-0.126.2.tar.gz", "yanked": "broken build", "url": ""},
+                release_file(filename="sqlbuild-0.126.2.tar.gz"),
+            ),
+            same_version=True,
+            expected_baseline="0.126.2",
+        ),
+        ChooseBaselineTestCase(
+            description="a release commit never compares with itself after it publishes",
+            candidate="0.126.2",
+            files=(
+                release_file(filename="sqlbuild-0.126.1.tar.gz"),
+                release_file(filename="sqlbuild-0.126.2.tar.gz"),
             ),
-            tags=("0.126.1", "0.126.2"),
+            same_version=False,
             expected_baseline="0.126.1",
         ),
         ChooseBaselineTestCase(
-            description="a published release without a local tag is still a baseline",
+            description="a yanked release is skipped",
             candidate="0.126.3",
             files=(
                 release_file(filename="sqlbuild-0.126.1.tar.gz"),
-                release_file(filename="sqlbuild-0.126.2.tar.gz"),
+                {"filename": "sqlbuild-0.126.2.tar.gz", "yanked": "broken build", "url": ""},
             ),
-            tags=(),
-            expected_baseline="0.126.2",
+            same_version=False,
+            expected_baseline="0.126.1",
         ),
         ChooseBaselineTestCase(
             description="nothing precedes the first release",
             candidate="0.1.0",
             files=(release_file(filename="sqlbuild-0.1.0.tar.gz"),),
-            tags=("0.1.0",),
+            same_version=False,
             expected_baseline=None,
         ),
     ),
     ids=lambda case: case.description,
 )
-def test_given_index_and_release_tags_when_choosing_baseline_then_skips_yanked_releases(
+def test_given_published_releases_when_choosing_baseline_then_uses_one_pypi_serves(
     test_case: ChooseBaselineTestCase,
 ) -> None:
     baseline: str | None = choose_baseline(
-        candidate=test_case.candidate, index={"files": list(test_case.files)}, tags=test_case.tags
+        candidate=test_case.candidate,
+        index={"files": list(test_case.files)},
+        same_version=test_case.same_version,
     )
 
     assert baseline == test_case.expected_baseline