diff --git a/.github/workflows/release-performance.yml b/.github/workflows/release-performance.yml index ba90a40606..c9ddef80e3 100644 --- a/.github/workflows/release-performance.yml +++ b/.github/workflows/release-performance.yml @@ -81,19 +81,6 @@ jobs: BASELINE: ${{ inputs.baseline }} EVENT_NAME: ${{ github.event_name }} run: | - # A main commit that is not itself a release still carries the latest release's version, - # so it is compared with that release rather than the one before it. A release commit - # bumps the version and is compared with the release before it. - if [[ "${EVENT_NAME}" == "push" && -z "${BASELINE}" ]]; then - project_version() { - python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["project"]["version"])' - } - version="$(project_version < pyproject.toml)" - previous="$(git show HEAD^:pyproject.toml | project_version)" || previous="" - if [[ "${version}" == "${previous}" ]]; then - BASELINE="${version}" - fi - fi wheels=("${RUNNER_TEMP}"/candidate/sqlbuild-*.whl) evidence="${RUNNER_TEMP}/release-performance" mkdir -p "${evidence}" @@ -105,6 +92,20 @@ jobs: if [[ -n "${BASELINE}" ]]; then args+=(--baseline "${BASELINE}") fi + # A main commit that is not itself a release still carries the latest release's version, + # so it is compared with that release once PyPI serves it, and with the newest published + # release before it until then. A release commit bumps the version and is compared with + # the newest published release before it. + if [[ "${EVENT_NAME}" == "push" ]]; then + project_version() { + python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["project"]["version"])' + } + version="$(project_version < pyproject.toml)" + previous="$(git show HEAD^:pyproject.toml | project_version)" || previous="" + if [[ "${version}" == "${previous}" ]]; then + args+=(--same-version-baseline) + fi + fi # The comparison's own exit status, not tee's, decides the outcome. set +e uv run python -m scripts.compare_release_performance "${args[@]}" \ diff --git a/scripts/release_performance/_helpers/versions.py b/scripts/release_performance/_helpers/versions.py index 5d66ce8cbe..946f42440b 100644 --- a/scripts/release_performance/_helpers/versions.py +++ b/scripts/release_performance/_helpers/versions.py @@ -23,7 +23,6 @@ PYPI_REQUEST_TIMEOUT_SECONDS, PYPI_SIMPLE_JSON, PYPI_SIMPLE_URL, - RELEASE_TAG_PATTERN, RELEASE_TAG_PREFIX, STDERR_TAIL_CHARACTERS, UNIVERSAL_WHEEL_MACHINE, @@ -48,16 +47,13 @@ def release_versions(*, index: dict[str, object]) -> tuple[str, ...]: return _index_versions(index=index, include_yanked=False) -def choose_baseline( - *, candidate: str, index: dict[str, object], tags: tuple[str, ...] -) -> str | None: - """Return the highest installable or still-publishing release below the candidate.""" +def choose_baseline(*, candidate: str, index: dict[str, object], same_version: bool) -> str | None: + """Return the highest release on PyPI below the candidate, or its own version when allowed.""" - listed: frozenset[str] = frozenset(_index_versions(index=index, include_yanked=True)) - publishing: tuple[str, ...] = tuple(tag for tag in tags if tag not in listed) - return previous_version( - candidate=candidate, versions=(*release_versions(index=index), *publishing) - ) + published: tuple[str, ...] = release_versions(index=index) + if same_version and candidate in published: + return candidate + return previous_version(candidate=candidate, versions=published) def previous_version(*, candidate: str, versions: tuple[str, ...]) -> str | None: @@ -107,24 +103,6 @@ def fetch_simple_index() -> dict[str, object]: return cast(dict[str, object], json.loads(response.read())) -def tagged_versions(*, repo_dir: Path) -> tuple[str, ...]: - """Return the X.Y.Z versions of the release tags in repo_dir, or none outside a clone.""" - - completed: subprocess.CompletedProcess[str] = subprocess.run( - ["git", "-C", str(repo_dir), "tag", "--list", RELEASE_TAG_PATTERN], - capture_output=True, - text=True, - check=False, - ) - if completed.returncode != 0: - return () - return tuple( - tag.removeprefix("v") - for tag in completed.stdout.split() - if _RELEASE_VERSION.fullmatch(tag.removeprefix("v")) - ) - - def release_source(*, version: str, repo_dir: Path, destination: Path) -> Path: """Extract the source tree of release tag v from repo_dir into destination.""" @@ -147,13 +125,11 @@ def release_source(*, version: str, repo_dir: Path, destination: Path) -> Path: return destination -def resolve_baseline(*, candidate: str, repo_dir: Path) -> str: - """Return the highest published or not-yet-published tagged release below the candidate.""" +def resolve_baseline(*, candidate: str, same_version: bool) -> str: + """Return the baseline release that PyPI actually serves for the candidate.""" baseline: str | None = choose_baseline( - candidate=candidate, - index=fetch_simple_index(), - tags=tagged_versions(repo_dir=repo_dir), + candidate=candidate, index=fetch_simple_index(), same_version=same_version ) if baseline is None: raise ReleasePerformanceError(f"No published sqlbuild release precedes {candidate}") diff --git a/scripts/release_performance/_helpers/workflow.py b/scripts/release_performance/_helpers/workflow.py index eac8ba6776..b05be29949 100644 --- a/scripts/release_performance/_helpers/workflow.py +++ b/scripts/release_performance/_helpers/workflow.py @@ -97,7 +97,7 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison: wait_seconds=0.0, ) baseline_spec: str = options.baseline or resolve_baseline( - candidate=candidate.version, repo_dir=REPO_ROOT + candidate=candidate.version, same_version=options.same_version_baseline ) print(f"Installing baseline {baseline_spec}", file=sys.stderr) baseline: InstalledVersion = _install( diff --git a/scripts/release_performance/constants.py b/scripts/release_performance/constants.py index 9efc600af8..6656111a0a 100644 --- a/scripts/release_performance/constants.py +++ b/scripts/release_performance/constants.py @@ -30,7 +30,6 @@ PYPI_REQUEST_TIMEOUT_SECONDS: float = 60.0 BASELINE_PUBLICATION_WAIT_SECONDS: float = 1_800.0 BASELINE_PUBLICATION_POLL_SECONDS: float = 30.0 -RELEASE_TAG_PATTERN: str = "v*" WHEEL_PLATFORM_FAMILIES: dict[str, str] = {"linux": "linux", "darwin": "macosx", "win32": "win"} WHEEL_MACHINE_ALIASES: dict[str, str] = {"amd64": "x86_64", "arm64": "aarch64"} UNIVERSAL_WHEEL_MACHINE: str = "universal2" diff --git a/scripts/release_performance/main/compare.py b/scripts/release_performance/main/compare.py index 7592549490..9ac280a1eb 100644 --- a/scripts/release_performance/main/compare.py +++ b/scripts/release_performance/main/compare.py @@ -34,7 +34,15 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions: parser.add_argument( "--baseline", default=None, - help="Defaults to the highest published or release-tagged version below the candidate.", + help="Defaults to the highest version on PyPI below the candidate.", + ) + parser.add_argument( + "--same-version-baseline", + action="store_true", + help=( + "The candidate is an unreleased commit carrying the latest release's version, so " + "that release is the baseline once it is on PyPI." + ), ) parser.add_argument("--runs", type=int, default=DEFAULT_RUNS) parser.add_argument("--python", default=DEFAULT_PYTHON) @@ -66,4 +74,5 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions: work_dir=args.work_dir, output=args.output, baseline_source=args.baseline_source, + same_version_baseline=args.same_version_baseline, ) diff --git a/scripts/release_performance/models.py b/scripts/release_performance/models.py index 7d1ae2a418..dff69c26da 100644 --- a/scripts/release_performance/models.py +++ b/scripts/release_performance/models.py @@ -94,3 +94,4 @@ class ComparisonOptions: work_dir: Path | None output: Path | None baseline_source: Path | None = None + same_version_baseline: bool = False diff --git a/tests/unit/scripts/release_performance/_helpers/_test_types.py b/tests/unit/scripts/release_performance/_helpers/_test_types.py index e9128dfd49..6a4994d5ed 100644 --- a/tests/unit/scripts/release_performance/_helpers/_test_types.py +++ b/tests/unit/scripts/release_performance/_helpers/_test_types.py @@ -73,7 +73,7 @@ class ChooseBaselineTestCase: description: str candidate: str files: tuple[dict[str, object], ...] - tags: tuple[str, ...] + same_version: bool expected_baseline: str | None diff --git a/tests/unit/scripts/release_performance/_helpers/test_versions.py b/tests/unit/scripts/release_performance/_helpers/test_versions.py index 0aae6d9e73..77b488f1c1 100644 --- a/tests/unit/scripts/release_performance/_helpers/test_versions.py +++ b/tests/unit/scripts/release_performance/_helpers/test_versions.py @@ -134,47 +134,62 @@ def test_given_release_files_when_selecting_wheel_urls_then_matches_runner_platf "test_case", ( ChooseBaselineTestCase( - description="a tagged release that is still publishing is the baseline", + description="a main commit whose own release is not on PyPI uses the newest one that is", candidate="0.126.3", - files=(release_file(filename="sqlbuild-0.126.1.tar.gz"),), - tags=("0.126.1", "0.126.2"), + files=( + release_file(filename="sqlbuild-0.126.1.tar.gz"), + release_file(filename="sqlbuild-0.126.2.tar.gz"), + ), + same_version=True, expected_baseline="0.126.2", ), ChooseBaselineTestCase( - description="a yanked release is skipped even though its tag exists", - candidate="0.126.3", + description="a main commit uses its own release once PyPI serves it", + candidate="0.126.2", files=( release_file(filename="sqlbuild-0.126.1.tar.gz"), - {"filename": "sqlbuild-0.126.2.tar.gz", "yanked": "broken build", "url": ""}, + release_file(filename="sqlbuild-0.126.2.tar.gz"), + ), + same_version=True, + expected_baseline="0.126.2", + ), + ChooseBaselineTestCase( + description="a release commit never compares with itself after it publishes", + candidate="0.126.2", + files=( + release_file(filename="sqlbuild-0.126.1.tar.gz"), + release_file(filename="sqlbuild-0.126.2.tar.gz"), ), - tags=("0.126.1", "0.126.2"), + same_version=False, expected_baseline="0.126.1", ), ChooseBaselineTestCase( - description="a published release without a local tag is still a baseline", + description="a yanked release is skipped", candidate="0.126.3", files=( release_file(filename="sqlbuild-0.126.1.tar.gz"), - release_file(filename="sqlbuild-0.126.2.tar.gz"), + {"filename": "sqlbuild-0.126.2.tar.gz", "yanked": "broken build", "url": ""}, ), - tags=(), - expected_baseline="0.126.2", + same_version=False, + expected_baseline="0.126.1", ), ChooseBaselineTestCase( description="nothing precedes the first release", candidate="0.1.0", files=(release_file(filename="sqlbuild-0.1.0.tar.gz"),), - tags=("0.1.0",), + same_version=False, expected_baseline=None, ), ), ids=lambda case: case.description, ) -def test_given_index_and_release_tags_when_choosing_baseline_then_skips_yanked_releases( +def test_given_published_releases_when_choosing_baseline_then_uses_one_pypi_serves( test_case: ChooseBaselineTestCase, ) -> None: baseline: str | None = choose_baseline( - candidate=test_case.candidate, index={"files": list(test_case.files)}, tags=test_case.tags + candidate=test_case.candidate, + index={"files": list(test_case.files)}, + same_version=test_case.same_version, ) assert baseline == test_case.expected_baseline