Skip to content

Commit 7a2c531

Browse files
kvlongekvlonge
authored
ci: compare release performance with a release PyPI serves (#1314)
Release Performance now compares against the newest release PyPI actually serves. A main commit uses its own release once published and the newest earlier one until then; a release commit never compares with itself. A missing or uninstallable baseline still fails the job.
1 parent 9082be1 commit 7a2c531

8 files changed

Lines changed: 65 additions & 64 deletions

File tree

‎.github/workflows/release-performance.yml‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -81,19 +81,6 @@ jobs:
8181
BASELINE: ${{ inputs.baseline }}
8282
EVENT_NAME: ${{ github.event_name }}
8383
run: |
84-
# A main commit that is not itself a release still carries the latest release's version,
85-
# so it is compared with that release rather than the one before it. A release commit
86-
# bumps the version and is compared with the release before it.
87-
if [[ "${EVENT_NAME}" == "push" && -z "${BASELINE}" ]]; then
88-
project_version() {
89-
python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["project"]["version"])'
90-
}
91-
version="$(project_version < pyproject.toml)"
92-
previous="$(git show HEAD^:pyproject.toml | project_version)" || previous=""
93-
if [[ "${version}" == "${previous}" ]]; then
94-
BASELINE="${version}"
95-
fi
96-
fi
9784
wheels=("${RUNNER_TEMP}"/candidate/sqlbuild-*.whl)
9885
evidence="${RUNNER_TEMP}/release-performance"
9986
mkdir -p "${evidence}"
@@ -105,6 +92,20 @@ jobs:
10592
if [[ -n "${BASELINE}" ]]; then
10693
args+=(--baseline "${BASELINE}")
10794
fi
95+
# A main commit that is not itself a release still carries the latest release's version,
96+
# so it is compared with that release once PyPI serves it, and with the newest published
97+
# release before it until then. A release commit bumps the version and is compared with
98+
# the newest published release before it.
99+
if [[ "${EVENT_NAME}" == "push" ]]; then
100+
project_version() {
101+
python3 -c 'import sys, tomllib; print(tomllib.loads(sys.stdin.read())["project"]["version"])'
102+
}
103+
version="$(project_version < pyproject.toml)"
104+
previous="$(git show HEAD^:pyproject.toml | project_version)" || previous=""
105+
if [[ "${version}" == "${previous}" ]]; then
106+
args+=(--same-version-baseline)
107+
fi
108+
fi
108109
# The comparison's own exit status, not tee's, decides the outcome.
109110
set +e
110111
uv run python -m scripts.compare_release_performance "${args[@]}" \

‎scripts/release_performance/_helpers/versions.py‎

Lines changed: 9 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,6 @@
2323
PYPI_REQUEST_TIMEOUT_SECONDS,
2424
PYPI_SIMPLE_JSON,
2525
PYPI_SIMPLE_URL,
26-
RELEASE_TAG_PATTERN,
2726
RELEASE_TAG_PREFIX,
2827
STDERR_TAIL_CHARACTERS,
2928
UNIVERSAL_WHEEL_MACHINE,
@@ -48,16 +47,13 @@ def release_versions(*, index: dict[str, object]) -> tuple[str, ...]:
4847
return _index_versions(index=index, include_yanked=False)
4948

5049

51-
def choose_baseline(
52-
*, candidate: str, index: dict[str, object], tags: tuple[str, ...]
53-
) -> str | None:
54-
"""Return the highest installable or still-publishing release below the candidate."""
50+
def choose_baseline(*, candidate: str, index: dict[str, object], same_version: bool) -> str | None:
51+
"""Return the highest release on PyPI below the candidate, or its own version when allowed."""
5552

56-
listed: frozenset[str] = frozenset(_index_versions(index=index, include_yanked=True))
57-
publishing: tuple[str, ...] = tuple(tag for tag in tags if tag not in listed)
58-
return previous_version(
59-
candidate=candidate, versions=(*release_versions(index=index), *publishing)
60-
)
53+
published: tuple[str, ...] = release_versions(index=index)
54+
if same_version and candidate in published:
55+
return candidate
56+
return previous_version(candidate=candidate, versions=published)
6157

6258

6359
def previous_version(*, candidate: str, versions: tuple[str, ...]) -> str | None:
@@ -107,24 +103,6 @@ def fetch_simple_index() -> dict[str, object]:
107103
return cast(dict[str, object], json.loads(response.read()))
108104

109105

110-
def tagged_versions(*, repo_dir: Path) -> tuple[str, ...]:
111-
"""Return the X.Y.Z versions of the release tags in repo_dir, or none outside a clone."""
112-
113-
completed: subprocess.CompletedProcess[str] = subprocess.run(
114-
["git", "-C", str(repo_dir), "tag", "--list", RELEASE_TAG_PATTERN],
115-
capture_output=True,
116-
text=True,
117-
check=False,
118-
)
119-
if completed.returncode != 0:
120-
return ()
121-
return tuple(
122-
tag.removeprefix("v")
123-
for tag in completed.stdout.split()
124-
if _RELEASE_VERSION.fullmatch(tag.removeprefix("v"))
125-
)
126-
127-
128106
def release_source(*, version: str, repo_dir: Path, destination: Path) -> Path:
129107
"""Extract the source tree of release tag v<version> from repo_dir into destination."""
130108

@@ -147,13 +125,11 @@ def release_source(*, version: str, repo_dir: Path, destination: Path) -> Path:
147125
return destination
148126

149127

150-
def resolve_baseline(*, candidate: str, repo_dir: Path) -> str:
151-
"""Return the highest published or not-yet-published tagged release below the candidate."""
128+
def resolve_baseline(*, candidate: str, same_version: bool) -> str:
129+
"""Return the baseline release that PyPI actually serves for the candidate."""
152130

153131
baseline: str | None = choose_baseline(
154-
candidate=candidate,
155-
index=fetch_simple_index(),
156-
tags=tagged_versions(repo_dir=repo_dir),
132+
candidate=candidate, index=fetch_simple_index(), same_version=same_version
157133
)
158134
if baseline is None:
159135
raise ReleasePerformanceError(f"No published sqlbuild release precedes {candidate}")

‎scripts/release_performance/_helpers/workflow.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ def _compare(*, options: ComparisonOptions, root: Path) -> ReleaseComparison:
9797
wait_seconds=0.0,
9898
)
9999
baseline_spec: str = options.baseline or resolve_baseline(
100-
candidate=candidate.version, repo_dir=REPO_ROOT
100+
candidate=candidate.version, same_version=options.same_version_baseline
101101
)
102102
print(f"Installing baseline {baseline_spec}", file=sys.stderr)
103103
baseline: InstalledVersion = _install(

‎scripts/release_performance/constants.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@
3030
PYPI_REQUEST_TIMEOUT_SECONDS: float = 60.0
3131
BASELINE_PUBLICATION_WAIT_SECONDS: float = 1_800.0
3232
BASELINE_PUBLICATION_POLL_SECONDS: float = 30.0
33-
RELEASE_TAG_PATTERN: str = "v*"
3433
WHEEL_PLATFORM_FAMILIES: dict[str, str] = {"linux": "linux", "darwin": "macosx", "win32": "win"}
3534
WHEEL_MACHINE_ALIASES: dict[str, str] = {"amd64": "x86_64", "arm64": "aarch64"}
3635
UNIVERSAL_WHEEL_MACHINE: str = "universal2"

‎scripts/release_performance/main/compare.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,15 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions:
3434
parser.add_argument(
3535
"--baseline",
3636
default=None,
37-
help="Defaults to the highest published or release-tagged version below the candidate.",
37+
help="Defaults to the highest version on PyPI below the candidate.",
38+
)
39+
parser.add_argument(
40+
"--same-version-baseline",
41+
action="store_true",
42+
help=(
43+
"The candidate is an unreleased commit carrying the latest release's version, so "
44+
"that release is the baseline once it is on PyPI."
45+
),
3846
)
3947
parser.add_argument("--runs", type=int, default=DEFAULT_RUNS)
4048
parser.add_argument("--python", default=DEFAULT_PYTHON)
@@ -66,4 +74,5 @@ def _parse_args(argv: list[str] | None) -> ComparisonOptions:
6674
work_dir=args.work_dir,
6775
output=args.output,
6876
baseline_source=args.baseline_source,
77+
same_version_baseline=args.same_version_baseline,
6978
)

‎scripts/release_performance/models.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -94,3 +94,4 @@ class ComparisonOptions:
9494
work_dir: Path | None
9595
output: Path | None
9696
baseline_source: Path | None = None
97+
same_version_baseline: bool = False

‎tests/unit/scripts/release_performance/_helpers/_test_types.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,7 @@ class ChooseBaselineTestCase:
7373
description: str
7474
candidate: str
7575
files: tuple[dict[str, object], ...]
76-
tags: tuple[str, ...]
76+
same_version: bool
7777
expected_baseline: str | None
7878

7979

‎tests/unit/scripts/release_performance/_helpers/test_versions.py‎

Lines changed: 29 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -134,47 +134,62 @@ def test_given_release_files_when_selecting_wheel_urls_then_matches_runner_platf
134134
"test_case",
135135
(
136136
ChooseBaselineTestCase(
137-
description="a tagged release that is still publishing is the baseline",
137+
description="a main commit whose own release is not on PyPI uses the newest one that is",
138138
candidate="0.126.3",
139-
files=(release_file(filename="sqlbuild-0.126.1.tar.gz"),),
140-
tags=("0.126.1", "0.126.2"),
139+
files=(
140+
release_file(filename="sqlbuild-0.126.1.tar.gz"),
141+
release_file(filename="sqlbuild-0.126.2.tar.gz"),
142+
),
143+
same_version=True,
141144
expected_baseline="0.126.2",
142145
),
143146
ChooseBaselineTestCase(
144-
description="a yanked release is skipped even though its tag exists",
145-
candidate="0.126.3",
147+
description="a main commit uses its own release once PyPI serves it",
148+
candidate="0.126.2",
146149
files=(
147150
release_file(filename="sqlbuild-0.126.1.tar.gz"),
148-
{"filename": "sqlbuild-0.126.2.tar.gz", "yanked": "broken build", "url": ""},
151+
release_file(filename="sqlbuild-0.126.2.tar.gz"),
152+
),
153+
same_version=True,
154+
expected_baseline="0.126.2",
155+
),
156+
ChooseBaselineTestCase(
157+
description="a release commit never compares with itself after it publishes",
158+
candidate="0.126.2",
159+
files=(
160+
release_file(filename="sqlbuild-0.126.1.tar.gz"),
161+
release_file(filename="sqlbuild-0.126.2.tar.gz"),
149162
),
150-
tags=("0.126.1", "0.126.2"),
163+
same_version=False,
151164
expected_baseline="0.126.1",
152165
),
153166
ChooseBaselineTestCase(
154-
description="a published release without a local tag is still a baseline",
167+
description="a yanked release is skipped",
155168
candidate="0.126.3",
156169
files=(
157170
release_file(filename="sqlbuild-0.126.1.tar.gz"),
158-
release_file(filename="sqlbuild-0.126.2.tar.gz"),
171+
{"filename": "sqlbuild-0.126.2.tar.gz", "yanked": "broken build", "url": ""},
159172
),
160-
tags=(),
161-
expected_baseline="0.126.2",
173+
same_version=False,
174+
expected_baseline="0.126.1",
162175
),
163176
ChooseBaselineTestCase(
164177
description="nothing precedes the first release",
165178
candidate="0.1.0",
166179
files=(release_file(filename="sqlbuild-0.1.0.tar.gz"),),
167-
tags=("0.1.0",),
180+
same_version=False,
168181
expected_baseline=None,
169182
),
170183
),
171184
ids=lambda case: case.description,
172185
)
173-
def test_given_index_and_release_tags_when_choosing_baseline_then_skips_yanked_releases(
186+
def test_given_published_releases_when_choosing_baseline_then_uses_one_pypi_serves(
174187
test_case: ChooseBaselineTestCase,
175188
) -> None:
176189
baseline: str | None = choose_baseline(
177-
candidate=test_case.candidate, index={"files": list(test_case.files)}, tags=test_case.tags
190+
candidate=test_case.candidate,
191+
index={"files": list(test_case.files)},
192+
same_version=test_case.same_version,
178193
)
179194

180195
assert baseline == test_case.expected_baseline

0 commit comments

Comments
 (0)