diff --git a/.github/scripts/release-notes.py b/.github/scripts/release-notes.py new file mode 100644 index 0000000..ce1da14 --- /dev/null +++ b/.github/scripts/release-notes.py @@ -0,0 +1,32 @@ +"""Print the requested Release Please section from a mirrored changelog.""" + +import argparse +from pathlib import Path +import re +import sys + + +def release_notes(changelog, version): + if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version): + raise ValueError(f"Invalid release version: {version}") + headings = list(re.finditer(r"^## \[([0-9]+\.[0-9]+\.[0-9]+)\][^\n]*$", changelog, re.MULTILINE)) + matches = [i for i, heading in enumerate(headings) if heading[1] == version] + if len(matches) != 1: + raise ValueError(f"Expected one changelog section for {version}, found {len(matches)}") + index = matches[0] + end = headings[index + 1].start() if index + 1 < len(headings) else len(changelog) + notes = changelog[headings[index].end():end].strip("\n") + if not notes.strip(): + raise ValueError(f"Empty changelog section for {version}") + return notes + "\n" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("version") + parser.add_argument("changelog", type=Path) + args = parser.parse_args() + try: + sys.stdout.write(release_notes(args.changelog.read_text(encoding="utf-8"), args.version)) + except ValueError as err: + parser.exit(1, f"{err}\n") diff --git a/.github/scripts/report-release.sh b/.github/scripts/report-release.sh new file mode 100644 index 0000000..6186c99 --- /dev/null +++ b/.github/scripts/report-release.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash +# Invoked for a merged mirror commit, from the protected main branch. +set -euo pipefail + +version="$(jq -er '.version | select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))' .release-source.json)" +source_sha="$(jq -er '.sourceSha | select(test("^[0-9a-f]{40}$"))' .release-source.json)" +branch="release/code-scan-action-v${version}" +repo_api="repos/${GITHUB_REPOSITORY}" +repo_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}" +run_url="${repo_url}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}" +marker='' + +# Match the merge commit and the release branch; a maintenance push is not a release PR. +pr="$(gh api "${repo_api}/commits/${GITHUB_SHA}/pulls" | jq -r \ + --arg repository "$GITHUB_REPOSITORY" --arg branch "$branch" --arg sha "$GITHUB_SHA" ' + .[] | select(.head.repo.full_name == $repository and .head.ref == $branch + and .base.ref == "main" and .merged_at and .merge_commit_sha == $sha) | .number + ')" +if [[ -z "$pr" ]]; then + echo "No matching merged release PR for ${GITHUB_SHA}." + exit 0 +fi + +cli_version="$(gh api -H 'Accept: application/vnd.github.raw+json' \ + "repos/promptfoo/promptfoo/contents/package.json?ref=${source_sha}" | \ + jq -er '.version | select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))')" +body="${RUNNER_TEMP}/action-publication.md" +tags=("v${version}" "v${version%.*}" "v${version%%.*}") +refs=() +for tag in "${tags[@]}"; do + refs+=("refs/tags/$tag" "refs/tags/${tag}^{}") +done +remote_refs="$(git ls-remote origin refs/heads/main "${refs[@]}")" +main_sha="$(awk '$2 == "refs/heads/main" { print $1 }' <<< "$remote_refs")" +[[ "$main_sha" =~ ^[0-9a-f]{40}$ ]] +# The same ref snapshot identifies both the tag targets and the source currently on main. +current_release=true +if [[ "$main_sha" != "$GITHUB_SHA" ]]; then + current_release="$(gh api -H 'Accept: application/vnd.github.raw+json' \ + "${repo_api}/contents/.release-source.json?ref=${main_sha}" | jq -r \ + --arg version "$version" --arg sha "$source_sha" '.version == $version and .sourceSha == $sha')" +fi + +release_commit() { + # Annotated tags have an object ID and a peeled commit; lightweight tags have only the commit. + awk -v ref="refs/tags/$1" '$2 == ref { tag=$1 } $2 == ref "^{}" { peeled=$1 } + END { print peeled ? peeled : tag }' <<< "$remote_refs" +} +aliases_match=true +for alias in "${tags[@]:1}"; do + if [[ "$(release_commit "$alias")" != "$GITHUB_SHA" ]]; then + aliases_match=false + fi +done +exact_sha="$(release_commit "${tags[0]}")" +released=false +if [[ "$exact_sha" == "$GITHUB_SHA" ]]; then + if draft="$(gh release view "${tags[0]}" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft)" && [[ "$draft" == "false" ]]; then + released=true + fi +fi + +{ + printf '%s\n\n### Code Scan Action v%s\n\n' "$marker" "$version" + if [[ "$released" == true ]]; then + printf '[Published release](%s/releases/tag/v%s) at [%s](%s/commit/%s).\n\n' \ + "$repo_url" "$version" "${GITHUB_SHA:0:7}" "$repo_url" "$GITHUB_SHA" + elif [[ "$PUBLICATION_RESULT" == "skipped" ]]; then + printf 'Publication skipped because the source release on `main` changed.\n\n' + else + printf 'Publication could not be verified for the merged commit. See the release workflow.\n\n' + fi + for tag in "${tags[@]}"; do + target="$(release_commit "$tag")" + if [[ -n "$target" ]]; then + printf -- '- `%s` → [%s](%s/commit/%s)\n' "$tag" "${target:0:7}" "$repo_url" "$target" + else + printf -- '- `%s` has no published ref.\n' "$tag" + fi + done + printf '\n' + if [[ "$current_release" == true && "$aliases_match" != true ]]; then + printf 'The current release is missing one or more matching floating tags.\n\n' + fi + printf 'The action pins [promptfoo@%s](https://www.npmjs.com/package/promptfoo/v/%s).\n\n' \ + "$cli_version" "$cli_version" + printf '[Source commit](https://github.com/promptfoo/promptfoo/commit/%s) · [Release workflow](%s)\n' \ + "$source_sha" "$run_url" +} > "$body" +cat "$body" >> "$GITHUB_STEP_SUMMARY" + +comment_id="$(gh api --paginate "${repo_api}/issues/${pr}/comments?per_page=100" | + jq -r --arg marker "$marker" '.[] | select(.user.login == "github-actions[bot]" and (.body | startswith($marker))) | .id')" +if [[ -n "$comment_id" ]]; then + gh api --silent --method PATCH "${repo_api}/issues/comments/${comment_id}" -F "body=@${body}" +else + gh api --silent --method POST "${repo_api}/issues/${pr}/comments" -F "body=@${body}" +fi + +if [[ "$current_release" == true && "$aliases_match" != true ]]; then + echo "::error::One or more floating tags do not point to current release ${GITHUB_SHA}." + exit 1 +fi + +if [[ "$PUBLICATION_RESULT" == "published" && "$released" != true ]]; then + echo "::error::Could not verify public release v${version} at ${GITHUB_SHA}." + exit 1 +fi diff --git a/.github/tests/test_release_notes.py b/.github/tests/test_release_notes.py new file mode 100644 index 0000000..eb4b468 --- /dev/null +++ b/.github/tests/test_release_notes.py @@ -0,0 +1,28 @@ +from pathlib import Path +import runpy +import unittest + + +script = Path(__file__).resolve().parents[1] / "scripts/release-notes.py" +release_notes = runpy.run_path(str(script))["release_notes"] + + +class ReleaseNotesTest(unittest.TestCase): + def test_extracts_only_the_requested_release(self): + notes = "### Before upgrading\n\nUse the pinned scanner.\n\n
\nAll changes\n\n- Fix\n
\n" + changelog = f"## [0.2.4](next)\n\nNewer\n\n## [0.2.3](current)\n\n{notes}\n## [0.2.2](old)\n\nOlder\n" + self.assertEqual(release_notes(changelog, "0.2.3"), notes) + + def test_fails_on_missing_ambiguous_or_empty_notes(self): + for changelog in ( + "## [0x2x3](wrong)\nSomething\n", + "## [0.2.30](different)\nSomething\n", + "## [0.2.3](first)\nOne\n## [0.2.3](duplicate)\nTwo\n", + "## [0.2.3](empty)\n \n## [0.2.2](older)\nOld notes\n", + ): + with self.subTest(changelog=changelog), self.assertRaises(ValueError): + release_notes(changelog, "0.2.3") + + +if __name__ == "__main__": + unittest.main() diff --git a/.github/workflows/release-tags.yml b/.github/workflows/release-tags.yml index 6e81b65..2328966 100644 --- a/.github/workflows/release-tags.yml +++ b/.github/workflows/release-tags.yml @@ -7,8 +7,7 @@ on: paths: - .release-source.json -permissions: - contents: write +permissions: {} concurrency: group: release-tags-main @@ -18,6 +17,11 @@ jobs: move-release-tags: runs-on: ubuntu-latest timeout-minutes: 5 + permissions: + contents: write + outputs: + version: ${{ steps.release-metadata.outputs.version }} + publication: ${{ steps.release.outputs.publication }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 with: @@ -45,6 +49,12 @@ jobs: echo "major=${version%%.*}" >> "$GITHUB_OUTPUT" echo "minor=${version%.*}" >> "$GITHUB_OUTPUT" + - name: Read the versioned release notes before moving tags + if: ${{ steps.release-metadata.outputs.release_commit == 'true' }} + env: + VERSION: ${{ steps.release-metadata.outputs.version }} + run: python3 .github/scripts/release-notes.py "$VERSION" CHANGELOG.md > "$RUNNER_TEMP/release-notes.md" + - name: Update floating and exact version tags if: ${{ steps.release-metadata.outputs.release_commit == 'true' }} env: @@ -72,7 +82,8 @@ jobs: git push --force origin "refs/tags/v${VERSION}" "refs/tags/v${MINOR}" "refs/tags/v${MAJOR}" - - name: Create or update GitHub release + - name: Publish GitHub release + id: release if: ${{ steps.release-metadata.outputs.release_commit == 'true' }} env: GH_TOKEN: ${{ github.token }} @@ -81,44 +92,47 @@ jobs: run: | set -euo pipefail - notes_file="$RUNNER_TEMP/release-notes.md" - awk -v version="$VERSION" ' - $0 ~ "^## \\[" version "\\]" { - in_section=1 - next - } - in_section && /^## \[/ { - exit - } - in_section { - print - } - ' CHANGELOG.md > "$notes_file" - - if [[ ! -s "$notes_file" ]]; then - printf 'Release v%s\n' "$VERSION" > "$notes_file" - fi - git fetch origin main main_version="$(git show origin/main:.release-source.json | jq -r '.version')" main_source_sha="$(git show origin/main:.release-source.json | jq -r '.sourceSha')" if [[ "$VERSION" != "$main_version" || "$SOURCE_SHA" != "$main_source_sha" ]]; then + echo "publication=skipped" >> "$GITHUB_OUTPUT" echo "Skipping stale release update for release $VERSION; origin/main now points to $main_version from $main_source_sha" exit 0 fi - if gh release view "v${VERSION}" --repo "$GITHUB_REPOSITORY" > /dev/null 2>&1; then - gh release edit "v${VERSION}" \ - --repo "$GITHUB_REPOSITORY" \ - --title "v${VERSION}" \ - --notes-file "$notes_file" \ - --target "$GITHUB_SHA" \ - --verify-tag + # A replay may encounter a published release whose notes maintainers have edited. + if draft="$(gh release view "v${VERSION}" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft)"; then + if [[ "$draft" == "true" ]]; then + echo "::error::Release v${VERSION} already exists as a draft." + exit 1 + fi + echo "Release v${VERSION} already exists." else gh release create "v${VERSION}" \ --repo "$GITHUB_REPOSITORY" \ --title "v${VERSION}" \ - --notes-file "$notes_file" \ + --notes-file "$RUNNER_TEMP/release-notes.md" \ --target "$GITHUB_SHA" \ --verify-tag fi + echo "publication=published" >> "$GITHUB_OUTPUT" + + report-release: + needs: move-release-tags + if: ${{ !cancelled() && needs.move-release-tags.outputs.version != '' }} + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: write + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7 + with: + persist-credentials: false + + - name: Report publication on the merged mirror PR + env: + GH_TOKEN: ${{ github.token }} + PUBLICATION_RESULT: ${{ needs.move-release-tags.outputs.publication || needs.move-release-tags.result }} + run: bash .github/scripts/report-release.sh diff --git a/.github/workflows/validate-release-pr.yml b/.github/workflows/validate-release-pr.yml index a51b70b..37266ee 100644 --- a/.github/workflows/validate-release-pr.yml +++ b/.github/workflows/validate-release-pr.yml @@ -32,6 +32,10 @@ jobs: echo "Branch ${{ github.head_ref }} is not a generated release branch; artifact mirror files are unchanged." + - name: Test release note extraction + if: ${{ !startsWith(github.head_ref, 'release/code-scan-action-v') }} + run: python3 .github/tests/test_release_notes.py + - name: Verify release source if: ${{ startsWith(github.head_ref, 'release/code-scan-action-v') }} env: @@ -127,3 +131,11 @@ jobs: echo "Generated mirror dist/ does not match promptfoo/promptfoo@$source_sha" exit 1 fi + + - name: Preview the notes that will be published + if: ${{ startsWith(github.head_ref, 'release/code-scan-action-v') }} + run: | + set -euo pipefail + version="$(jq -er '.version' .release-source.json)" + echo "## Code Scan Action v${version}" >> "$GITHUB_STEP_SUMMARY" + python3 .github/scripts/release-notes.py "$version" CHANGELOG.md >> "$GITHUB_STEP_SUMMARY"