Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions .github/scripts/release-notes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
"""Print the requested Release Please section from a mirrored changelog."""

import argparse
from pathlib import Path
import re
import sys


def release_notes(changelog, version):
if not re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version):
raise ValueError(f"Invalid release version: {version}")
headings = list(re.finditer(r"^## \[([0-9]+\.[0-9]+\.[0-9]+)\][^\n]*$", changelog, re.MULTILINE))
matches = [i for i, heading in enumerate(headings) if heading[1] == version]
if len(matches) != 1:
raise ValueError(f"Expected one changelog section for {version}, found {len(matches)}")
index = matches[0]
end = headings[index + 1].start() if index + 1 < len(headings) else len(changelog)
notes = changelog[headings[index].end():end].strip("\n")
if not notes.strip():
raise ValueError(f"Empty changelog section for {version}")
return notes + "\n"


if __name__ == "__main__":
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("version")
parser.add_argument("changelog", type=Path)
args = parser.parse_args()
try:
sys.stdout.write(release_notes(args.changelog.read_text(encoding="utf-8"), args.version))
except ValueError as err:
parser.exit(1, f"{err}\n")
108 changes: 108 additions & 0 deletions .github/scripts/report-release.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,108 @@
#!/usr/bin/env bash
# Invoked for a merged mirror commit, from the protected main branch.
set -euo pipefail

version="$(jq -er '.version | select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))' .release-source.json)"
source_sha="$(jq -er '.sourceSha | select(test("^[0-9a-f]{40}$"))' .release-source.json)"
branch="release/code-scan-action-v${version}"
repo_api="repos/${GITHUB_REPOSITORY}"
repo_url="${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}"
run_url="${repo_url}/actions/runs/${GITHUB_RUN_ID}/attempts/${GITHUB_RUN_ATTEMPT}"
marker='<!-- code-scan-action-publication -->'

# Match the merge commit and the release branch; a maintenance push is not a release PR.
pr="$(gh api "${repo_api}/commits/${GITHUB_SHA}/pulls" | jq -r \
--arg repository "$GITHUB_REPOSITORY" --arg branch "$branch" --arg sha "$GITHUB_SHA" '
.[] | select(.head.repo.full_name == $repository and .head.ref == $branch
and .base.ref == "main" and .merged_at and .merge_commit_sha == $sha) | .number
')"
if [[ -z "$pr" ]]; then
echo "No matching merged release PR for ${GITHUB_SHA}."
exit 0
fi

cli_version="$(gh api -H 'Accept: application/vnd.github.raw+json' \
"repos/promptfoo/promptfoo/contents/package.json?ref=${source_sha}" | \
jq -er '.version | select(test("^[0-9]+\\.[0-9]+\\.[0-9]+$"))')"
body="${RUNNER_TEMP}/action-publication.md"
tags=("v${version}" "v${version%.*}" "v${version%%.*}")
refs=()
for tag in "${tags[@]}"; do
refs+=("refs/tags/$tag" "refs/tags/${tag}^{}")
done
remote_refs="$(git ls-remote origin refs/heads/main "${refs[@]}")"
main_sha="$(awk '$2 == "refs/heads/main" { print $1 }' <<< "$remote_refs")"
[[ "$main_sha" =~ ^[0-9a-f]{40}$ ]]
# The same ref snapshot identifies both the tag targets and the source currently on main.
current_release=true
if [[ "$main_sha" != "$GITHUB_SHA" ]]; then
current_release="$(gh api -H 'Accept: application/vnd.github.raw+json' \
"${repo_api}/contents/.release-source.json?ref=${main_sha}" | jq -r \
--arg version "$version" --arg sha "$source_sha" '.version == $version and .sourceSha == $sha')"
fi

release_commit() {
# Annotated tags have an object ID and a peeled commit; lightweight tags have only the commit.
awk -v ref="refs/tags/$1" '$2 == ref { tag=$1 } $2 == ref "^{}" { peeled=$1 }
END { print peeled ? peeled : tag }' <<< "$remote_refs"
}
aliases_match=true
for alias in "${tags[@]:1}"; do
if [[ "$(release_commit "$alias")" != "$GITHUB_SHA" ]]; then
aliases_match=false
fi
done
exact_sha="$(release_commit "${tags[0]}")"
released=false
if [[ "$exact_sha" == "$GITHUB_SHA" ]]; then
if draft="$(gh release view "${tags[0]}" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft)" && [[ "$draft" == "false" ]]; then
released=true
fi
fi

{
printf '%s\n\n### Code Scan Action v%s\n\n' "$marker" "$version"
if [[ "$released" == true ]]; then
printf '[Published release](%s/releases/tag/v%s) at [%s](%s/commit/%s).\n\n' \
"$repo_url" "$version" "${GITHUB_SHA:0:7}" "$repo_url" "$GITHUB_SHA"
elif [[ "$PUBLICATION_RESULT" == "skipped" ]]; then
printf 'Publication skipped because the source release on `main` changed.\n\n'
else
printf 'Publication could not be verified for the merged commit. See the release workflow.\n\n'
fi
for tag in "${tags[@]}"; do
target="$(release_commit "$tag")"
if [[ -n "$target" ]]; then
printf -- '- `%s` → [%s](%s/commit/%s)\n' "$tag" "${target:0:7}" "$repo_url" "$target"
else
printf -- '- `%s` has no published ref.\n' "$tag"
fi
done
printf '\n'
if [[ "$current_release" == true && "$aliases_match" != true ]]; then
printf 'The current release is missing one or more matching floating tags.\n\n'
fi
printf 'The action pins [promptfoo@%s](https://www.npmjs.com/package/promptfoo/v/%s).\n\n' \
"$cli_version" "$cli_version"
printf '[Source commit](https://github.com/promptfoo/promptfoo/commit/%s) · [Release workflow](%s)\n' \
"$source_sha" "$run_url"
} > "$body"
cat "$body" >> "$GITHUB_STEP_SUMMARY"

comment_id="$(gh api --paginate "${repo_api}/issues/${pr}/comments?per_page=100" |
jq -r --arg marker "$marker" '.[] | select(.user.login == "github-actions[bot]" and (.body | startswith($marker))) | .id')"
if [[ -n "$comment_id" ]]; then
gh api --silent --method PATCH "${repo_api}/issues/comments/${comment_id}" -F "body=@${body}"
else
gh api --silent --method POST "${repo_api}/issues/${pr}/comments" -F "body=@${body}"
fi

if [[ "$current_release" == true && "$aliases_match" != true ]]; then
echo "::error::One or more floating tags do not point to current release ${GITHUB_SHA}."
exit 1
fi

if [[ "$PUBLICATION_RESULT" == "published" && "$released" != true ]]; then
echo "::error::Could not verify public release v${version} at ${GITHUB_SHA}."
exit 1
fi
28 changes: 28 additions & 0 deletions .github/tests/test_release_notes.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
from pathlib import Path
import runpy
import unittest


script = Path(__file__).resolve().parents[1] / "scripts/release-notes.py"
release_notes = runpy.run_path(str(script))["release_notes"]


class ReleaseNotesTest(unittest.TestCase):
def test_extracts_only_the_requested_release(self):
notes = "### Before upgrading\n\nUse the pinned scanner.\n\n<details>\n<summary>All changes</summary>\n\n- Fix\n</details>\n"
changelog = f"## [0.2.4](next)\n\nNewer\n\n## [0.2.3](current)\n\n{notes}\n## [0.2.2](old)\n\nOlder\n"
self.assertEqual(release_notes(changelog, "0.2.3"), notes)

def test_fails_on_missing_ambiguous_or_empty_notes(self):
for changelog in (
"## [0x2x3](wrong)\nSomething\n",
"## [0.2.30](different)\nSomething\n",
"## [0.2.3](first)\nOne\n## [0.2.3](duplicate)\nTwo\n",
"## [0.2.3](empty)\n \n## [0.2.2](older)\nOld notes\n",
):
with self.subTest(changelog=changelog), self.assertRaises(ValueError):
release_notes(changelog, "0.2.3")


if __name__ == "__main__":
unittest.main()
72 changes: 43 additions & 29 deletions .github/workflows/release-tags.yml
Original file line number Diff line number Diff line change
Expand Up @@ -7,8 +7,7 @@ on:
paths:
- .release-source.json

permissions:
contents: write
permissions: {}

concurrency:
group: release-tags-main
Expand All @@ -18,6 +17,11 @@ jobs:
move-release-tags:
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: write
outputs:
version: ${{ steps.release-metadata.outputs.version }}
publication: ${{ steps.release.outputs.publication }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
Expand Down Expand Up @@ -45,6 +49,12 @@ jobs:
echo "major=${version%%.*}" >> "$GITHUB_OUTPUT"
echo "minor=${version%.*}" >> "$GITHUB_OUTPUT"

- name: Read the versioned release notes before moving tags
if: ${{ steps.release-metadata.outputs.release_commit == 'true' }}
env:
VERSION: ${{ steps.release-metadata.outputs.version }}
run: python3 .github/scripts/release-notes.py "$VERSION" CHANGELOG.md > "$RUNNER_TEMP/release-notes.md"

- name: Update floating and exact version tags
if: ${{ steps.release-metadata.outputs.release_commit == 'true' }}
env:
Expand Down Expand Up @@ -72,7 +82,8 @@ jobs:

git push --force origin "refs/tags/v${VERSION}" "refs/tags/v${MINOR}" "refs/tags/v${MAJOR}"

- name: Create or update GitHub release
- name: Publish GitHub release
id: release
if: ${{ steps.release-metadata.outputs.release_commit == 'true' }}
env:
GH_TOKEN: ${{ github.token }}
Expand All @@ -81,44 +92,47 @@ jobs:
run: |
set -euo pipefail

notes_file="$RUNNER_TEMP/release-notes.md"
awk -v version="$VERSION" '
$0 ~ "^## \\[" version "\\]" {
in_section=1
next
}
in_section && /^## \[/ {
exit
}
in_section {
print
}
' CHANGELOG.md > "$notes_file"

if [[ ! -s "$notes_file" ]]; then
printf 'Release v%s\n' "$VERSION" > "$notes_file"
fi

git fetch origin main
main_version="$(git show origin/main:.release-source.json | jq -r '.version')"
main_source_sha="$(git show origin/main:.release-source.json | jq -r '.sourceSha')"
if [[ "$VERSION" != "$main_version" || "$SOURCE_SHA" != "$main_source_sha" ]]; then
echo "publication=skipped" >> "$GITHUB_OUTPUT"
echo "Skipping stale release update for release $VERSION; origin/main now points to $main_version from $main_source_sha"
exit 0
fi

if gh release view "v${VERSION}" --repo "$GITHUB_REPOSITORY" > /dev/null 2>&1; then
gh release edit "v${VERSION}" \
--repo "$GITHUB_REPOSITORY" \
--title "v${VERSION}" \
--notes-file "$notes_file" \
--target "$GITHUB_SHA" \
--verify-tag
# A replay may encounter a published release whose notes maintainers have edited.
if draft="$(gh release view "v${VERSION}" --repo "$GITHUB_REPOSITORY" --json isDraft --jq .isDraft)"; then
if [[ "$draft" == "true" ]]; then
echo "::error::Release v${VERSION} already exists as a draft."
exit 1
fi
echo "Release v${VERSION} already exists."
else
gh release create "v${VERSION}" \
--repo "$GITHUB_REPOSITORY" \
--title "v${VERSION}" \
--notes-file "$notes_file" \
--notes-file "$RUNNER_TEMP/release-notes.md" \
--target "$GITHUB_SHA" \
--verify-tag
fi
echo "publication=published" >> "$GITHUB_OUTPUT"

report-release:
needs: move-release-tags
if: ${{ !cancelled() && needs.move-release-tags.outputs.version != '' }}
runs-on: ubuntu-latest
timeout-minutes: 5
permissions:
contents: read
pull-requests: write
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false

- name: Report publication on the merged mirror PR
env:
GH_TOKEN: ${{ github.token }}
PUBLICATION_RESULT: ${{ needs.move-release-tags.outputs.publication || needs.move-release-tags.result }}
run: bash .github/scripts/report-release.sh
12 changes: 12 additions & 0 deletions .github/workflows/validate-release-pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,10 @@ jobs:

echo "Branch ${{ github.head_ref }} is not a generated release branch; artifact mirror files are unchanged."

- name: Test release note extraction
if: ${{ !startsWith(github.head_ref, 'release/code-scan-action-v') }}
run: python3 .github/tests/test_release_notes.py

- name: Verify release source
if: ${{ startsWith(github.head_ref, 'release/code-scan-action-v') }}
env:
Expand Down Expand Up @@ -127,3 +131,11 @@ jobs:
echo "Generated mirror dist/ does not match promptfoo/promptfoo@$source_sha"
exit 1
fi

- name: Preview the notes that will be published
if: ${{ startsWith(github.head_ref, 'release/code-scan-action-v') }}
run: |
set -euo pipefail
version="$(jq -er '.version' .release-source.json)"
echo "## Code Scan Action v${version}" >> "$GITHUB_STEP_SUMMARY"
python3 .github/scripts/release-notes.py "$version" CHANGELOG.md >> "$GITHUB_STEP_SUMMARY"
Loading