Relock uv.lock daily and propose the upgrades as a pull request

.github/scripts/locked_packages.py

@@ -0,0 +1,145 @@
+"""Read package names and versions from uv.lock for the daily relock.
+
+The relock upgrades every locked package except the PolicyEngine .py bundle
+(``policyengine``) and the packages that the bundle's extras pin: PolicyEngine
+Core, the country models and the SPM calculator. Those change only through the
+bundle update workflow (update-policyengine-bundle.yaml), whose pull requests
+run the check that the simulation API supports the bundle. The project itself
+is never upgraded.
+
+Usage:
+    locked_packages.py upgradeable LOCK  names it may upgrade, one per line
+    locked_packages.py held LOCK         "name version" of each package it holds
+    locked_packages.py summary OLD NEW   Markdown summary of the changes
+"""
+
+from __future__ import annotations
+
+import argparse
+import sys
+import tomllib
+from pathlib import Path
+
+BUNDLE_PACKAGE = "policyengine"
+PROJECT_SOURCES = ("editable", "virtual")
+
+
+def _locked_versions(lock_path: Path) -> tuple[dict[str, list[str]], set[str]]:
+    """Return each locked package's versions, and the names the relock holds.
+
+    A package can be locked at more than one version, one per set of Python
+    or platform markers, so each name maps to a sorted list of versions. The
+    project's own entry is left out.
+
+    Raises:
+        SystemExit: when the bundle package is not in the lock, since the held
+            packages can't be identified without it.
+
+    """
+    with lock_path.open("rb") as lock_file:
+        packages = tomllib.load(lock_file).get("package", [])
+
+    versions: dict[str, set[str]] = {}
+    held = {BUNDLE_PACKAGE}
+    bundle_locked = False
+    for package in packages:
+        source = package.get("source", {})
+        if any(kind in source for kind in PROJECT_SOURCES):
+            continue
+        name = package["name"]
+        versions.setdefault(name, set()).add(package["version"])
+        if name == BUNDLE_PACKAGE:
+            bundle_locked = True
+            for members in package.get("optional-dependencies", {}).values():
+                held.update(member["name"] for member in members)
+
+    if not bundle_locked:
+        message = (
+            f"{BUNDLE_PACKAGE} is not locked in {lock_path}, so the packages "
+            "its extras pin can't be held back."
+        )
+        raise SystemExit(message)
+    return {name: sorted(found) for name, found in versions.items()}, held
+
+
+def upgradeable(lock_path: Path) -> list[str]:
+    """Return the sorted names of the locked packages the relock may upgrade."""
+    versions, held = _locked_versions(lock_path)
+    return sorted(name for name in versions if name not in held)
+
+
+def held_versions(lock_path: Path) -> list[str]:
+    """Return "name version" for each locked package the relock must not move."""
+    versions, held = _locked_versions(lock_path)
+    return [
+        f"{name} {', '.join(versions[name])}"
+        for name in sorted(held)
+        if name in versions
+    ]
+
+
+def summary(old_lock: Path, new_lock: Path) -> str:
+    """Return a Markdown summary of the version changes from old_lock to new_lock."""
+    old_versions, _ = _locked_versions(old_lock)
+    new_versions, held = _locked_versions(new_lock)
+
+    changes = []
+    for name in sorted(old_versions.keys() | new_versions.keys()):
+        old = ", ".join(old_versions.get(name, []))
+        new = ", ".join(new_versions.get(name, []))
+        if not old:
+            changes.append(f"- `{name}`: newly locked at {new}")
+        elif not new:
+            changes.append(f"- `{name}`: no longer locked (was {old})")
+        elif old != new:
+            changes.append(f"- `{name}`: {old} → {new}")
+    if not changes:
+        changes.append(
+            "- No package versions changed; only the files or metadata recorded "
+            "for the locked versions did."
+        )
+
+    held_lines = [
+        f"- `{name}` {', '.join(new_versions[name])}"
+        for name in sorted(held)
+        if name in new_versions
+    ]
+    return "\n".join(
+        [
+            "## Version changes",
+            "",
+            *changes,
+            "",
+            "## Held for the bundle update workflow",
+            "",
+            "These packages change only through the PolicyEngine bundle update "
+            "workflow (`update-policyengine-bundle.yaml`), whose pull requests "
+            "check that the simulation API supports the bundle:",
+            "",
+            *held_lines,
+            "",
+        ]
+    )
+
+
+def main() -> None:
+    """Print the result of the requested command."""
+    parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+    commands = parser.add_subparsers(dest="command", required=True)
+    commands.add_parser("upgradeable").add_argument("lock", type=Path)
+    commands.add_parser("held").add_argument("lock", type=Path)
+    summary_parser = commands.add_parser("summary")
+    summary_parser.add_argument("old_lock", type=Path)
+    summary_parser.add_argument("new_lock", type=Path)
+    arguments = parser.parse_args()
+
+    if arguments.command == "upgradeable":
+        print("\n".join(upgradeable(arguments.lock)))
+    elif arguments.command == "held":
+        print("\n".join(held_versions(arguments.lock)))
+    else:
+        sys.stdout.write(summary(arguments.old_lock, arguments.new_lock))
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/open_relock_pull_request.sh

@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+
+# Commit a relocked uv.lock with a changelog fragment, push it to the relock
+# branch, and open the relock pull request, or update the open one.
+#
+# - The branch is rebuilt on every run from the checked-out commit of the base
+#   branch, so its diff is always the base branch plus one relock. The push
+#   replaces the branch (with a lease on the commit it last saw), and is
+#   skipped when the branch already holds the same files.
+# - The tracking issue is the oldest open issue titled "Relock dependencies",
+#   or a new one when none is open. The first line of the pull request
+#   description is "Fixes #N", so merging the pull request closes it and the
+#   next relock opens a new one.
+# - The description lists the version changes from RELOCK_SUMMARY_FILE.
+#
+# Run from the repository root, after relock_dependencies.sh reported a change.
+#
+# Environment:
+#   GITHUB_REPOSITORY    owner/name of this repository (set by the runner)
+#   GH_TOKEN             read by gh: a token that can write issues and pull
+#                        requests. git pushes with the credentials the checkout
+#                        stored.
+#   RELOCK_SUMMARY_FILE  Markdown summary written by relock_dependencies.sh
+#   BASE_BRANCH          branch the pull request merges into (default: master)
+#   RELOCK_BRANCH        branch the pull request is opened from
+#                        (default: auto/relock-dependencies)
+
+set -euo pipefail
+
+: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}"
+: "${RELOCK_SUMMARY_FILE:?RELOCK_SUMMARY_FILE is required}"
+if [[ ! -r "${RELOCK_SUMMARY_FILE}" ]]; then
+  echo "Relock summary ${RELOCK_SUMMARY_FILE} is missing or unreadable." >&2
+  exit 1
+fi
+
+base_branch="${BASE_BRANCH:-master}"
+relock_branch="${RELOCK_BRANCH:-auto/relock-dependencies}"
+title="Relock dependencies"
+fragment="changelog.d/relock-dependencies.changed.md"
+issue_body="Tracks the daily relock of uv.lock by .github/workflows/relock-dependencies.yml. The relock pull request names this issue on the first line of its description and closes it when it merges."
+
+# Set ISSUE_NUMBER to the oldest open tracking issue, opening one when none is
+# open. This sets a variable rather than printing, because bash turns off
+# set -e inside $(...), and a failed lookup must not open a duplicate issue.
+find_or_open_tracking_issue() {
+  local issue_number
+  local issue_url
+
+  issue_number="$(
+    gh issue list \
+      --repo "${GITHUB_REPOSITORY}" \
+      --state open \
+      --limit 1000 \
+      --json number,title \
+      --jq "map(select(.title == \"${title}\")) | sort_by(.number) | .[0].number // empty"
+  )"
+  if [[ -z "${issue_number}" ]]; then
+    issue_url="$(
+      gh issue create \
+        --repo "${GITHUB_REPOSITORY}" \
+        --title "${title}" \
+        --body "${issue_body}"
+    )"
+    issue_number="${issue_url##*/}"
+  fi
+
+  if [[ ! "${issue_number}" =~ ^[0-9]+$ ]]; then
+    echo "Could not resolve the relock tracking issue: ${issue_number}" >&2
+    exit 1
+  fi
+  ISSUE_NUMBER="${issue_number}"
+}
+
+pr_description() {
+  cat <<EOF
+Fixes #${ISSUE_NUMBER}
+
+## Summary
+
+Automated relock of \`uv.lock\` by \`.github/workflows/relock-dependencies.yml\`, which runs daily. Every locked package is upgraded to the newest version that \`pyproject.toml\` allows, except the PolicyEngine .py bundle and the packages its extras pin. CI on this pull request tests the relocked versions.
+
+Each run rebuilds \`${relock_branch}\` from \`${base_branch}\` and replaces it, so commits pushed to this branch by hand are lost on the next run.
+
+EOF
+  cat "${RELOCK_SUMMARY_FILE}"
+}
+
+# --head matches the branch name in forks too; only this repository's branch
+# is the relock branch.
+open_pr="$(
+  gh pr list \
+    --repo "${GITHUB_REPOSITORY}" \
+    --base "${base_branch}" \
+    --head "${relock_branch}" \
+    --state open \
+    --json number,isCrossRepository \
+    --jq 'map(select(.isCrossRepository | not)) | .[0].number // empty'
+)"
+ISSUE_NUMBER=""
+find_or_open_tracking_issue
+
+git config user.name "github-actions[bot]"
+git config user.email "github-actions[bot]@users.noreply.github.com"
+echo "Update the locked versions of the Public API's dependencies." >"${fragment}"
+git checkout --quiet -B "${relock_branch}"
+git add uv.lock "${fragment}"
+git commit --quiet -m "Relock dependencies"
+
+remote_commit="$(git ls-remote --heads origin "refs/heads/${relock_branch}" | cut -f1)"
+if [[ -n "${remote_commit}" ]]; then
+  git fetch --quiet --depth=1 origin "refs/heads/${relock_branch}"
+fi
+if [[ -n "${remote_commit}" ]] && git diff --quiet "${remote_commit}" HEAD --; then
+  echo "${relock_branch} already holds this relock; not pushing."
+else
+  git push --quiet \
+    --force-with-lease="refs/heads/${relock_branch}:${remote_commit}" \
+    origin "HEAD:refs/heads/${relock_branch}"
+fi
+
+description="$(pr_description)"
+if [[ -z "${open_pr}" ]]; then
+  gh pr create \
+    --repo "${GITHUB_REPOSITORY}" \
+    --base "${base_branch}" \
+    --head "${relock_branch}" \
+    --title "${title}" \
+    --body-file - \
+    <<<"${description}"
+else
+  gh pr edit "${open_pr}" \
+    --repo "${GITHUB_REPOSITORY}" \
+    --body-file - \
+    <<<"${description}"
+  echo "Updated pull request #${open_pr}."
+fi

.github/scripts/relock_dependencies.sh

@@ -0,0 +1,71 @@
+#!/usr/bin/env bash
+
+# Relock uv.lock: upgrade every locked package to the newest version that
+# pyproject.toml allows, except the PolicyEngine .py bundle (policyengine) and
+# the packages its extras pin, which change only through the bundle update
+# workflow. Then check the lock, and fail if a held package moved anyway.
+#
+# Run from the repository root. uv resolves the lock for every Python version
+# that requires-python allows, whichever interpreter runs it; the interpreter
+# only builds packages that publish no static metadata.
+#
+# Environment:
+#   RELOCK_PYTHON        Python version uv runs with (default: 3.12, the
+#                        version the Public API runs on; .python-version names
+#                        a newer one)
+#   RELOCK_SUMMARY_FILE  where to write the Markdown summary of the version
+#                        changes (default: relock-summary.md)
+#   GITHUB_OUTPUT        when set, receives changed=true or changed=false
+#   PYTHON               interpreter that reads the lock (Python 3.11+ for
+#                        tomllib; default: python3)
+
+set -euo pipefail
+
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+locked_packages=("${PYTHON:-python3}" "${script_dir}/locked_packages.py")
+python_version="${RELOCK_PYTHON:-3.12}"
+summary_file="${RELOCK_SUMMARY_FILE:-relock-summary.md}"
+
+upgradeable="$("${locked_packages[@]}" upgradeable uv.lock)"
+held_before="$("${locked_packages[@]}" held uv.lock)"
+if [[ -z "${upgradeable}" ]]; then
+  echo "uv.lock has no packages to upgrade." >&2
+  exit 1
+fi
+
+upgrade_arguments=()
+while IFS= read -r package_name; do
+  upgrade_arguments+=(--upgrade-package "${package_name}")
+done <<<"${upgradeable}"
+
+previous_lock="$(mktemp)"
+trap 'rm -f "${previous_lock}"' EXIT
+cp uv.lock "${previous_lock}"
+
+uv lock --python "${python_version}" "${upgrade_arguments[@]}"
+uv lock --check --python "${python_version}"
+
+held_after="$("${locked_packages[@]}" held uv.lock)"
+if [[ "${held_after}" != "${held_before}" ]]; then
+  {
+    echo "::error::The relock moved packages that only the bundle update workflow may change."
+    echo "Before:"
+    echo "${held_before}"
+    echo "After:"
+    echo "${held_after}"
+  } >&2
+  exit 1
+fi
+
+if cmp -s "${previous_lock}" uv.lock; then
+  echo "uv.lock is unchanged; nothing to propose."
+  changed=false
+else
+  "${locked_packages[@]}" summary "${previous_lock}" uv.lock >"${summary_file}"
+  cat "${summary_file}"
+  changed=true
+fi
+
+if [[ -n "${GITHUB_OUTPUT:-}" ]]; then
+  echo "changed=${changed}" >>"${GITHUB_OUTPUT}"
+fi