From 4fb16d438cf2f7ccd320b12a7a52538abbae75d3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 21 Oct 2026 23:57:37 +0400 Subject: [PATCH 1/4] Add a relock script that holds the PolicyEngine bundle relock_dependencies.sh upgrades every package in uv.lock to the newest version pyproject.toml allows, passing --upgrade-package for each locked package except policyengine and the packages its extras pin (Core, the country models and the SPM calculator), which change only through the bundle update workflow. It resolves with Python 3.12 by default, runs uv lock --check, fails if a held package moved anyway, and reports changed=true or changed=false with a Markdown summary of the version changes. locked_packages.py reads the lock for it. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/locked_packages.py | 145 ++++++++++ .github/scripts/relock_dependencies.sh | 71 +++++ tests/unit/test_relock_dependencies_script.py | 248 ++++++++++++++++++ 3 files changed, 464 insertions(+) create mode 100644 .github/scripts/locked_packages.py create mode 100644 .github/scripts/relock_dependencies.sh create mode 100644 tests/unit/test_relock_dependencies_script.py diff --git a/.github/scripts/locked_packages.py b/.github/scripts/locked_packages.py new file mode 100644 index 000000000..2391e2b24 --- /dev/null +++ b/.github/scripts/locked_packages.py @@ -0,0 +1,145 @@ +"""Read package names and versions from uv.lock for the daily relock. + +The relock upgrades every locked package except the PolicyEngine .py bundle +(``policyengine``) and the packages that the bundle's extras pin: PolicyEngine +Core, the country models and the SPM calculator. Those change only through the +bundle update workflow (update-policyengine-bundle.yaml), whose pull requests +run the check that the simulation API supports the bundle. The project itself +is never upgraded. + +Usage: + locked_packages.py upgradeable LOCK names it may upgrade, one per line + locked_packages.py held LOCK "name version" of each package it holds + locked_packages.py summary OLD NEW Markdown summary of the changes +""" + +from __future__ import annotations + +import argparse +import sys +import tomllib +from pathlib import Path + +BUNDLE_PACKAGE = "policyengine" +PROJECT_SOURCES = ("editable", "virtual") + + +def _locked_versions(lock_path: Path) -> tuple[dict[str, list[str]], set[str]]: + """Return each locked package's versions, and the names the relock holds. + + A package can be locked at more than one version, one per set of Python + or platform markers, so each name maps to a sorted list of versions. The + project's own entry is left out. + + Raises: + SystemExit: when the bundle package is not in the lock, since the held + packages can't be identified without it. + + """ + with lock_path.open("rb") as lock_file: + packages = tomllib.load(lock_file).get("package", []) + + versions: dict[str, set[str]] = {} + held = {BUNDLE_PACKAGE} + bundle_locked = False + for package in packages: + source = package.get("source", {}) + if any(kind in source for kind in PROJECT_SOURCES): + continue + name = package["name"] + versions.setdefault(name, set()).add(package["version"]) + if name == BUNDLE_PACKAGE: + bundle_locked = True + for members in package.get("optional-dependencies", {}).values(): + held.update(member["name"] for member in members) + + if not bundle_locked: + message = ( + f"{BUNDLE_PACKAGE} is not locked in {lock_path}, so the packages " + "its extras pin can't be held back." + ) + raise SystemExit(message) + return {name: sorted(found) for name, found in versions.items()}, held + + +def upgradeable(lock_path: Path) -> list[str]: + """Return the sorted names of the locked packages the relock may upgrade.""" + versions, held = _locked_versions(lock_path) + return sorted(name for name in versions if name not in held) + + +def held_versions(lock_path: Path) -> list[str]: + """Return "name version" for each locked package the relock must not move.""" + versions, held = _locked_versions(lock_path) + return [ + f"{name} {', '.join(versions[name])}" + for name in sorted(held) + if name in versions + ] + + +def summary(old_lock: Path, new_lock: Path) -> str: + """Return a Markdown summary of the version changes from old_lock to new_lock.""" + old_versions, _ = _locked_versions(old_lock) + new_versions, held = _locked_versions(new_lock) + + changes = [] + for name in sorted(old_versions.keys() | new_versions.keys()): + old = ", ".join(old_versions.get(name, [])) + new = ", ".join(new_versions.get(name, [])) + if not old: + changes.append(f"- `{name}`: newly locked at {new}") + elif not new: + changes.append(f"- `{name}`: no longer locked (was {old})") + elif old != new: + changes.append(f"- `{name}`: {old} → {new}") + if not changes: + changes.append( + "- No package versions changed; only the files or metadata recorded " + "for the locked versions did." + ) + + held_lines = [ + f"- `{name}` {', '.join(new_versions[name])}" + for name in sorted(held) + if name in new_versions + ] + return "\n".join( + [ + "## Version changes", + "", + *changes, + "", + "## Held for the bundle update workflow", + "", + "These packages change only through the PolicyEngine bundle update " + "workflow (`update-policyengine-bundle.yaml`), whose pull requests " + "check that the simulation API supports the bundle:", + "", + *held_lines, + "", + ] + ) + + +def main() -> None: + """Print the result of the requested command.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + commands = parser.add_subparsers(dest="command", required=True) + commands.add_parser("upgradeable").add_argument("lock", type=Path) + commands.add_parser("held").add_argument("lock", type=Path) + summary_parser = commands.add_parser("summary") + summary_parser.add_argument("old_lock", type=Path) + summary_parser.add_argument("new_lock", type=Path) + arguments = parser.parse_args() + + if arguments.command == "upgradeable": + print("\n".join(upgradeable(arguments.lock))) + elif arguments.command == "held": + print("\n".join(held_versions(arguments.lock))) + else: + sys.stdout.write(summary(arguments.old_lock, arguments.new_lock)) + + +if __name__ == "__main__": + main() diff --git a/.github/scripts/relock_dependencies.sh b/.github/scripts/relock_dependencies.sh new file mode 100644 index 000000000..5757250a0 --- /dev/null +++ b/.github/scripts/relock_dependencies.sh @@ -0,0 +1,71 @@ +#!/usr/bin/env bash + +# Relock uv.lock: upgrade every locked package to the newest version that +# pyproject.toml allows, except the PolicyEngine .py bundle (policyengine) and +# the packages its extras pin, which change only through the bundle update +# workflow. Then check the lock, and fail if a held package moved anyway. +# +# Run from the repository root. uv resolves the lock for every Python version +# that requires-python allows, whichever interpreter runs it; the interpreter +# only builds packages that publish no static metadata. +# +# Environment: +# RELOCK_PYTHON Python version uv runs with (default: 3.12, the +# version the Public API runs on; .python-version names +# a newer one) +# RELOCK_SUMMARY_FILE where to write the Markdown summary of the version +# changes (default: relock-summary.md) +# GITHUB_OUTPUT when set, receives changed=true or changed=false +# PYTHON interpreter that reads the lock (Python 3.11+ for +# tomllib; default: python3) + +set -euo pipefail + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +locked_packages=("${PYTHON:-python3}" "${script_dir}/locked_packages.py") +python_version="${RELOCK_PYTHON:-3.12}" +summary_file="${RELOCK_SUMMARY_FILE:-relock-summary.md}" + +upgradeable="$("${locked_packages[@]}" upgradeable uv.lock)" +held_before="$("${locked_packages[@]}" held uv.lock)" +if [[ -z "${upgradeable}" ]]; then + echo "uv.lock has no packages to upgrade." >&2 + exit 1 +fi + +upgrade_arguments=() +while IFS= read -r package_name; do + upgrade_arguments+=(--upgrade-package "${package_name}") +done <<<"${upgradeable}" + +previous_lock="$(mktemp)" +trap 'rm -f "${previous_lock}"' EXIT +cp uv.lock "${previous_lock}" + +uv lock --python "${python_version}" "${upgrade_arguments[@]}" +uv lock --check --python "${python_version}" + +held_after="$("${locked_packages[@]}" held uv.lock)" +if [[ "${held_after}" != "${held_before}" ]]; then + { + echo "::error::The relock moved packages that only the bundle update workflow may change." + echo "Before:" + echo "${held_before}" + echo "After:" + echo "${held_after}" + } >&2 + exit 1 +fi + +if cmp -s "${previous_lock}" uv.lock; then + echo "uv.lock is unchanged; nothing to propose." + changed=false +else + "${locked_packages[@]}" summary "${previous_lock}" uv.lock >"${summary_file}" + cat "${summary_file}" + changed=true +fi + +if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + echo "changed=${changed}" >>"${GITHUB_OUTPUT}" +fi diff --git a/tests/unit/test_relock_dependencies_script.py b/tests/unit/test_relock_dependencies_script.py new file mode 100644 index 000000000..ae8d59bbc --- /dev/null +++ b/tests/unit/test_relock_dependencies_script.py @@ -0,0 +1,248 @@ +"""Tests for .github/scripts/relock_dependencies.sh and locked_packages.py. + +The relock upgrades every package in uv.lock except the PolicyEngine .py +bundle (``policyengine``) and the packages its extras pin, which move only +through the bundle update workflow. A fake ``uv`` stands in for resolution, so +the tests need no network access. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parents[2] +RELOCK_SCRIPT = REPO / ".github/scripts/relock_dependencies.sh" +LOCKED_PACKAGES_SCRIPT = REPO / ".github/scripts/locked_packages.py" +REGISTRY_SOURCE = '{ registry = "https://pypi.org/simple" }' +BUNDLE_PINS = ( + "policyengine-core", + "policyengine-uk", + "policyengine-us", + "spm-calculator", +) + +FAKE_UV = f"""#!{sys.executable} +import json +import os +import shutil +import sys + +arguments = sys.argv[1:] +with open(os.environ["RELOCK_TEST_CALLS"], "a") as calls: + calls.write(json.dumps(["uv", *arguments]) + "\\n") +if arguments[:1] == ["lock"] and "--check" in arguments: + sys.exit(int(os.environ.get("RELOCK_TEST_CHECK_EXIT", "0"))) +if arguments[:1] == ["lock"]: + relocked = os.environ.get("RELOCK_TEST_RELOCKED") + if relocked: + shutil.copyfile(relocked, "uv.lock") + sys.exit(int(os.environ.get("RELOCK_TEST_LOCK_EXIT", "0"))) +""" + + +def _package( + name: str, + version: str, + *, + source: str = REGISTRY_SOURCE, + extras: dict[str, tuple[str, ...]] | None = None, +) -> str: + entry = f'[[package]]\nname = "{name}"\nversion = "{version}"\nsource = {source}\n' + for extra_name, members in (extras or {}).items(): + listed = "".join(f' {{ name = "{member}" }},\n' for member in members) + entry += f"\n[package.optional-dependencies]\n{extra_name} = [\n{listed}]\n" + return entry + + +def _lock_text(*packages: str) -> str: + header = 'version = 1\nrevision = 3\nrequires-python = ">=3.11"\n' + return "\n".join([header, *packages]) + + +def _project_lock(**versions: str) -> str: + """Return a lock with the project, the bundle, its pins and other packages.""" + locked = { + "numpy": "2.3.0", + "pandas": "2.3.1", + "policyengine": "6.7.5", + "policyengine-core": "3.33.0", + "policyengine-uk": "2.130.1", + "policyengine-us": "2.2.1", + "ruff": "0.15.12", + "spm-calculator": "1.0.0", + } + locked.update({name.replace("_", "-"): value for name, value in versions.items()}) + entries = [_package("policyengine-api", "5.6.3", source='{ editable = "." }')] + for name, version in sorted(locked.items()): + if not version: + continue + extras = {"models": BUNDLE_PINS} if name == "policyengine" else None + entries.append(_package(name, version, extras=extras)) + return _lock_text(*entries) + + +def _relock( + checkout: Path, + lock: str, + *, + relocked: str | None = None, + **environment: str, +) -> tuple[subprocess.CompletedProcess[str], list[list[str]], str]: + """Run the relock in ``checkout`` with a fake uv; return result, calls, outputs.""" + binaries = checkout / "bin" + binaries.mkdir() + fake_uv = binaries / "uv" + fake_uv.write_text(FAKE_UV) + fake_uv.chmod(0o755) + (checkout / "uv.lock").write_text(lock) + calls_log = checkout / "calls.jsonl" + outputs = checkout / "github_output" + run_environment = { + **os.environ, + "PATH": f"{binaries}{os.pathsep}{os.environ['PATH']}", + "PYTHON": sys.executable, + "GITHUB_OUTPUT": str(outputs), + "RELOCK_SUMMARY_FILE": str(checkout / "summary.md"), + "RELOCK_TEST_CALLS": str(calls_log), + } + run_environment.pop("RELOCK_PYTHON", None) + run_environment.update(environment) + if relocked is not None: + relocked_lock = checkout / "relocked.lock" + relocked_lock.write_text(relocked) + run_environment["RELOCK_TEST_RELOCKED"] = str(relocked_lock) + result = subprocess.run( + ["bash", str(RELOCK_SCRIPT)], + cwd=checkout, + env=run_environment, + text=True, + capture_output=True, + check=False, + ) + calls = ( + [json.loads(call) for call in calls_log.read_text().splitlines()] + if calls_log.exists() + else [] + ) + written_outputs = outputs.read_text() if outputs.exists() else "" + return result, calls, written_outputs + + +def _upgraded_packages(lock_call: list[str]) -> list[str]: + return [ + lock_call[position + 1] + for position, argument in enumerate(lock_call) + if argument == "--upgrade-package" + ] + + +def test_upgrades_every_package_except_the_bundle_and_its_pins(tmp_path: Path) -> None: + lock = _project_lock() + "\n" + _package("numpy", "2.2.6") + + result, calls, _ = _relock(tmp_path, lock) + + assert result.returncode == 0, result.stderr + lock_call, check_call = calls + assert lock_call[:4] == ["uv", "lock", "--python", "3.12"] + assert _upgraded_packages(lock_call) == ["numpy", "pandas", "ruff"] + assert check_call == ["uv", "lock", "--check", "--python", "3.12"] + + +def test_resolves_with_the_requested_python(tmp_path: Path) -> None: + result, calls, _ = _relock(tmp_path, _project_lock(), RELOCK_PYTHON="3.11") + + assert result.returncode == 0, result.stderr + assert [call[:4] for call in calls] == [ + ["uv", "lock", "--python", "3.11"], + ["uv", "lock", "--check", "--python"], + ] + assert calls[1][4] == "3.11" + + +def test_reports_no_change_when_the_lock_is_unchanged(tmp_path: Path) -> None: + result, _, outputs = _relock(tmp_path, _project_lock()) + + assert result.returncode == 0, result.stderr + assert outputs.splitlines() == ["changed=false"] + assert "uv.lock is unchanged" in result.stdout + + +def test_lists_version_changes_when_the_lock_changes(tmp_path: Path) -> None: + relocked = _project_lock(numpy="2.3.1", ruff="", tzdata="2025.2") + + result, _, outputs = _relock(tmp_path, _project_lock(), relocked=relocked) + + assert result.returncode == 0, result.stderr + assert outputs.splitlines() == ["changed=true"] + summary = (tmp_path / "summary.md").read_text() + assert "- `numpy`: 2.3.0 → 2.3.1" in summary + assert "- `ruff`: no longer locked (was 0.15.12)" in summary + assert "- `tzdata`: newly locked at 2025.2" in summary + assert "pandas" not in summary.split("## Held")[0] + assert "- `policyengine` 6.7.5" in summary + assert "- `policyengine-us` 2.2.1" in summary + + +def test_fails_when_the_relock_moves_a_bundle_package(tmp_path: Path) -> None: + relocked = _project_lock(numpy="2.3.1", policyengine_us="2.3.0") + + result, _, outputs = _relock(tmp_path, _project_lock(), relocked=relocked) + + assert result.returncode != 0 + assert "policyengine-us" in result.stderr + assert "changed=" not in outputs + + +def test_stops_when_uv_lock_fails(tmp_path: Path) -> None: + result, calls, outputs = _relock( + tmp_path, _project_lock(), RELOCK_TEST_LOCK_EXIT="1" + ) + + assert result.returncode != 0 + assert len(calls) == 1 + assert "--check" not in calls[0] + assert outputs == "" + + +def test_fails_when_the_lock_check_fails(tmp_path: Path) -> None: + result, calls, outputs = _relock( + tmp_path, _project_lock(), RELOCK_TEST_CHECK_EXIT="1" + ) + + assert result.returncode != 0 + assert len(calls) == 2 + assert outputs == "" + + +def test_fails_before_locking_when_the_bundle_is_not_locked(tmp_path: Path) -> None: + lock = _lock_text(_package("numpy", "2.3.0")) + + result, calls, _ = _relock(tmp_path, lock) + + assert result.returncode != 0 + assert "policyengine is not locked" in result.stderr + assert calls == [] + + +def _locked_packages(command: str, lock: Path) -> list[str]: + result = subprocess.run( + [sys.executable, str(LOCKED_PACKAGES_SCRIPT), command, str(lock)], + text=True, + capture_output=True, + check=True, + ) + return result.stdout.splitlines() + + +def test_the_repository_lock_holds_the_bundle_and_its_pinned_packages() -> None: + held = {entry.split()[0] for entry in _locked_packages("held", REPO / "uv.lock")} + upgradeable = set(_locked_packages("upgradeable", REPO / "uv.lock")) + + assert held == {"policyengine", *BUNDLE_PINS} + assert not held & upgradeable + assert "policyengine-api" not in upgradeable + assert {"fastapi", "numpy", "policyengine-observability"} <= upgradeable From 4fc9fbded6c95323b61846ca828ad1cc19e439a0 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Wed, 21 Oct 2026 23:59:53 +0400 Subject: [PATCH 2/4] Add a script that opens or updates the relock pull request open_relock_pull_request.sh commits the relocked uv.lock with a changelog fragment on the fixed relock branch, rebuilt from the base branch on every run. It pushes with a lease on the commit it last saw, and skips the push when the branch already holds the same files. It reuses the oldest open "Relock dependencies" issue or opens one, then opens the pull request with "Fixes #N" as the first line of its description and the version changes below it, or updates the description of the open relock pull request instead of opening another. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/open_relock_pull_request.sh | 135 ++++++++++ tests/unit/test_relock_pull_request_script.py | 252 ++++++++++++++++++ 2 files changed, 387 insertions(+) create mode 100644 .github/scripts/open_relock_pull_request.sh create mode 100644 tests/unit/test_relock_pull_request_script.py diff --git a/.github/scripts/open_relock_pull_request.sh b/.github/scripts/open_relock_pull_request.sh new file mode 100644 index 000000000..d086a5929 --- /dev/null +++ b/.github/scripts/open_relock_pull_request.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash + +# Commit a relocked uv.lock with a changelog fragment, push it to the relock +# branch, and open the relock pull request, or update the open one. +# +# - The branch is rebuilt on every run from the checked-out commit of the base +# branch, so its diff is always the base branch plus one relock. The push +# replaces the branch (with a lease on the commit it last saw), and is +# skipped when the branch already holds the same files. +# - The tracking issue is the oldest open issue titled "Relock dependencies", +# or a new one when none is open. The first line of the pull request +# description is "Fixes #N", so merging the pull request closes it and the +# next relock opens a new one. +# - The description lists the version changes from RELOCK_SUMMARY_FILE. +# +# Run from the repository root, after relock_dependencies.sh reported a change. +# +# Environment: +# GITHUB_REPOSITORY owner/name of this repository (set by the runner) +# GH_TOKEN read by gh: a token that can write issues and pull +# requests. git pushes with the credentials the checkout +# stored. +# RELOCK_SUMMARY_FILE Markdown summary written by relock_dependencies.sh +# BASE_BRANCH branch the pull request merges into (default: master) +# RELOCK_BRANCH branch the pull request is opened from +# (default: auto/relock-dependencies) + +set -euo pipefail + +: "${GITHUB_REPOSITORY:?GITHUB_REPOSITORY is required}" +: "${RELOCK_SUMMARY_FILE:?RELOCK_SUMMARY_FILE is required}" +if [[ ! -r "${RELOCK_SUMMARY_FILE}" ]]; then + echo "Relock summary ${RELOCK_SUMMARY_FILE} is missing or unreadable." >&2 + exit 1 +fi + +base_branch="${BASE_BRANCH:-master}" +relock_branch="${RELOCK_BRANCH:-auto/relock-dependencies}" +title="Relock dependencies" +fragment="changelog.d/relock-dependencies.changed.md" +issue_body="Tracks the daily relock of uv.lock by .github/workflows/relock-dependencies.yml. The relock pull request names this issue on the first line of its description and closes it when it merges." + +# Set ISSUE_NUMBER to the oldest open tracking issue, opening one when none is +# open. This sets a variable rather than printing, because bash turns off +# set -e inside $(...), and a failed lookup must not open a duplicate issue. +find_or_open_tracking_issue() { + local issue_number + local issue_url + + issue_number="$( + gh issue list \ + --repo "${GITHUB_REPOSITORY}" \ + --state open \ + --limit 1000 \ + --json number,title \ + --jq "map(select(.title == \"${title}\")) | sort_by(.number) | .[0].number // empty" + )" + if [[ -z "${issue_number}" ]]; then + issue_url="$( + gh issue create \ + --repo "${GITHUB_REPOSITORY}" \ + --title "${title}" \ + --body "${issue_body}" + )" + issue_number="${issue_url##*/}" + fi + + if [[ ! "${issue_number}" =~ ^[0-9]+$ ]]; then + echo "Could not resolve the relock tracking issue: ${issue_number}" >&2 + exit 1 + fi + ISSUE_NUMBER="${issue_number}" +} + +pr_description() { + cat <"${fragment}" +git checkout --quiet -B "${relock_branch}" +git add uv.lock "${fragment}" +git commit --quiet -m "Relock dependencies" + +remote_commit="$(git ls-remote --heads origin "refs/heads/${relock_branch}" | cut -f1)" +if [[ -n "${remote_commit}" ]]; then + git fetch --quiet origin "refs/heads/${relock_branch}" +fi +if [[ -n "${remote_commit}" ]] && git diff --quiet "${remote_commit}" HEAD --; then + echo "${relock_branch} already holds this relock; not pushing." +else + git push --quiet \ + --force-with-lease="refs/heads/${relock_branch}:${remote_commit}" \ + origin "HEAD:refs/heads/${relock_branch}" +fi + +description="$(pr_description)" +if [[ -z "${open_pr}" ]]; then + gh pr create \ + --repo "${GITHUB_REPOSITORY}" \ + --base "${base_branch}" \ + --head "${relock_branch}" \ + --title "${title}" \ + --body-file - \ + <<<"${description}" +else + gh pr edit "${open_pr}" \ + --repo "${GITHUB_REPOSITORY}" \ + --body-file - \ + <<<"${description}" + echo "Updated pull request #${open_pr}." +fi diff --git a/tests/unit/test_relock_pull_request_script.py b/tests/unit/test_relock_pull_request_script.py new file mode 100644 index 000000000..952b631d1 --- /dev/null +++ b/tests/unit/test_relock_pull_request_script.py @@ -0,0 +1,252 @@ +"""Tests for .github/scripts/open_relock_pull_request.sh. + +Each test builds a temporary git repository whose ``origin`` is a local bare +repository, so pushes are real but stay on disk. A fake ``gh`` records its +calls and answers the pull-request and issue lookups. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path + +import pytest + +REPO = Path(__file__).resolve().parents[2] +SCRIPT = REPO / ".github/scripts/open_relock_pull_request.sh" +RELOCK_BRANCH = "auto/relock-dependencies" +FRAGMENT = "changelog.d/relock-dependencies.changed.md" +ISSUE_URL = "https://github.com/PolicyEngine/policyengine-api/issues/77" +SUMMARY = "## Version changes\n\n- `numpy`: 2.3.0 → 2.3.1\n" + +FAKE_GH = f"""#!{sys.executable} +import json +import os +import sys + +arguments = sys.argv[1:] +body = sys.stdin.read() if "--body-file" in arguments else None +with open(os.environ["RELOCK_TEST_CALLS"], "a") as calls: + calls.write(json.dumps({{"arguments": ["gh", *arguments], "body": body}}) + "\\n") +if arguments[:2] == ["pr", "list"]: + print(os.environ.get("RELOCK_TEST_OPEN_PR", "")) +elif arguments[:2] == ["issue", "list"]: + if os.environ.get("RELOCK_TEST_ISSUE_LIST_EXIT"): + sys.exit(1) + print(os.environ.get("RELOCK_TEST_OPEN_ISSUE", "")) +elif arguments[:2] == ["issue", "create"]: + print("{ISSUE_URL}") +""" + + +def _git(directory: Path, *arguments: str) -> str: + return subprocess.run( + ["git", *arguments], + cwd=directory, + env=_git_environment(), + text=True, + capture_output=True, + check=True, + ).stdout.strip() + + +def _git_environment() -> dict[str, str]: + """Return an environment that ignores the user's and system git settings.""" + return { + **os.environ, + "GIT_CONFIG_GLOBAL": os.devnull, + "GIT_CONFIG_NOSYSTEM": "1", + "GIT_AUTHOR_NAME": "Test", + "GIT_AUTHOR_EMAIL": "test@example.com", + "GIT_COMMITTER_NAME": "Test", + "GIT_COMMITTER_EMAIL": "test@example.com", + } + + +@pytest.fixture +def checkout(tmp_path: Path) -> Path: + """Return a master checkout with a relocked uv.lock and a local origin.""" + origin = tmp_path / "origin.git" + _git(tmp_path, "init", "--quiet", "--bare", "--initial-branch=master", str(origin)) + repository = tmp_path / "checkout" + repository.mkdir() + _git(repository, "init", "--quiet", "--initial-branch=master") + (repository / "changelog.d").mkdir() + (repository / "changelog.d/.gitkeep").write_text("") + (repository / "uv.lock").write_text('version = 1\nname = "numpy-2.3.0"\n') + _git(repository, "add", ".") + _git(repository, "commit", "--quiet", "-m", "Initial commit") + _git(repository, "remote", "add", "origin", str(origin)) + _git(repository, "push", "--quiet", "origin", "master") + (repository / "uv.lock").write_text('version = 1\nname = "numpy-2.3.1"\n') + return repository + + +def _open_pull_request( + repository: Path, **environment: str +) -> tuple[subprocess.CompletedProcess[str], list[dict]]: + """Run the script with a fake gh; return the result and the gh calls.""" + binaries = repository.parent / "bin" + binaries.mkdir(exist_ok=True) + fake_gh = binaries / "gh" + fake_gh.write_text(FAKE_GH) + fake_gh.chmod(0o755) + summary = repository.parent / "summary.md" + summary.write_text(SUMMARY) + calls_log = repository.parent / "gh-calls.jsonl" + calls_log.unlink(missing_ok=True) + run_environment = { + **_git_environment(), + "PATH": f"{binaries}{os.pathsep}{os.environ['PATH']}", + "GITHUB_REPOSITORY": "PolicyEngine/policyengine-api", + "RELOCK_SUMMARY_FILE": str(summary), + "RELOCK_TEST_CALLS": str(calls_log), + } + for name in ("BASE_BRANCH", "RELOCK_BRANCH"): + run_environment.pop(name, None) + run_environment.update(environment) + result = subprocess.run( + ["bash", str(SCRIPT)], + cwd=repository, + env=run_environment, + text=True, + capture_output=True, + check=False, + ) + calls = ( + [json.loads(call) for call in calls_log.read_text().splitlines()] + if calls_log.exists() + else [] + ) + return result, calls + + +def _calls_to(calls: list[dict], *command: str) -> list[dict]: + return [call for call in calls if call["arguments"][1:3] == list(command)] + + +def _remote_branch(repository: Path, branch: str = RELOCK_BRANCH) -> str: + return _git(repository, "ls-remote", "--heads", "origin", f"refs/heads/{branch}") + + +def test_opens_a_pull_request_naming_a_new_tracking_issue(checkout: Path) -> None: + result, calls = _open_pull_request(checkout) + + assert result.returncode == 0, result.stderr + pushed = _remote_branch(checkout).split()[0] + assert _git(checkout, "show", f"{pushed}:uv.lock").endswith('"numpy-2.3.1"') + assert _git(checkout, "show", f"{pushed}:{FRAGMENT}") + assert _git(checkout, "rev-parse", f"{pushed}^") == _git( + checkout, "rev-parse", "origin/master" + ) + (created_issue,) = _calls_to(calls, "issue", "create") + assert "Relock dependencies" in created_issue["arguments"] + (created_pr,) = _calls_to(calls, "pr", "create") + arguments = created_pr["arguments"] + assert arguments[arguments.index("--base") + 1] == "master" + assert arguments[arguments.index("--head") + 1] == RELOCK_BRANCH + assert arguments[arguments.index("--title") + 1] == "Relock dependencies" + assert created_pr["body"].splitlines()[0] == "Fixes #77" + assert SUMMARY in created_pr["body"] + assert not _calls_to(calls, "pr", "edit") + + +def test_reuses_the_open_tracking_issue(checkout: Path) -> None: + result, calls = _open_pull_request(checkout, RELOCK_TEST_OPEN_ISSUE="41") + + assert result.returncode == 0, result.stderr + (issue_lookup,) = _calls_to(calls, "issue", "list") + assert any('"Relock dependencies"' in part for part in issue_lookup["arguments"]) + assert not _calls_to(calls, "issue", "create") + (created_pr,) = _calls_to(calls, "pr", "create") + assert created_pr["body"].splitlines()[0] == "Fixes #41" + + +def test_updates_the_open_pull_request_instead_of_opening_another( + checkout: Path, +) -> None: + first_result, _ = _open_pull_request(checkout, RELOCK_TEST_OPEN_ISSUE="41") + assert first_result.returncode == 0, first_result.stderr + first_push = _remote_branch(checkout).split()[0] + _git(checkout, "checkout", "--quiet", "master") + (checkout / "uv.lock").write_text('version = 1\nname = "numpy-2.4.0"\n') + + result, calls = _open_pull_request( + checkout, RELOCK_TEST_OPEN_PR="12", RELOCK_TEST_OPEN_ISSUE="41" + ) + + assert result.returncode == 0, result.stderr + pushed = _remote_branch(checkout).split()[0] + assert pushed != first_push + assert _git(checkout, "show", f"{pushed}:uv.lock").endswith('"numpy-2.4.0"') + assert _git(checkout, "rev-parse", f"{pushed}^") == _git( + checkout, "rev-parse", "origin/master" + ) + assert not _calls_to(calls, "pr", "create") + (edited_pr,) = _calls_to(calls, "pr", "edit") + assert edited_pr["arguments"][3] == "12" + assert edited_pr["body"].splitlines()[0] == "Fixes #41" + + +def test_does_not_push_when_the_branch_already_holds_this_relock( + checkout: Path, +) -> None: + first_result, _ = _open_pull_request(checkout, RELOCK_TEST_OPEN_ISSUE="41") + assert first_result.returncode == 0, first_result.stderr + first_push = _remote_branch(checkout) + _git(checkout, "checkout", "--quiet", "master") + (checkout / "uv.lock").write_text('version = 1\nname = "numpy-2.3.1"\n') + + result, calls = _open_pull_request( + checkout, RELOCK_TEST_OPEN_PR="12", RELOCK_TEST_OPEN_ISSUE="41" + ) + + assert result.returncode == 0, result.stderr + assert _remote_branch(checkout) == first_push + assert "already holds this relock" in result.stdout + assert len(_calls_to(calls, "pr", "edit")) == 1 + + +def test_fails_before_pushing_when_the_issue_lookup_fails(checkout: Path) -> None: + result, calls = _open_pull_request(checkout, RELOCK_TEST_ISSUE_LIST_EXIT="1") + + assert result.returncode != 0 + assert _calls_to(calls, "issue", "list") + assert _remote_branch(checkout) == "" + assert not _calls_to(calls, "issue", "create") + assert not _calls_to(calls, "pr", "create") + + +def test_uses_the_base_and_relock_branches_it_is_given(checkout: Path) -> None: + _git(checkout, "push", "--quiet", "origin", "master:feature") + feature_relock = "auto/relock-dependencies-feature" + + result, calls = _open_pull_request( + checkout, BASE_BRANCH="feature", RELOCK_BRANCH=feature_relock + ) + + assert result.returncode == 0, result.stderr + assert _remote_branch(checkout, feature_relock) + assert _remote_branch(checkout) == "" + (pr_lookup,) = _calls_to(calls, "pr", "list") + lookup = pr_lookup["arguments"] + assert lookup[lookup.index("--base") + 1] == "feature" + assert lookup[lookup.index("--head") + 1] == feature_relock + (created_pr,) = _calls_to(calls, "pr", "create") + arguments = created_pr["arguments"] + assert arguments[arguments.index("--base") + 1] == "feature" + assert arguments[arguments.index("--head") + 1] == feature_relock + + +def test_fails_before_any_change_without_the_summary(checkout: Path) -> None: + missing_summary = str(checkout.parent / "missing-summary.md") + + result, calls = _open_pull_request(checkout, RELOCK_SUMMARY_FILE=missing_summary) + + assert result.returncode != 0 + assert "missing-summary.md" in result.stderr + assert calls == [] + assert _remote_branch(checkout) == "" From aaaf86b21989fceb7bd82970b173450f0e03b2ba Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 00:01:37 +0400 Subject: [PATCH 3/4] Relock uv.lock daily and propose the result as a pull request relock-dependencies.yml runs at 07:00 UTC every day and on manual dispatch. It checks out the branch with the policyengine App's token, relocks with Python 3.12 and the uv version push.yml locks releases with, and runs open_relock_pull_request.sh only when uv.lock changed, so pull_request CI runs on the result. A run on master proposes the relock from auto/relock-dependencies; a manual run on another branch proposes it to that branch from a branch of its own. The workflow token is read-only. The relock branch is now fetched with --depth=1, so a shallow checkout does not download the history behind an older relock. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/open_relock_pull_request.sh | 2 +- .github/workflows/relock-dependencies.yml | 72 +++++++++++ ...to-relock-dependencies-workflow.changed.md | 1 + .../unit/test_relock_dependencies_workflow.py | 114 ++++++++++++++++++ 4 files changed, 188 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/relock-dependencies.yml create mode 100644 changelog.d/auto-relock-dependencies-workflow.changed.md create mode 100644 tests/unit/test_relock_dependencies_workflow.py diff --git a/.github/scripts/open_relock_pull_request.sh b/.github/scripts/open_relock_pull_request.sh index d086a5929..954255406 100644 --- a/.github/scripts/open_relock_pull_request.sh +++ b/.github/scripts/open_relock_pull_request.sh @@ -107,7 +107,7 @@ git commit --quiet -m "Relock dependencies" remote_commit="$(git ls-remote --heads origin "refs/heads/${relock_branch}" | cut -f1)" if [[ -n "${remote_commit}" ]]; then - git fetch --quiet origin "refs/heads/${relock_branch}" + git fetch --quiet --depth=1 origin "refs/heads/${relock_branch}" fi if [[ -n "${remote_commit}" ]] && git diff --quiet "${remote_commit}" HEAD --; then echo "${relock_branch} already holds this relock; not pushing." diff --git a/.github/workflows/relock-dependencies.yml b/.github/workflows/relock-dependencies.yml new file mode 100644 index 000000000..6ab8fe9de --- /dev/null +++ b/.github/workflows/relock-dependencies.yml @@ -0,0 +1,72 @@ +name: Relock dependencies + +# Once a day, relock uv.lock to the newest versions pyproject.toml allows and +# propose the result as a pull request, whose CI tests it. The PolicyEngine .py +# bundle and the packages its extras pin are held at their locked versions: +# update-policyengine-bundle.yaml moves them. When the relock changes nothing, +# no pull request is opened. + +on: + schedule: + # Every day at 07:00 UTC. + - cron: "0 7 * * *" + workflow_dispatch: + +concurrency: + group: relock-dependencies-${{ github.ref_name }} + cancel-in-progress: false + +# The workflow token only reads; the policyengine App's token pushes the +# branch and writes the issue and pull request, so pull_request CI runs. +permissions: + contents: read + +jobs: + relock: + name: Relock dependencies + if: github.repository == 'PolicyEngine/policyengine-api' + runs-on: ubuntu-latest + env: + # A scheduled run relocks master and proposes it from the fixed branch. A + # manual run on another branch relocks that branch and proposes it to + # that branch from a branch of its own. + BASE_BRANCH: ${{ github.ref_name }} + RELOCK_BRANCH: ${{ github.ref_name == 'master' && 'auto/relock-dependencies' || format('auto/relock-dependencies-{0}', github.ref_name) }} + # The Public API runs on Python 3.12; .python-version names a newer one. + RELOCK_PYTHON: "3.12" + steps: + - name: Generate GitHub App token + id: app-token + uses: actions/create-github-app-token@v1 + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + + - name: Checkout repo + uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.12" + + # The same uv version as push.yml's versioning job, which locks releases. + - name: Set up uv + uses: astral-sh/setup-uv@v6 + with: + version: "0.13.4" + + - name: Relock uv.lock, holding the PolicyEngine bundle + id: relock + env: + RELOCK_SUMMARY_FILE: ${{ runner.temp }}/relock-summary.md + run: bash .github/scripts/relock_dependencies.sh + + - name: Open or update the relock pull request + if: steps.relock.outputs.changed == 'true' + env: + GH_TOKEN: ${{ steps.app-token.outputs.token }} + RELOCK_SUMMARY_FILE: ${{ runner.temp }}/relock-summary.md + run: bash .github/scripts/open_relock_pull_request.sh diff --git a/changelog.d/auto-relock-dependencies-workflow.changed.md b/changelog.d/auto-relock-dependencies-workflow.changed.md new file mode 100644 index 000000000..e98d25bd6 --- /dev/null +++ b/changelog.d/auto-relock-dependencies-workflow.changed.md @@ -0,0 +1 @@ +Added a daily workflow that relocks uv.lock, holding the PolicyEngine bundle and the packages it pins, and opens or updates a pull request with the version changes. diff --git a/tests/unit/test_relock_dependencies_workflow.py b/tests/unit/test_relock_dependencies_workflow.py new file mode 100644 index 000000000..eca73f224 --- /dev/null +++ b/tests/unit/test_relock_dependencies_workflow.py @@ -0,0 +1,114 @@ +"""Tests for the wiring of .github/workflows/relock-dependencies.yml. + +The scripts it runs are tested in test_relock_dependencies_script.py and +test_relock_pull_request_script.py. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest +import yaml + +REPO = Path(__file__).resolve().parents[2] +WORKFLOW = REPO / ".github/workflows/relock-dependencies.yml" +PUSH_WORKFLOW = REPO / ".github/workflows/push.yml" +APP_TOKEN = "${{ steps.app-token.outputs.token }}" +SUMMARY_FILE = "${{ runner.temp }}/relock-summary.md" + + +@pytest.fixture(scope="module") +def workflow() -> dict: + return yaml.safe_load(WORKFLOW.read_text()) + + +@pytest.fixture(scope="module") +def relock_job(workflow: dict) -> dict: + return workflow["jobs"]["relock"] + + +def _step(job: dict, *, uses: str | None = None, run: str | None = None) -> dict: + """Return the one step that uses the ``uses`` action or runs ``run``.""" + if uses is not None: + matches = [ + step for step in job["steps"] if step.get("uses", "").startswith(uses) + ] + else: + matches = [step for step in job["steps"] if step.get("run") == run] + (step,) = matches + return step + + +def test_runs_daily_and_on_manual_dispatch(workflow: dict) -> None: + # PyYAML reads the bare key "on" as the boolean True. + triggers = workflow[True] + + assert set(triggers) == {"schedule", "workflow_dispatch"} + (schedule,) = triggers["schedule"] + minute, hour, day, month, weekday = schedule["cron"].split() + assert (day, month, weekday) == ("*", "*", "*") + assert minute.isdigit() + assert hour.isdigit() + + +def test_runs_only_in_the_canonical_repository_with_a_read_only_token( + workflow: dict, relock_job: dict +) -> None: + assert relock_job["if"] == "github.repository == 'PolicyEngine/policyengine-api'" + assert workflow["permissions"] == {"contents": "read"} + assert workflow["concurrency"]["cancel-in-progress"] is False + + +def test_checks_out_and_opens_the_pull_request_with_the_app_token( + relock_job: dict, +) -> None: + token_step = _step(relock_job, uses="actions/create-github-app-token@") + assert token_step["id"] == "app-token" + assert token_step["with"] == { + "app-id": "${{ secrets.APP_ID }}", + "private-key": "${{ secrets.APP_PRIVATE_KEY }}", + } + checkout = _step(relock_job, uses="actions/checkout@") + assert checkout["with"]["token"] == APP_TOKEN + + pull_request_step = _step( + relock_job, run="bash .github/scripts/open_relock_pull_request.sh" + ) + assert pull_request_step["env"]["GH_TOKEN"] == APP_TOKEN + + +def test_relocks_with_python_312_and_the_uv_version_that_locks_releases( + relock_job: dict, +) -> None: + setup_python = _step(relock_job, uses="actions/setup-python@") + assert setup_python["with"]["python-version"] == "3.12" + assert relock_job["env"]["RELOCK_PYTHON"] == "3.12" + + push_jobs = yaml.safe_load(PUSH_WORKFLOW.read_text())["jobs"] + release_uv = _step(push_jobs["versioning"], uses="astral-sh/setup-uv@") + relock_uv = _step(relock_job, uses="astral-sh/setup-uv@") + assert relock_uv["uses"] == release_uv["uses"] + assert relock_uv["with"]["version"] == release_uv["with"]["version"] + + +def test_opens_a_pull_request_only_when_the_lock_changed(relock_job: dict) -> None: + relock_step = _step(relock_job, run="bash .github/scripts/relock_dependencies.sh") + pull_request_step = _step( + relock_job, run="bash .github/scripts/open_relock_pull_request.sh" + ) + + assert relock_step["id"] == "relock" + assert pull_request_step["if"] == "steps.relock.outputs.changed == 'true'" + assert relock_step["env"]["RELOCK_SUMMARY_FILE"] == SUMMARY_FILE + assert pull_request_step["env"]["RELOCK_SUMMARY_FILE"] == SUMMARY_FILE + steps = relock_job["steps"] + assert steps.index(relock_step) < steps.index(pull_request_step) + + +def test_proposes_master_relocks_from_the_fixed_branch(relock_job: dict) -> None: + assert relock_job["env"]["BASE_BRANCH"] == "${{ github.ref_name }}" + assert relock_job["env"]["RELOCK_BRANCH"] == ( + "${{ github.ref_name == 'master' && 'auto/relock-dependencies' " + "|| format('auto/relock-dependencies-{0}', github.ref_name) }}" + ) From f49df924f3f1a75fdd81ed2b653b01abef68d268 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 00:05:27 +0400 Subject: [PATCH 4/4] Ignore fork pull requests when finding the relock pull request gh pr list --head matches the head branch name in forks too, so a fork pull request from a branch named auto/relock-dependencies would have been treated as the relock pull request: its description edited, and no pull request opened for the pushed branch. The lookup now selects only pull requests whose head is in this repository. Co-Authored-By: Claude Opus 5.5 --- .github/scripts/open_relock_pull_request.sh | 6 ++++-- tests/unit/test_relock_pull_request_script.py | 14 ++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/scripts/open_relock_pull_request.sh b/.github/scripts/open_relock_pull_request.sh index 954255406..5c8d7c6a6 100644 --- a/.github/scripts/open_relock_pull_request.sh +++ b/.github/scripts/open_relock_pull_request.sh @@ -86,14 +86,16 @@ EOF cat "${RELOCK_SUMMARY_FILE}" } +# --head matches the branch name in forks too; only this repository's branch +# is the relock branch. open_pr="$( gh pr list \ --repo "${GITHUB_REPOSITORY}" \ --base "${base_branch}" \ --head "${relock_branch}" \ --state open \ - --json number \ - --jq '.[0].number // empty' + --json number,isCrossRepository \ + --jq 'map(select(.isCrossRepository | not)) | .[0].number // empty' )" ISSUE_NUMBER="" find_or_open_tracking_issue diff --git a/tests/unit/test_relock_pull_request_script.py b/tests/unit/test_relock_pull_request_script.py index 952b631d1..dc6241d20 100644 --- a/tests/unit/test_relock_pull_request_script.py +++ b/tests/unit/test_relock_pull_request_script.py @@ -250,3 +250,17 @@ def test_fails_before_any_change_without_the_summary(checkout: Path) -> None: assert "missing-summary.md" in result.stderr assert calls == [] assert _remote_branch(checkout) == "" + + +def test_ignores_pull_requests_from_forks_with_the_same_branch_name( + checkout: Path, +) -> None: + result, calls = _open_pull_request(checkout, RELOCK_TEST_OPEN_ISSUE="41") + + assert result.returncode == 0, result.stderr + (pr_lookup,) = _calls_to(calls, "pr", "list") + lookup = pr_lookup["arguments"] + assert "isCrossRepository" in lookup[lookup.index("--json") + 1] + assert lookup[lookup.index("--jq") + 1] == ( + "map(select(.isCrossRepository | not)) | .[0].number // empty" + )