From 708411db3876a06397995fde01a7c4158f97663c Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 8 Nov 2026 09:42:50 +0800 Subject: [PATCH 1/2] ci: require matching main proofs before publication Validate CI and native evidence for the landed source and its current run attempts before authorizing publication. Cover native test registration and results, pending counterparts, immutable evidence identity, missing metadata and validator refusal behavior. --- .github/workflows/ci-post.yml | 48 +- .github/workflows/native-feed-post.yml | 143 +- .github/workflows/release.yml | 2 +- scripts/release_chain/evidence.py | 302 ++++- scripts/release_chain/main_proofs.py | 333 +++++ scripts/release_chain/publishing.py | 91 +- scripts/release_chain/reporting.py | 17 +- tests/test_consumer_receipt.py | 1685 +++++++++++++++++++++++- tests/test_main_proofs.py | 892 +++++++++++++ tests/test_post_merge_workflows.py | 383 +++++- tests/test_release_adapter.py | 907 ++++++++++++- 11 files changed, 4704 insertions(+), 99 deletions(-) create mode 100644 scripts/release_chain/main_proofs.py create mode 100644 tests/test_main_proofs.py diff --git a/.github/workflows/ci-post.yml b/.github/workflows/ci-post.yml index 0b8ec50..92c7b51 100644 --- a/.github/workflows/ci-post.yml +++ b/.github/workflows/ci-post.yml @@ -182,6 +182,37 @@ jobs: - if: matrix.python-version == '3.13' run: python -m twine check --strict dist/* + network-proof: + name: Live PyPI hashes + needs: [authz, main-ref] + runs-on: ubuntu-24.04 + env: + PF_CI_LABELS: '["ubuntu-24.04"]' + PINEFORGE_NETWORK_TEST: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.13' + cache: pip + - run: python -m pip install -e '.[dev]' pytest-xdist + - name: Execute the required live PyPI hash row with one actual worker + run: pytest -n 1 tests/test_consumer_receipt.py::test_bootstrap_recorded_hashes_match_live_pypi --junitxml=network.xml + - name: Derive the required zero-skip inventory from the actual JUnit + env: + PYTHONPATH: scripts + run: python -m release_chain.main_proofs network --directory . --output network-proof.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: network-proof-${{ github.run_attempt }} + path: | + network.xml + network-proof.json + if-no-files-found: error + docker: name: Docker runtime needs: [authz, main-ref] @@ -399,7 +430,7 @@ jobs: ready: name: release/ready - needs: [authz, main-ref, test, contract, docker, database-e2e, distribution] + needs: [authz, main-ref, test, contract, docker, database-e2e, distribution, network-proof] runs-on: ubuntu-24.04 timeout-minutes: 15 env: @@ -421,6 +452,14 @@ jobs: RUN_ID: ${{ github.run_id }} ATTEMPT: ${{ github.run_attempt }} run: gh run download "$RUN_ID" --name "python-package-distributions-$ATTEMPT" --dir dist + - name: Download this attempt's actual live PyPI report + if: needs.authz.result == 'success' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + ATTEMPT: ${{ github.run_attempt }} + run: gh run download "$RUN_ID" --name "network-proof-$ATTEMPT" --dir reports/network - name: Require the complete job inventory and bind evidence to this head/base/tree if: needs.authz.result == 'success' env: @@ -428,9 +467,10 @@ jobs: NEEDS_JSON: >- {"test":${{ toJSON(needs.test) }},"contract":${{ toJSON(needs.contract) }}, "docker":${{ toJSON(needs.docker) }},"database-e2e":${{ toJSON(needs.database-e2e) }}, - "distribution":${{ toJSON(needs.distribution) }}} + "distribution":${{ toJSON(needs.distribution) }}, + "network-proof":${{ toJSON(needs.network-proof) }}} GH_TOKEN: ${{ github.token }} - run: python -m release_chain.evidence aggregate --path dist + run: python -m release_chain.evidence aggregate --path dist --network-proof reports/network - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: needs.authz.result == 'success' with: @@ -439,7 +479,7 @@ jobs: if-no-files-found: error check: - needs: [authz, main-ref, workflow-lint, test, docker, database-e2e, contract, distribution, docs-tests, ready] + needs: [authz, main-ref, workflow-lint, test, docker, database-e2e, contract, distribution, docs-tests, network-proof, ready] if: ${{ always() }} runs-on: ubuntu-24.04 timeout-minutes: 10 diff --git a/.github/workflows/native-feed-post.yml b/.github/workflows/native-feed-post.yml index 837100a..19b4ec6 100644 --- a/.github/workflows/native-feed-post.yml +++ b/.github/workflows/native-feed-post.yml @@ -4,7 +4,6 @@ name: Native feed on: push: branches: [main] - paths: ['native/feed/**', '.github/workflows/native-feed.yml', '.github/workflows/native-feed-post.yml'] workflow_dispatch: permissions: @@ -111,12 +110,30 @@ jobs: run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl" - name: Configure, build and test run: | + set -euo pipefail export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" export DYLD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${DYLD_LIBRARY_PATH:-}" cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \ -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" cmake --build build-feed -j4 - ctest --test-dir build-feed --output-on-failure + ctest --test-dir build-feed --show-only=json-v1 > native-registration.json + ctest --test-dir build-feed --output-on-failure --output-junit "$GITHUB_WORKSPACE/native-junit.xml" + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.13' + - name: Bind the actual registration and JUnit to this leg + env: + PYTHONPATH: scripts + run: python -m release_chain.main_proofs native-leg --leg release-amd64 --directory . --output native-leg.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: native-leg-release-amd64-${{ github.run_attempt }} + path: | + native-registration.json + native-junit.xml + native-leg.json + if-no-files-found: error release-arm64: name: Release (ubuntu-24.04-arm) @@ -139,12 +156,30 @@ jobs: run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl" - name: Configure, build and test run: | + set -euo pipefail export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" export DYLD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${DYLD_LIBRARY_PATH:-}" cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \ -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" cmake --build build-feed -j4 - ctest --test-dir build-feed --output-on-failure + ctest --test-dir build-feed --show-only=json-v1 > native-registration.json + ctest --test-dir build-feed --output-on-failure --output-junit "$GITHUB_WORKSPACE/native-junit.xml" + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.13' + - name: Bind the actual registration and JUnit to this leg + env: + PYTHONPATH: scripts + run: python -m release_chain.main_proofs native-leg --leg release-arm64 --directory . --output native-leg.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: native-leg-release-arm64-${{ github.run_attempt }} + path: | + native-registration.json + native-junit.xml + native-leg.json + if-no-files-found: error release-macos: name: Release (macos-15) @@ -167,12 +202,30 @@ jobs: run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl" - name: Configure, build and test run: | + set -euo pipefail export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" export DYLD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${DYLD_LIBRARY_PATH:-}" cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \ -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" cmake --build build-feed -j4 - ctest --test-dir build-feed --output-on-failure + ctest --test-dir build-feed --show-only=json-v1 > native-registration.json + ctest --test-dir build-feed --output-on-failure --output-junit "$GITHUB_WORKSPACE/native-junit.xml" + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.13' + - name: Bind the actual registration and JUnit to this leg + env: + PYTHONPATH: scripts + run: python -m release_chain.main_proofs native-leg --leg release-macos --directory . --output native-leg.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: native-leg-release-macos-${{ github.run_attempt }} + path: | + native-registration.json + native-junit.xml + native-leg.json + if-no-files-found: error asan-ubsan: name: ASan + UBSan @@ -190,12 +243,30 @@ jobs: ASAN_OPTIONS: detect_leaks=1:abort_on_error=1 UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1 run: | + set -euo pipefail export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_C_COMPILER=clang-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \ -DPINEFORGE_FEED_SANITIZER=address,undefined cmake --build build-feed -j4 - ctest --test-dir build-feed --output-on-failure + ctest --test-dir build-feed --show-only=json-v1 > native-registration.json + ctest --test-dir build-feed --output-on-failure --output-junit "$GITHUB_WORKSPACE/native-junit.xml" + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.13' + - name: Bind the actual registration and JUnit to this leg + env: + PYTHONPATH: scripts + run: python -m release_chain.main_proofs native-leg --leg asan-ubsan --directory . --output native-leg.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: native-leg-asan-ubsan-${{ github.run_attempt }} + path: | + native-registration.json + native-junit.xml + native-leg.json + if-no-files-found: error tsan: name: TSan @@ -220,10 +291,70 @@ jobs: env: TSAN_OPTIONS: halt_on_error=1 run: | + set -euo pipefail export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_C_COMPILER=clang-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \ -DPINEFORGE_FEED_SANITIZER=thread \ -DPINEFORGE_FEED_TSAN_OPTIONS_EXTRA=external_symbolizer_path=/usr/lib/llvm-18/bin/llvm-symbolizer cmake --build build-feed -j4 - ctest --test-dir build-feed --output-on-failure + ctest --test-dir build-feed --show-only=json-v1 > native-registration.json + ctest --test-dir build-feed --output-on-failure --output-junit "$GITHUB_WORKSPACE/native-junit.xml" + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.13' + - name: Bind the actual registration and JUnit to this leg + env: + PYTHONPATH: scripts + run: python -m release_chain.main_proofs native-leg --leg tsan --directory . --output native-leg.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: native-leg-tsan-${{ github.run_attempt }} + path: | + native-registration.json + native-junit.xml + native-leg.json + if-no-files-found: error + + native-proof: + name: Native proof + needs: [authz, main-ref, release-amd64, release-arm64, release-macos, asan-ubsan, tsan] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.13' + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: native-leg-release-amd64-${{ github.run_attempt }} + path: native-reports/release-amd64 + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: native-leg-release-arm64-${{ github.run_attempt }} + path: native-reports/release-arm64 + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: native-leg-release-macos-${{ github.run_attempt }} + path: native-reports/release-macos + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: native-leg-asan-ubsan-${{ github.run_attempt }} + path: native-reports/asan-ubsan + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: native-leg-tsan-${{ github.run_attempt }} + path: native-reports/tsan + - name: Require authoritative leg results and reparse their actual reports + env: + PYTHONPATH: scripts + NEEDS_JSON: ${{ toJSON(needs) }} + run: python -m release_chain.main_proofs native-aggregate --directory native-reports --output native-evidence.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-evidence-${{ github.run_attempt }} + path: native-evidence.json + if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 49fac1e..2f766b0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,7 +23,7 @@ on: required: false type: boolean workflow_run: - workflows: [CI] + workflows: [CI, Native feed] types: [completed] branches: [main] diff --git a/scripts/release_chain/evidence.py b/scripts/release_chain/evidence.py index 062d0e2..03c8607 100644 --- a/scripts/release_chain/evidence.py +++ b/scripts/release_chain/evidence.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy import io import json import os @@ -11,14 +12,24 @@ from typing import Any from xml.etree import ElementTree -from .contract import gh_bytes, json_bytes, object_json, require, sha256 +from .contract import HEX40, gh_bytes, json_bytes, object_json, require, sha256 +from .main_proofs import ( + NATIVE_WORKFLOW, + NETWORK_JOB, + NETWORK_NODE, + consume_network, + successful_jobs, + validate_identity, + validate_main, +) +from .main_proofs import validate_inventory as validate_proof_inventory # SHA256 of both reviewed CI definitions: no continue-on-error, matrix leg names = JOBS. # Edits must update the pin in the same PR after re-review: needs..result == "success" proves # every leg succeeded (2026-10-23, VP-145 review P0: a lexical test alone misses escaped YAML keys). REVIEWED_CI_WORKFLOW_SHA256 = "e2b41563fbb8ab32c3c16675fc3d411b9dade719593ab0157fd47d641062ea2a" REVIEWED_CI_POST_WORKFLOW_SHA256 = ( - "4551f97c4d3f2590dc62cb7ce5652991d7ea17f3ba92f6af880af8269efe58ad" + "b8d3d92bb4b106f8985f8bbe4d96f12bff67d4d2b71b5bbf07bb46c76f9a8d49" ) JOBS = ( @@ -31,6 +42,7 @@ "Distribution", ) NEEDS = {"test", "contract", "docker", "database-e2e", "distribution"} +PENDING_STATUSES = {"queued", "in_progress", "requested", "waiting", "pending"} TESTS = { "docker": { "test_local_runtime_uses_published_release_image", @@ -46,20 +58,104 @@ class GateRejected(ValueError): - def __init__(self, message: str, run: dict[str, Any], jobs: list[dict[str, Any]]) -> None: + def __init__( + self, message: str, run: dict[str, Any] | None, jobs: list[dict[str, Any]] | None + ) -> None: super().__init__(message) + self.run = run + self.jobs = jobs self.evidence = { - "run_id": run["id"], - "attempt": run["run_attempt"], - "head": run["head_sha"], + "run_id": run.get("id") if run is not None else None, + "attempt": run.get("run_attempt") if run is not None else None, + "head": run.get("head_sha") if run is not None else None, "jobs": { name: next( - (job["conclusion"] or job["status"] for job in jobs if job["name"] == name), + ( + job.get("conclusion") or job.get("status") + for job in jobs if job.get("name") == name + ), "missing", ) for name in (*JOBS, "release/ready") - }, + } if jobs is not None else None, + } + + +class ProofPending(GateRejected): + """An observed main counterpart is absent or has not completed yet.""" + + def __init__( + self, + message: str, + run: dict[str, Any] | None, + jobs: list[dict[str, Any]] | None, + *, + workflow: str, + ) -> None: + super().__init__(message, run, jobs) + self.workflow = workflow + self.observations = {workflow: {"run": run, "jobs": jobs}} + + +def require_main_ready( + run: dict[str, Any], run_id: int, *, head: str, workflow: str, message: str +) -> None: + require( + type(run_id) is int + and run_id > 0 + and type(run.get("id")) is int + and run["id"] == run_id + and run.get("path") == workflow + and run.get("event") == "push" + and run.get("head_branch") == "main" + and isinstance(head, str) + and HEX40.fullmatch(head) + and run.get("head_sha") == head + and type(run.get("run_attempt")) is int + and run["run_attempt"] > 0, + message, + ) + require(isinstance(run.get("status"), str) and "conclusion" in run, message) + if run["status"] in PENDING_STATUSES and run["conclusion"] is None: + raise ProofPending(message, run, None, workflow=workflow) + require(run["status"] == "completed" and run["conclusion"] == "success", message) + + +def semantic_evidence_identity(evidence: dict[str, Any]) -> dict[str, Any]: + """Preserve proof content while projecting raw API run/job identity.""" + identity = copy.deepcopy(evidence) + proofs = identity.get("main_proofs") + if isinstance(proofs, dict) and isinstance(proofs.get("native"), dict): + native = proofs["native"] + run = native.get("run") + require(isinstance(run, dict), "native run: expected an object") + native["run"] = { + field: run[field] + for field in ( + "id", "workflow_id", "path", "event", "head_branch", "head_sha", + "run_attempt", "status", "conclusion", + ) + if field in run } + jobs = native.get("jobs") + require( + isinstance(jobs, list) and all(isinstance(job, dict) for job in jobs), + "missing observed jobs", + ) + native["jobs"] = sorted( + ( + { + field: job[field] + for field in ( + "id", "name", "run_id", "run_attempt", "head_sha", "status", "conclusion", + ) + if field in job + } + for job in jobs + ), + key=json_bytes, + ) + return identity def check_junit(path: Path, suite: str) -> None: @@ -87,16 +183,22 @@ def distribution_hashes(directory: Path) -> dict[str, str]: def attempt_jobs(repo: str, run_id: int, attempt: int) -> list[dict[str, Any]]: - return object_json( + jobs = object_json( gh_bytes(f"repos/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100") - )["jobs"] + ).get("jobs") + require( + isinstance(jobs, list) and all(isinstance(job, dict) for job in jobs), + "missing observed jobs", + ) + assert isinstance(jobs, list) + return jobs # Completion and success come from needs (authoritative). # The jobs API supplies names only: it can lag a finished leg by minutes (2026-10-22). # No timer by owner rule. -def validate_inventory(jobs: list[dict[str, Any]]) -> None: - for name in JOBS: +def validate_inventory(jobs: list[dict[str, Any]], *, main: bool = False) -> None: + for name in (*JOBS, NETWORK_JOB) if main else JOBS: matching = [job for job in jobs if job["name"] == name] require( len(matching) == 1, @@ -120,8 +222,14 @@ def require_no_continue_on_error(root: Path) -> None: ) -def create_evidence(root: Path, directory: Path, needs: dict[str, Any]) -> dict[str, Any]: - require(set(needs) == NEEDS, "required CI job inventory changed") +def create_evidence( + root: Path, directory: Path, needs: dict[str, Any], *, network: Path | None = None +) -> dict[str, Any]: + main = os.environ["GITHUB_EVENT_NAME"] == "push" + require( + set(needs) == NEEDS | ({"network-proof"} if main else set()), + "required CI job inventory changed", + ) require( all(value.get("result") == "success" for value in needs.values()), "a required CI job failed, was cancelled, missing or skipped", @@ -135,8 +243,8 @@ def create_evidence(root: Path, directory: Path, needs: dict[str, Any]) -> dict[ run_id = int(os.environ["GITHUB_RUN_ID"]) attempt = int(os.environ["GITHUB_RUN_ATTEMPT"]) jobs = attempt_jobs(repo, run_id, attempt) - validate_inventory(jobs) - return { + validate_inventory(jobs, main=main) + result: dict[str, Any] = { "run_id": run_id, "attempt": attempt, "event": os.environ["GITHUB_EVENT_NAME"], @@ -153,9 +261,14 @@ def create_evidence(root: Path, directory: Path, needs: dict[str, Any]) -> dict[ ], "distributions": distribution_hashes(directory), } + if main: + require(network is not None, "main requires the actual live-PyPI report") + assert network is not None + result["main_proofs"] = {"network": consume_network(root, network)} + return result -def validate_gate( +def _validate_ci_evidence( evidence: dict[str, Any], run: dict[str, Any], jobs: list[dict[str, Any]], @@ -186,6 +299,45 @@ def validate_gate( validate_jobs(jobs, aggregate=True) +def validate_gate( + evidence: dict[str, Any], + run: dict[str, Any], + jobs: list[dict[str, Any]], + *, + head: str, + base: str | None, + pr: int | None, +) -> None: + _validate_ci_evidence(evidence, run, jobs, head=head, base=base, pr=pr) + if pr is None: + validate_main(evidence, run, jobs) + + +def _validate_waiting_python( + evidence: dict[str, Any], + run: dict[str, Any], + jobs: list[dict[str, Any]], + *, + head: str, + base: str | None, +) -> None: + _validate_ci_evidence(evidence, run, jobs, head=head, base=base, pr=None) + require( + evidence.get("tested_sha") == evidence.get("head"), + "main proof must test the landed main head", + ) + network = evidence["main_proofs"]["network"] + require(isinstance(network, dict), "network: expected an object") + validate_identity(network, run, evidence, "network") + validate_proof_inventory(network, "network") + require( + network.get("node_id") == NETWORK_NODE + and network["registered"] == network["executed"] == [NETWORK_NODE], + "network: missing unique required live-PyPI row", + ) + successful_jobs(jobs, (NETWORK_JOB,), run=run) + + def validate_jobs(jobs: list[dict[str, Any]], *, aggregate: bool) -> None: for name in (*JOBS, "release/ready") if aggregate else JOBS: matching = [job for job in jobs if job["name"] == name] @@ -209,21 +361,121 @@ def artifact_json(repo: str, run_id: int, name: str, filename: str) -> dict[str, return object_json(zipped.read(filename)) +def matching_main_runs(observed: Any, workflow: str, *, head: str) -> list[dict[str, Any]]: + require(isinstance(head, str) and HEX40.fullmatch(head), "invalid main proof head") + require(isinstance(observed, list), "missing workflow run observations") + runs = [] + for row in observed: + require( + isinstance(row, dict) and type(row.get("id")) is int and row["id"] > 0, + "workflow observation has no valid run identity", + ) + require( + type(row.get("run_attempt")) is int + and row["run_attempt"] > 0 + and all( + isinstance(row.get(field), str) and row[field] + for field in ("event", "head_branch", "head_sha", "path", "status") + ) + and HEX40.fullmatch(row["head_sha"]) + and row["status"] in PENDING_STATUSES | {"completed"} + and "conclusion" in row + and ( + row["conclusion"] is None + or (isinstance(row["conclusion"], str) and bool(row["conclusion"])) + ), + "malformed workflow run observation", + ) + if ( + row["event"] == "push" + and row["head_branch"] == "main" + and row["head_sha"] == head + and row["path"] == f".github/workflows/{workflow}" + ): + runs.append(row) + return runs + + +def latest_main_run(repo: str, workflow: str, *, head: str) -> dict[str, Any]: + """Choose by identity/id before examining status; never fall back to green.""" + runs = [] + page = 1 + while True: + suffix = "" if page == 1 else f"&page={page}" + observed = object_json( + gh_bytes( + f"repos/{repo}/actions/workflows/{workflow}/runs" + f"?event=push&head_sha={head}&per_page=100{suffix}" + ) + ).get("workflow_runs") + runs.extend(matching_main_runs(observed, workflow, head=head)) + assert isinstance(observed, list) + if len(observed) < 100: + break + page += 1 + if not runs: + raise ProofPending( + f"missing exact-head main {workflow} run", None, None, + workflow=f".github/workflows/{workflow}", + ) + return max(runs, key=lambda row: row["id"]) + + def gate_for_run( repo: str, run_id: int, *, head: str, base: str | None = None, pr: int | None = None ) -> dict[str, Any]: run = object_json(gh_bytes(f"repos/{repo}/actions/runs/{run_id}")) - jobs = object_json( - gh_bytes( - f"repos/{repo}/actions/runs/{run_id}/attempts/{run['run_attempt']}/jobs?per_page=100" - ) - )["jobs"] + jobs: list[dict[str, Any]] | None = None try: + if pr is None: + require_main_ready( + run, run_id, head=head, workflow=".github/workflows/ci-post.yml", + message="CI has not completed successfully", + ) + jobs = attempt_jobs(repo, run_id, run["run_attempt"]) validate_jobs(jobs, aggregate=True) evidence = artifact_json( repo, run_id, f"release-evidence-{run['run_attempt']}", "evidence.json" ) + if pr is None: + proofs = evidence.get("main_proofs") + require( + isinstance(proofs, dict) and set(proofs) == {"network"}, + "Python producer must supply its network proof only", + ) + assert isinstance(proofs, dict) + try: + native = latest_main_run(repo, "native-feed-post.yml", head=head) + # Bind the current attempt rather than the cached list observation. + current = object_json(gh_bytes(f"repos/{repo}/actions/runs/{native['id']}")) + require_main_ready( + current, native["id"], head=head, workflow=NATIVE_WORKFLOW, + message="newest native proof has not completed successfully", + ) + except ProofPending: + # A native wait cannot hide invalid completed Python evidence. + _validate_waiting_python(evidence, run, jobs, head=head, base=base) + raise + native_jobs = attempt_jobs(repo, current["id"], current["run_attempt"]) + attestation = artifact_json( + repo, current["id"], f"native-evidence-{current['run_attempt']}", + "native-evidence.json", + ) + evidence["main_proofs"] = { + **proofs, + "native": { + "workflow_path": NATIVE_WORKFLOW, + "run": current, + "jobs": native_jobs, + "attestation": attestation, + }, + } validate_gate(evidence, run, jobs, head=head, base=base, pr=pr) + except ProofPending as exc: + exc.observations.setdefault( + ".github/workflows/ci-post.yml", {"run": run, "jobs": jobs} + ) + raise except ValueError as exc: raise GateRejected(str(exc), run, jobs) from exc return evidence @@ -236,11 +488,15 @@ def main() -> None: parser.add_argument("command", choices=("junit", "aggregate")) parser.add_argument("--path", type=Path, required=True) parser.add_argument("--suite", choices=TESTS) + parser.add_argument("--network-proof", type=Path) args = parser.parse_args() if args.command == "junit": check_junit(args.path, args.suite) else: - evidence = create_evidence(Path.cwd(), args.path, json.loads(os.environ["NEEDS_JSON"])) + evidence = create_evidence( + Path.cwd(), args.path, json.loads(os.environ["NEEDS_JSON"]), + network=args.network_proof, + ) Path("evidence.json").write_bytes(json_bytes(evidence)) diff --git a/scripts/release_chain/main_proofs.py b/scripts/release_chain/main_proofs.py new file mode 100644 index 0000000..b2636ba --- /dev/null +++ b/scripts/release_chain/main_proofs.py @@ -0,0 +1,333 @@ +"""Produce and validate the actual native and live-PyPI main inventories.""" + +from __future__ import annotations + +import argparse +import os +import subprocess +from pathlib import Path +from typing import Any, cast +from xml.etree import ElementTree + +from .contract import HEX40, HEX64, json_bytes, object_json, require, sha256 + +NATIVE_WORKFLOW = ".github/workflows/native-feed-post.yml" +NATIVE_LEGS = { + "release-amd64": "Release (ubuntu-24.04)", + "release-arm64": "Release (ubuntu-24.04-arm)", + "release-macos": "Release (macos-15)", + "asan-ubsan": "ASan + UBSan", + "tsan": "TSan", +} +NETWORK_JOB = "Live PyPI hashes" +NETWORK_NODE = ( + "tests/test_consumer_receipt.py::test_bootstrap_recorded_hashes_match_live_pypi" +) +NATIVE_WORKFLOW_SHA256 = "2f720000c43809ac80e98e6ab7a96f58dff461691bb2c68f0f2fd149e167ffaa" + + +def object_value(value: Any, label: str) -> dict[str, Any]: + require(isinstance(value, dict), f"{label}: expected an object") + return cast(dict[str, Any], value) + + +def names(value: Any, label: str) -> list[str]: + require( + isinstance(value, list) + and bool(value) + and all(isinstance(name, str) and name for name in value), + f"{label}: missing nonempty inventory", + ) + require(len(value) == len(set(value)), f"{label}: duplicate inventory") + return cast(list[str], value) + + +def validate_inventory(value: Any, label: str) -> None: + row = object_value(value, label) + registered = names(row.get("registered"), f"{label} registered") + executed = names(row.get("executed"), f"{label} executed") + require(set(registered) == set(executed), f"{label}: registration/execution disagree") + for field in ("skipped", "failures", "errors"): + require(row.get(field) == [], f"{label}: required rows {field}") + digest = row.get("junit_sha256") + require( + isinstance(digest, str) and HEX64.fullmatch(digest), + f"{label}: missing raw JUnit SHA256", + ) + + +def _validate_native_inventory(value: Any, leg: str) -> None: + label = f"native {leg}" + require(leg in NATIVE_LEGS, f"{label}: unknown leg") + validate_inventory(value, label) + required = {"feed-unit", "feed-export-unit", "feed-mock-export", "feed-mock"} + if leg == "tsan": + required.add("feed-tsan-seeded-race") + observed = set(value["registered"]) & set(value["executed"]) + missing = sorted(required - observed) + require( + not missing, + f"{label}: missing required native rows: {', '.join(missing)}", + ) + + +def successful_jobs( + jobs: Any, required: tuple[str, ...], *, run: dict[str, Any] +) -> None: + require(isinstance(jobs, list), "missing observed jobs") + for name in required: + matches = [row for row in jobs if isinstance(row, dict) and row.get("name") == name] + require( + len(matches) == 1 + and matches[0].get("status") == "completed" + and matches[0].get("conclusion") == "success", + f"required job {name!r} missing, duplicated, failed, cancelled or skipped", + ) + for field, expected in ( + ("run_id", run.get("id")), + ("run_attempt", run.get("run_attempt")), + ("head_sha", run.get("head_sha")), + ): + if field in matches[0]: + require(matches[0][field] == expected, f"{name}: wrong observed {field}") + + +def validate_identity( + value: Any, run: dict[str, Any], evidence: dict[str, Any], label: str +) -> None: + row = object_value(value, label) + for field, expected in ( + ("run_id", run.get("id")), + ("attempt", run.get("run_attempt")), + ("head", evidence.get("head")), + ("tested_sha", evidence.get("tested_sha")), + ("tree", evidence.get("tree")), + ): + require(expected is not None and row.get(field) == expected, f"{label}: wrong {field}") + require( + type(run.get("id")) is int + and type(run.get("run_attempt")) is int + and type(row["run_id"]) is int + and row["run_id"] > 0 + and type(row["attempt"]) is int + and row["attempt"] > 0, + f"{label}: invalid run/attempt", + ) + for field in ("head", "tested_sha", "tree"): + require( + isinstance(row[field], str) and HEX40.fullmatch(row[field]), + f"{label}: invalid {field}", + ) + + +def validate_main( + evidence: dict[str, Any], run: dict[str, Any], jobs: list[dict[str, Any]] +) -> None: + proofs = object_value(evidence.get("main_proofs"), "main_proofs") + require(set(proofs) == {"native", "network"}, "main requires exactly native/network proofs") + require( + run.get("head_branch") == "main" + and evidence.get("tested_sha") == evidence.get("head"), + "main proof must test the landed main head", + ) + network = object_value(proofs["network"], "network") + validate_identity(network, run, evidence, "network") + validate_inventory(network, "network") + require( + network.get("node_id") == NETWORK_NODE + and network["registered"] == network["executed"] == [NETWORK_NODE], + "network: missing unique required live-PyPI row", + ) + successful_jobs(jobs, (NETWORK_JOB,), run=run) + + native = object_value(proofs["native"], "native") + observed = object_value(native.get("run"), "native run") + require( + native.get("workflow_path") == observed.get("path") == NATIVE_WORKFLOW + and observed.get("event") == "push" + and observed.get("head_branch") == "main" + and observed.get("head_sha") == evidence["head"] + and observed.get("status") == "completed" + and observed.get("conclusion") == "success", + "native: wrong source/workflow or incomplete run", + ) + successful_jobs(native.get("jobs"), (*NATIVE_LEGS.values(), "Native proof"), run=observed) + attestation = object_value(native.get("attestation"), "native attestation") + validate_identity(attestation, observed, evidence, "native") + legs = object_value(attestation.get("legs"), "native legs") + require(set(legs) == set(NATIVE_LEGS), "native: missing or unexpected leg") + for leg in NATIVE_LEGS: + _validate_native_inventory(legs[leg], leg) + + +def junit_inventory(path: Path, *, network: bool = False) -> dict[str, Any]: + raw = path.read_bytes() + root = ElementTree.fromstring(raw) + require(root.tag in {"testsuite", "testsuites"}, f"{path}: not a JUnit report") + cases = list(root.iter("testcase")) + registered, executed = [], [] + problems: dict[str, list[str]] = {"skipped": [], "failures": [], "errors": []} + for case in cases: + name = case.attrib.get("name", "") + if network: + classname = case.attrib.get("classname", "") + require(bool(classname), "network JUnit has no source classname") + name = f"{classname.replace('.', '/')}.py::{name}" + registered.append(name) + require( + case.get("status") in {None, "run", "passed", "notrun", "disabled"}, + f"{path}: unsuccessful or unknown testcase status", + ) + if case.find("skipped") is not None or case.get("status") in {"notrun", "disabled"}: + problems["skipped"].append(name) + else: + executed.append(name) + for tag, field in (("failure", "failures"), ("error", "errors")): + if case.find(tag) is not None: + problems[field].append(name) + row = { + "registered": registered, + "executed": executed, + **problems, + "junit_sha256": sha256(raw), + } + validate_inventory(row, str(path)) + # Collection/setup errors may be reported on a suite rather than a testcase. + for suite in root.iter(): + if suite.tag in {"testsuite", "testsuites"}: + if "tests" in suite.attrib: + require( + int(suite.attrib["tests"]) == len(list(suite.iter("testcase"))), + f"{path}: suite count disagrees with actual testcase rows", + ) + for field in ("failures", "errors", "skipped", "disabled"): + require(int(suite.get(field, "0")) == 0, f"{path}: suite {field}") + for tag in ("failure", "error", "skipped"): + require( + suite.find(tag) is None, f"{path}: suite contains {tag} element" + ) + return row + + +def native_inventory(registration: Path, junit: Path) -> dict[str, Any]: + raw = registration.read_bytes() + collected = object_json(raw) + tests = collected.get("tests") + require(isinstance(tests, list), "CTest registration has no tests list") + assert isinstance(tests, list) + registered = names( + [object_value(test, "CTest test").get("name") for test in tests], "CTest registration" + ) + row = junit_inventory(junit) + row["registered"] = registered + row["registration_sha256"] = sha256(raw) + validate_inventory(row, "CTest") + return row + + +def local_identity(root: Path, workflow: str) -> dict[str, Any]: + require( + os.environ["GITHUB_EVENT_NAME"] == "push" + and os.environ["GITHUB_REF"] == "refs/heads/main" + and os.environ["GITHUB_WORKFLOW_REF"].split("@", 1)[0] + == f"{os.environ['GITHUB_REPOSITORY']}/{workflow}", + "producer is not the required push/main workflow", + ) + tested = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + tree = subprocess.check_output( + ["git", "rev-parse", "HEAD^{tree}"], cwd=root, text=True + ).strip() + require(tested == os.environ["GITHUB_SHA"], "producer checkout differs from event head") + return { + "run_id": int(os.environ["GITHUB_RUN_ID"]), + "attempt": int(os.environ["GITHUB_RUN_ATTEMPT"]), + "head": os.environ["GITHUB_SHA"], + "tested_sha": tested, + "tree": tree, + } + + +def require_native_workflow(root: Path) -> None: + raw = (root / NATIVE_WORKFLOW).read_bytes() + require( + sha256(raw) == NATIVE_WORKFLOW_SHA256 and b"continue-on-error" not in raw, + "native workflow changed: authoritative needs require independent workflow review", + ) + + +def network_proof(root: Path, directory: Path) -> dict[str, Any]: + identity = local_identity(root, ".github/workflows/ci-post.yml") + row = junit_inventory(directory / "network.xml", network=True) + require(row["registered"] == row["executed"] == [NETWORK_NODE], "wrong live-PyPI inventory") + return {**identity, "node_id": NETWORK_NODE, **row} + + +def consume_network(root: Path, directory: Path) -> dict[str, Any]: + recorded = object_json((directory / "network-proof.json").read_bytes()) + actual = network_proof(root, directory) + require(recorded == actual, "network report bytes or producer identity changed") + return actual + + +def native_leg(root: Path, leg: str, directory: Path) -> dict[str, Any]: + require_native_workflow(root) + identity = local_identity(root, NATIVE_WORKFLOW) + inventory = native_inventory( + directory / "native-registration.json", directory / "native-junit.xml" + ) + _validate_native_inventory(inventory, leg) + return { + **identity, + "leg": leg, + "inventory": inventory, + } + + +def native_aggregate(root: Path, directory: Path, needs: dict[str, Any]) -> dict[str, Any]: + require(set(needs) == {"authz", "main-ref", *NATIVE_LEGS}, "native needs inventory changed") + require( + all(isinstance(row, dict) and row.get("result") == "success" for row in needs.values()), + "native dependency failed, cancelled or skipped", + ) + require_native_workflow(root) + identity = local_identity(root, NATIVE_WORKFLOW) + legs = {} + for leg in NATIVE_LEGS: + reports = directory / leg + recorded = object_json((reports / "native-leg.json").read_bytes()) + actual = native_leg(root, leg, reports) + require(recorded == actual, f"{leg}: report bytes or producer identity changed") + require( + all(recorded.get(key) == value for key, value in identity.items()), + f"{leg}: stale producer", + ) + legs[leg] = { + **actual["inventory"], + "artifact": f"native-leg-{leg}-{identity['attempt']}", + } + # Do not query/wait for this running aggregate's own completed job. + return {**identity, "legs": legs} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("network", "native-leg", "native-aggregate")) + parser.add_argument("--directory", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--leg", choices=NATIVE_LEGS) + args = parser.parse_args() + root = Path.cwd() + if args.command == "network": + value = network_proof(root, args.directory) + elif args.command == "native-leg": + require(args.leg is not None, "native leg identifier is required") + value = native_leg(root, args.leg, args.directory) + else: + value = native_aggregate( + root, args.directory, object_json(os.environ["NEEDS_JSON"].encode()) + ) + args.output.write_bytes(json_bytes(value)) + + +if __name__ == "__main__": + main() diff --git a/scripts/release_chain/publishing.py b/scripts/release_chain/publishing.py index b605b54..dea3400 100644 --- a/scripts/release_chain/publishing.py +++ b/scripts/release_chain/publishing.py @@ -14,7 +14,7 @@ from typing import Any from urllib.error import HTTPError -from .automation import api, finish, git, inventory, marker, pr_identity, receipt, snapshot +from .automation import api, finish, git, inventory, marker, pr_identity, snapshot from .contract import ( LOCK, MANIFEST, @@ -31,9 +31,16 @@ verify_artifacts, verify_local, ) -from .evidence import distribution_hashes, gate_for_run +from .evidence import ( + ProofPending, + distribution_hashes, + gate_for_run, + latest_main_run, + matching_main_runs, +) +from .main_proofs import NATIVE_WORKFLOW from .receipts import candidate as record_candidate -from .receipts import canonical, served, timestamp +from .receipts import canonical, receipt, served, timestamp # TEMPORARY first-adopter exemption (TX-DATA-BASELINE2, L6 10-19): # remove the data exemption and the hub form together, only after 0.3.1 is verified AND @@ -125,6 +132,7 @@ def candidate_gate(repo: str, pr: dict[str, Any]) -> dict[str, Any]: meta = marker(pr) head = (pr.get("head") or {}).get("sha") require(head, "pre-merge CI evidence missing") + assert head is not None runs = inventory( repo, "actions/workflows/ci.yml/runs?event=pull_request&head_sha=" + head, @@ -133,6 +141,8 @@ def candidate_gate(repo: str, pr: dict[str, Any]) -> dict[str, Any]: runs = [run for run in runs if run_belongs_to_pr(run, pr)] require(runs, "pre-merge CI evidence missing") run = max(runs, key=lambda row: row["id"]) + require(meta is not None, "pre-merge release marker missing") + assert meta is not None evidence = gate_for_run( repo, run["id"], head=pr["head"]["sha"], base=meta["base"], pr=pr["number"] ) @@ -161,6 +171,7 @@ def verified_package( and sum(name.endswith(".tar.gz") for name in files) == 1, "previous publication needs a wheel and sdist", ) + assert files is not None metadata = object_json(fetch(f"https://pypi.org/pypi/pineforge-data/{version}/json")) require( {entry["filename"]: entry["digests"]["sha256"] for entry in metadata["urls"]} == files, @@ -235,6 +246,7 @@ def verified_package( "previous distribution lacks manifest/facts provenance; " "establish a baseline before enablement", ) + assert lock is not None wheel = next(data for name, data in downloads.items() if name.endswith(".whl")) with zipfile.ZipFile(io.BytesIO(wheel)) as zipped: for name in (LOCK, MANIFEST, RUNTIME): @@ -281,32 +293,52 @@ def recover_applied(repo: str, source: Path, report: dict[str, Any]) -> None: saved["state"] == "verified" and saved["event_id"] == report["event_id"], "stored publication identity differs", ) + image_digest = verified_package(repo, version, saved)["image_digest"] + lock = verify_local(source) + require(lock is not None, "stored publication source lock missing") + assert lock is not None require( - verified_package(repo, version, saved)["image_digest"] - == verify_local(source)["event"]["image"]["digest"], + image_digest == lock["event"]["image"]["digest"], "stored publication does not match installed defaults", ) report.update(saved) return prs = inventory(repo, "pulls?state=closed&base=main&sort=updated&direction=desc") candidates = [ - pr for pr in prs if pr.get("merged_at") and marker(pr) and marker(pr)["version"] == version + pr + for pr in prs + if pr.get("merged_at") and (meta := marker(pr)) and meta["version"] == version ] require(len(candidates) == 1, "applied release has no unique merged update PR") pr = candidates[0] + meta = marker(pr) + assert meta is not None report["pr"] = pr_identity({**pr, "merged": True}) - report["pr"]["base_sha"] = marker(pr)["base"] - runs = api( - repo, - "actions/workflows/ci-post.yml/runs?event=push&per_page=100&head_sha=" - + pr["merge_commit_sha"], - )["workflow_runs"] - runs = [run for run in runs if run["head_sha"] == pr["merge_commit_sha"]] - if not runs or max(runs, key=lambda run: run["id"])["status"] != "completed": + report["pr"]["base_sha"] = meta["base"] + runs = [] + page = 1 + while True: + suffix = "" if page == 1 else f"&page={page}" + response = api( + repo, + "actions/workflows/ci-post.yml/runs?event=push&per_page=100&head_sha=" + + pr["merge_commit_sha"] + suffix, + ) + require(isinstance(response, dict), "missing workflow run observations") + observed = response.get("workflow_runs") + runs.extend(matching_main_runs(observed, "ci-post.yml", head=pr["merge_commit_sha"])) + if len(observed) < 100: + break + page += 1 + if not runs: report["state"] = "testing" return run = max(runs, key=lambda run: run["id"]) - gate_for_run(repo, run["id"], head=pr["merge_commit_sha"]) + try: + gate_for_run(repo, run["id"], head=pr["merge_commit_sha"]) + except ProofPending: + report["state"] = "testing" + return lock = verify_local(source) if lock: record_candidate(report, lock["event"], candidate_gate(repo, pr), version) @@ -399,6 +431,14 @@ def prepare( run = api(repo, f"actions/runs/{run_id}") require(run["event"] == "push" and run["head_branch"] == "main", "not landed main CI") source = run["head_sha"] + resolve_pair = ( + os.environ.get("GITHUB_EVENT_NAME") == "workflow_run" or run.get("path") == NATIVE_WORKFLOW + ) + if resolve_pair: + require( + run.get("path") in {".github/workflows/ci-post.yml", NATIVE_WORKFLOW}, + "unexpected main proof callback workflow", + ) # Check the associated PR before executing or checking out any candidate code. prs = api(repo, f"commits/{source}/pulls?per_page=100") matches = [ @@ -415,7 +455,26 @@ def prepare( pr = api(repo, f"pulls/{matches[0]['number']}") meta = marker(pr) require(pr["user"]["login"] == os.environ["RELEASE_BOT_LOGIN"], "unexpected release PR owner") + require(meta is not None, "release PR marker missing") + assert meta is not None require(pr["head"]["sha"] == meta["generated_head"], "release PR head changed") + try: + if resolve_pair: + # Either completion re-resolves the newest pair, including current attempts. + selected = latest_main_run(repo, "ci-post.yml", head=source) + run_id = selected["id"] + run = api(repo, f"actions/runs/{run_id}") + require( + run.get("id") == run_id + and run.get("event") == "push" + and run.get("head_branch") == "main" + and run.get("head_sha") == source + and run.get("path") == ".github/workflows/ci-post.yml", + "counterpart is not exact-head main CI", + ) + evidence = gate_for_run(repo, run_id, head=source) + except ProofPending: + return {"eligible": "false", "waiting": "main-proofs-pending"} git("fetch", "origin", source) # Keep this process and scripts at the trusted default branch. Read source as data. candidate = output / "source" @@ -426,7 +485,6 @@ def prepare( report.update(receipt(event, "prepare-publication")) report["pr"] = pr_identity(pr) report["pr"]["base_sha"] = meta["base"] - evidence = gate_for_run(repo, run_id, head=source) require( git("rev-parse", f"{source}^{{tree}}") == evidence["tree"] and evidence["tested_sha"] == source, @@ -435,6 +493,7 @@ def prepare( # Retain the independent pre-merge candidate gates, as required by the hub. # Main CI still authorizes the exact distributions we publish. if event: + assert lock is not None record_candidate(report, event, candidate_gate(repo, pr), meta["version"]) previous_version = lock["previous"]["package_version"] report["previous"] = verified_package(repo, previous_version, adopter_lock=lock) diff --git a/scripts/release_chain/reporting.py b/scripts/release_chain/reporting.py index d41c7bb..77b3f8e 100644 --- a/scripts/release_chain/reporting.py +++ b/scripts/release_chain/reporting.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any -from .automation import api, finish, inventory, marker, pr_identity, receipt +from .automation import api, finish, inventory, marker, pr_identity from .contract import ( HUB, gh_bytes, @@ -22,7 +22,7 @@ sha256, verify_local, ) -from .evidence import artifact_json, gate_for_run +from .evidence import artifact_json, gate_for_run, semantic_evidence_identity from .publishing import candidate_gate, stored_receipt, verified_package from .receipts import ( DEPLOY_JOBS, @@ -30,6 +30,7 @@ FIELDS, WORKFLOW, candidate, + receipt, signal, timestamp, ) @@ -192,9 +193,11 @@ def publication(repo: str, run_id: int, report: dict[str, Any]) -> None: pr["merged"] and pr["head"]["repo"]["full_name"] == repo and pr["base"]["ref"] == "main" + and meta is not None and pr["head"]["sha"] == meta["generated_head"], "publication PR identity moved", ) + assert meta is not None source = pr["merge_commit_sha"] require(report["pr"] == pr_identity(pr, base=meta["base"]), "publication PR evidence differs") commit = api(repo, f"commits/{source}") @@ -221,12 +224,15 @@ def publication(repo: str, run_id: int, report: dict[str, Any]) -> None: check=True, ) main_gate = object_json((bundle / "evidence.json").read_bytes()) + fresh_gate = gate_for_run(repo, main_gate["run_id"], head=source) require( - gate_for_run(repo, main_gate["run_id"], head=source) == main_gate, + json_bytes(semantic_evidence_identity(fresh_gate)) + == json_bytes(semantic_evidence_identity(main_gate)), "landed CI distributions/evidence moved", ) lock = verify_local(bundle / "source") require(lock is not None and lock["event"] == event, "served bundle target differs") + assert lock is not None version = lock["package_version"] require(version == meta["version"], "served package differs from PR") candidate(report, event, candidate_gate(repo, pr), version) @@ -299,7 +305,10 @@ def retain(repo: str, report: dict[str, Any]) -> None: if report["state"] != "verified": return pr = api(repo, f"pulls/{report['pr']['number']}") - version = marker(pr)["version"] + meta = marker(pr) + require(meta is not None, "retained release marker missing") + assert meta is not None + version = meta["version"] saved = stored_receipt(repo, version) if saved: require( diff --git a/tests/test_consumer_receipt.py b/tests/test_consumer_receipt.py index c4ff25f..9321eba 100644 --- a/tests/test_consumer_receipt.py +++ b/tests/test_consumer_receipt.py @@ -7,17 +7,1700 @@ import os import zipfile from pathlib import Path +from urllib.parse import parse_qs, urlsplit +from xml.etree import ElementTree import hub_receipt_validator as hub import pytest import yaml -from release_chain import automation, contract, publishing, receipts, reporting +from release_chain import automation, contract, evidence, publishing, receipts, reporting FIXTURES = Path(__file__).parent / "fixtures/release_chain" REPO = "owner/data" HEAD, BASE, MERGE = "a" * 40, "b" * 40, "c" * 40 +@pytest.fixture(params=[("transport-one", 480, 2), ("transport-two", 580, 5)]) +def main_proof_transport(monkeypatch, request): + """Offline GitHub observations; only the bytes transport is replaced.""" + label, python_id, attempt = request.param + head = contract.sha256(label.encode())[:40] + tree = contract.sha256(f"{label}:tree".encode())[:40] + node = "tests/test_consumer_receipt.py::test_bootstrap_recorded_hashes_match_live_pypi" + native_names = { + "release-amd64": "Release (ubuntu-24.04)", + "release-arm64": "Release (ubuntu-24.04-arm)", + "release-macos": "Release (macos-15)", + "asan-ubsan": "ASan + UBSan", + "tsan": "TSan", + } + python_run = { + "id": python_id, "run_attempt": attempt, "head_sha": head, "head_branch": "main", + "status": "completed", "conclusion": "success", "event": "push", + "path": ".github/workflows/ci-post.yml", + } + native_run = { + **python_run, "id": python_id + 20, "run_attempt": attempt + 1, + "path": ".github/workflows/native-feed-post.yml", + } + legs = {} + for leg in native_names: + names = ["feed-unit", "feed-export-unit", "feed-mock-export", "feed-mock"] + if leg == "tsan": + names.append("feed-tsan-seeded-race") + junit = ElementTree.Element("testsuite", name=f"{label}:{leg}") + for name in names: + ElementTree.SubElement(junit, "testcase", name=name) + legs[leg] = { + "registered": names, "executed": list(names), "skipped": [], + "failures": [], "errors": [], + "junit_sha256": contract.sha256(ElementTree.tostring(junit)), + } + network_junit = ElementTree.Element("testsuite", name=f"{label}:network") + ElementTree.SubElement(network_junit, "testcase", name=node) + network = { + "run_id": python_id, "attempt": attempt, "head": head, "tested_sha": head, + "tree": tree, "node_id": node, "registered": [node], "executed": [node], + "skipped": [], "failures": [], "errors": [], + "junit_sha256": contract.sha256(ElementTree.tostring(network_junit)), + } + python_evidence = { + "run_id": python_id, "attempt": attempt, "head": head, "tested_sha": head, + "tree": tree, "base": None, "pr": None, "event": "push", + "jobs": dict.fromkeys(evidence.JOBS, "success"), + "e2e": {key: sorted(value) for key, value in evidence.TESTS.items()}, + # Native proof must be discovered independently, never supplied by this artifact. + "main_proofs": {"network": network}, + } + workflow = yaml.load( + (Path(__file__).parents[1] / ".github/workflows/ci-post.yml").read_text(), + Loader=yaml.BaseLoader, + ) + network_names = [ + job.get("name", job_id) + for job_id, job in workflow["jobs"].items() + if any(node in step.get("run", "") for step in job.get("steps", [])) + ] + network_name = network_names[0] if len(network_names) == 1 else "Live PyPI hashes" + python_jobs = [ + {"name": name, "status": "completed", "conclusion": "success"} + for name in (*evidence.JOBS, "release/ready") + ] + if not any(job["name"] == network_name for job in python_jobs): + python_jobs.append( + {"name": network_name, "status": "completed", "conclusion": "success"} + ) + native_jobs = [ + {"name": name, "status": "completed", "conclusion": "success"} + for name in (*native_names.values(), "Native proof") + ] + attestation = { + "run_id": native_run["id"], "attempt": native_run["run_attempt"], + "head": head, "tested_sha": head, "tree": tree, "legs": legs, + } + state = { + "python_run": python_run, "python_evidence": python_evidence, + "python_jobs": python_jobs, "network_name": network_name, + "native_run": native_run, "native_jobs": native_jobs, + "native_runs": [native_run], "attestation": attestation, + "other_native_jobs": copy.deepcopy(native_jobs), + "other_attestation": copy.deepcopy(attestation), + "native_artifact_missing": False, "calls": [], + "native_artifact_attempt_delta": 0, + "python_runs": [python_run], + "other_python_jobs": copy.deepcopy(python_jobs), + "other_python_evidence": copy.deepcopy(python_evidence), + "python_artifact_missing": False, "python_artifact_attempt_delta": 0, + "artifact_observations": [], + "artifact_raw": {}, + } + + def gh_bytes(route, **kwargs): + state["calls"].append(route) + assert set(kwargs) <= {"accept"}, "the gate transport must remain read-only" + parsed = urlsplit(route) + path = parsed.path.removeprefix(f"repos/{REPO}/") + if path == "actions/workflows/native-feed-post.yml/runs": + query = parse_qs(parsed.query) + assert query.get("event") == ["push"] + assert query.get("head_sha") == [head] + return contract.json_bytes({"workflow_runs": state["native_runs"]}) + if path == "actions/workflows/ci-post.yml/runs": + query = parse_qs(parsed.query) + assert query.get("event") == ["push"] + assert query.get("head_sha") == [head] + return contract.json_bytes({"workflow_runs": state["python_runs"]}) + observed_python = {row["id"]: row for row in state["python_runs"]} + # A list observation may predate a rerun. Detail reads expose the current attempt. + observed_python[python_id] = python_run + selected_python = None + for observed in observed_python.values(): + observed_id, observed_attempt = observed["id"], observed["run_attempt"] + if path == f"actions/runs/{observed_id}": + return contract.json_bytes(observed) + if path == f"actions/runs/{observed_id}/attempts/{observed_attempt}/jobs": + observed_jobs = ( + python_jobs if observed_id == python_id else state["other_python_jobs"] + ) + return contract.json_bytes({"jobs": observed_jobs}) + if path == f"actions/runs/{observed_id}/artifacts": + artifact_attempt = observed_attempt + state["python_artifact_attempt_delta"] + name = f"release-evidence-{artifact_attempt}" + state["artifact_observations"].append((observed_id, observed_attempt, name)) + missing = observed_id == python_id and state["python_artifact_missing"] + return contract.json_bytes({"artifacts": [] if missing else [{ + "id": observed_id * 100, "name": name, "expired": False, + }]}) + if path == f"actions/artifacts/{observed_id * 100}/zip": + selected_python = observed + break + if selected_python is not None: + filename = "evidence.json" + payload = copy.deepcopy( + python_evidence if selected_python["id"] == python_id + else state["other_python_evidence"] + ) + if selected_python["id"] != python_id: + payload.update( + run_id=selected_python["id"], attempt=selected_python["run_attempt"], + head=selected_python["head_sha"], tested_sha=selected_python["head_sha"], + ) + network = payload.get("main_proofs", {}).get("network") + if network is not None: + network.update( + run_id=selected_python["id"], attempt=selected_python["run_attempt"], + head=selected_python["head_sha"], tested_sha=selected_python["head_sha"], + ) + else: + # Serve a fully usable older green proof too: an erroneous fallback + # must be caught by the verdict/call assertions, not by absent mock setup. + observed_runs = {row["id"]: row for row in state["native_runs"]} + observed_runs[native_run["id"]] = native_run + selected = None + for observed in observed_runs.values(): + native_id = observed["id"] + native_attempt = observed["run_attempt"] + if path == f"actions/runs/{native_id}": + return contract.json_bytes(observed) + if path == f"actions/runs/{native_id}/attempts/{native_attempt}/jobs": + observed_jobs = ( + native_jobs if native_id == native_run["id"] + else state["other_native_jobs"] + ) + return contract.json_bytes({"jobs": observed_jobs}) + if path == f"actions/runs/{native_id}/artifacts": + missing = ( + native_id == native_run["id"] and state["native_artifact_missing"] + ) + artifacts = [] if missing else [{ + "id": native_id * 100, + "name": ( + "native-evidence-" + f"{native_attempt + state['native_artifact_attempt_delta']}" + ), + "expired": False, + }] + state["artifact_observations"].append(( + native_id, native_attempt, + "native-evidence-" + f"{native_attempt + state['native_artifact_attempt_delta']}", + )) + return contract.json_bytes({"artifacts": artifacts}) + if path == f"actions/artifacts/{native_id * 100}/zip": + selected = observed + break + if selected is None: + raise AssertionError(f"unexpected gate transport read: {route}") + payload = copy.deepcopy( + attestation if selected["id"] == native_run["id"] + else state["other_attestation"] + ) + if selected["id"] != native_run["id"]: + payload.update( + run_id=selected["id"], attempt=selected["run_attempt"], + head=selected["head_sha"], tested_sha=selected["head_sha"], + ) + filename = "native-evidence.json" + archive = io.BytesIO() + with zipfile.ZipFile(archive, "w") as zipped: + raw = state["artifact_raw"].get(filename, contract.json_bytes(payload)) + zipped.writestr(filename, raw) + return archive.getvalue() + + monkeypatch.setattr(evidence, "gh_bytes", gh_bytes) + return state + + +def assert_native_discovery(state): + assert any( + "/actions/workflows/native-feed-post.yml/runs?" in route + for route in state["calls"] + ), "the real gate never requested native workflow observations" + + +@pytest.fixture +def observed_main_parser(main_proof_transport, monkeypatch): + """Record inputs while executing the original JSON and proof validators.""" + state = main_proof_transport + state.update( + parsed_bytes=[], parsed_objects=[], gate_inputs=[], main_inputs=[], main_validations=[], + waiting_inputs=[], waiting_validations=[], + ) + real_object_json = evidence.object_json + real_validate_gate = evidence.validate_gate + real_validate_main = evidence.validate_main + real_validate_waiting_python = evidence._validate_waiting_python + + def parse(raw): + state["parsed_bytes"].append(raw) + result = real_object_json(raw) + state["parsed_objects"].append(copy.deepcopy(result)) + return result + + def validate_gate(proof, run, jobs, **kwargs): + state["gate_inputs"].append(copy.deepcopy((proof, run, jobs, kwargs))) + return real_validate_gate(proof, run, jobs, **kwargs) + + def validate_main(proof, run, jobs): + state["main_inputs"].append(copy.deepcopy((proof, run, jobs))) + result = real_validate_main(proof, run, jobs) + state["main_validations"].append(copy.deepcopy(proof)) + return result + + def validate_waiting_python(proof, run, jobs, **kwargs): + state["waiting_inputs"].append(copy.deepcopy((proof, run, jobs, kwargs))) + result = real_validate_waiting_python(proof, run, jobs, **kwargs) + state["waiting_validations"].append(copy.deepcopy(proof)) + return result + + monkeypatch.setattr(evidence, "object_json", parse) + monkeypatch.setattr(evidence, "validate_gate", validate_gate) + monkeypatch.setattr(evidence, "validate_main", validate_main) + monkeypatch.setattr(evidence, "_validate_waiting_python", validate_waiting_python) + return state + + +@pytest.fixture +def incident_transport(monkeypatch): + """Real incident/inventory logic, with mutable issue observations at the API.""" + state = {"issues": [], "reads": [], "writes": [], "comments": []} + monkeypatch.setenv("GITHUB_RUN_ID", "930") + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "2") + + def api(repo, endpoint, method="GET", data=None): + assert repo == REPO + if method == "GET": + state["reads"].append(endpoint) + assert data is None + assert endpoint == ( + "issues?state=all&sort=updated&direction=desc&per_page=100&page=1" + ) + return copy.deepcopy(state["issues"]) + state["writes"].append((endpoint, method, copy.deepcopy(data))) + if endpoint == "issues" and method == "POST": + issue = {"number": 41, "state": "open", **copy.deepcopy(data)} + state["issues"].append(issue) + return copy.deepcopy(issue) + assert len(state["issues"]) == 1 + issue = state["issues"][0] + if endpoint == f"issues/{issue['number']}/comments" and method == "POST": + state["comments"].append(copy.deepcopy(data)) + return {"id": len(state["comments"]), **data} + assert endpoint == f"issues/{issue['number']}" and method == "PATCH" + issue.update(copy.deepcopy(data)) + return copy.deepcopy(issue) + + # inventory's original implementation resolves automation.api at call time. + monkeypatch.setattr(automation, "api", api) + monkeypatch.setattr(reporting, "api", api) + return state + + +def collect_callback_without_incident(state, incidents): + directory = state["callback_root"] / "downloaded-stages" + assert list(directory.glob("release-stage-*/release-receipt.json")) == [] + report = reporting.collect(directory, failed=False, stage="prepare-publication") + assert report["state"] == "pending" and report["event_id"] is None + assert report["pr"] is report["ci"] is report["deployed"] is None + reporting.incident(REPO, report, "release-owner", "prepare-publication") + assert incidents["reads"] == incidents["writes"] == [] + return report + + +def test_main_gate_consumes_matching_native_transport_positive_twin(main_proof_transport): + state = main_proof_transport + run = state["python_run"] + native = state["native_run"] + # An older green match and higher-id unrelated runs cannot replace the selected pair. + older = {**native, "id": native["id"] - 1, "run_attempt": 1} + unrelated = [ + {**native, "id": native["id"] + 1, "head_sha": "e" * 40}, + {**native, "id": native["id"] + 2, "head_branch": "topic"}, + {**native, "id": native["id"] + 3, "event": "pull_request"}, + {**native, "id": native["id"] + 4, "path": ".github/workflows/native-feed.yml"}, + ] + state["native_runs"] = [older, *unrelated, native] + if run["run_attempt"] % 2 == 0: + state["native_runs"].reverse() + result = publishing.gate_for_run(REPO, run["id"], head=run["head_sha"]) + assert_native_discovery(state) + prefix = f"repos/{REPO}/actions" + assert f"{prefix}/runs/{run['id']}" in state["calls"] + assert ( + f"{prefix}/runs/{run['id']}/attempts/{run['run_attempt']}/jobs?per_page=100" + in state["calls"] + ) + assert f"{prefix}/runs/{run['id']}/artifacts?per_page=100" in state["calls"] + assert f"{prefix}/artifacts/{run['id'] * 100}/zip" in state["calls"] + assert ( + f"{prefix}/runs/{native['id']}/attempts/{native['run_attempt']}/jobs?per_page=100" + in state["calls"] + ) + assert f"{prefix}/runs/{native['id']}/artifacts?per_page=100" in state["calls"] + assert f"{prefix}/artifacts/{native['id'] * 100}/zip" in state["calls"] + assert not any(f"/runs/{older['id']}/" in route for route in state["calls"]) + assert set(result.get("main_proofs", {})) == {"native", "network"} + assert result["main_proofs"]["native"] == { + "workflow_path": native["path"], "run": native, + "jobs": state["native_jobs"], "attestation": state["attestation"], + } + assert result["main_proofs"]["network"] == state["python_evidence"]["main_proofs"]["network"] + + +@pytest.mark.parametrize( + "fault", + [ + "missing-run", "wrong-head", "failed-run", "missing-artifact", "wrong-tree", + "wrong-attempt", "old-attempt-artifact", "native-job-red", "aggregate-red", + "network-disabled", "network-skipped", "network-job-red", "network-job-missing", + "network-job-duplicate", + ], +) +def test_main_gate_refuses_incomplete_observed_proofs(main_proof_transport, fault): + state = main_proof_transport + run = state["python_run"] + if fault == "missing-run": + state["native_runs"] = [] + elif fault == "wrong-head": + state["native_run"]["head_sha"] = "e" * 40 + elif fault == "failed-run": + state["native_run"]["conclusion"] = "failure" + elif fault == "missing-artifact": + state["native_artifact_missing"] = True + elif fault == "wrong-tree": + state["attestation"]["tree"] = "e" * 40 + elif fault == "wrong-attempt": + state["attestation"]["attempt"] -= 1 + elif fault == "old-attempt-artifact": + state["native_artifact_attempt_delta"] = -1 + elif fault in {"native-job-red", "aggregate-red"}: + name = "TSan" if fault == "native-job-red" else "Native proof" + next(job for job in state["native_jobs"] if job["name"] == name)["conclusion"] = "failure" + elif fault in {"network-job-red", "network-job-missing", "network-job-duplicate"}: + job = next( + job for job in state["python_jobs"] if job["name"] == state["network_name"] + ) + if fault == "network-job-missing": + state["python_jobs"].remove(job) + elif fault == "network-job-duplicate": + state["python_jobs"].append(copy.deepcopy(job)) + else: + job["conclusion"] = "skipped" + else: + network = state["python_evidence"]["main_proofs"]["network"] + network["executed"] = [] + if fault == "network-skipped": + network["skipped"] = [network["node_id"]] + with pytest.raises(evidence.GateRejected): + publishing.gate_for_run(REPO, run["id"], head=run["head_sha"]) + + +@pytest.mark.parametrize( + "fault", + ["in_progress", "failure", "cancelled", "missing-artifact", "job-failure", "wrong-tree"], +) +def test_main_gate_never_falls_back_from_newer_bad_native_proof(main_proof_transport, fault): + state = main_proof_transport + run, native = state["python_run"], state["native_run"] + older = {**native, "id": native["id"] - 1, "run_attempt": 1} + state["native_runs"] = [older, native] + if fault == "in_progress": + native.update(status="in_progress", conclusion=None) + elif fault == "missing-artifact": + state["native_artifact_missing"] = True + elif fault == "job-failure": + next(job for job in state["native_jobs"] if job["name"] == "TSan")[ + "conclusion" + ] = "failure" + elif fault == "wrong-tree": + state["attestation"]["tree"] = "e" * 40 + else: + native["conclusion"] = fault + with pytest.raises(evidence.GateRejected): + publishing.gate_for_run(REPO, run["id"], head=run["head_sha"]) + assert_native_discovery(state) + assert not any( + f"/runs/{older['id']}/" in route or f"/artifacts/{older['id'] * 100}/" in route + for route in state["calls"] + ), "an older successful native run must never substitute for the newest matching run" + + +def test_pr_gate_keeps_its_existing_contract_without_main_discovery(main_proof_transport): + state = main_proof_transport + gate, run = state["python_evidence"], state["python_run"] + gate.update(event="pull_request", pr=7, base=BASE) + gate.pop("main_proofs") + gate["jobs"].pop(state["network_name"], None) + state["python_jobs"][:] = [ + job for job in state["python_jobs"] if job["name"] != state["network_name"] + ] + run.update(event="pull_request", path=".github/workflows/ci.yml") + result = publishing.gate_for_run( + REPO, run["id"], head=run["head_sha"], base=BASE, pr=7 + ) + assert result == gate + assert not any("native-feed-post.yml" in route for route in state["calls"]) + + +@pytest.fixture +def candidate_marker_transport(main_proof_transport, monkeypatch): + state = main_proof_transport + gate, run = state["python_evidence"], state["python_run"] + gate.update(event="pull_request", pr=7, base=BASE) + gate.pop("main_proofs") + state["python_jobs"][:] = [ + job for job in state["python_jobs"] if job["name"] != state["network_name"] + ] + for job in state["python_jobs"]: + job["completed_at"] = "2026-11-07T11:59:00Z" + run.update( + event="pull_request", path=".github/workflows/ci.yml", + pull_requests=[{"number": 7}], + ) + pr = { + "number": 7, "merged_at": "2026-11-07T12:00:00Z", + "head": {"sha": run["head_sha"], "repo": {"full_name": REPO}}, + "body": automation.body({ + "kind": "release", "version": "0.3.1", "base": BASE, + "generated_head": run["head_sha"], + }), + } + state.update(candidate_pr=pr, candidate_reads=[]) + + def api(repo, endpoint, method="GET", data=None): + assert repo == REPO and method == "GET" and data is None + state["candidate_reads"].append(endpoint) + if endpoint == ( + "actions/workflows/ci.yml/runs?event=pull_request&head_sha=" + + run["head_sha"] + "&per_page=100&page=1" + ): + return {"workflow_runs": copy.deepcopy(state["python_runs"])} + assert endpoint == ( + f"actions/runs/{run['id']}/attempts/{run['run_attempt']}/jobs" + "?per_page=100&page=1" + ) + return contract.object_json(evidence.gh_bytes(f"repos/{repo}/{endpoint}")) + + monkeypatch.setattr(automation, "api", api) + return state + + +@pytest.mark.parametrize("body", [None, "", "Release notes without machine metadata"]) +def test_candidate_gate_refuses_missing_release_marker(candidate_marker_transport, body): + state = candidate_marker_transport + pr = state["candidate_pr"] + pr["body"] = body + assert automation.marker(pr) is None + with pytest.raises(ValueError, match=r"^pre-merge release marker missing$"): + publishing.candidate_gate(REPO, pr) + assert len(state["candidate_reads"]) == 1 + assert state["calls"] == state["artifact_observations"] == [] + + +def test_candidate_gate_accepts_release_marker_with_real_gate(candidate_marker_transport): + state = candidate_marker_transport + result = publishing.candidate_gate(REPO, state["candidate_pr"]) + assert result == state["python_evidence"] + assert len(state["candidate_reads"]) == 2 + run = state["python_run"] + assert f"repos/{REPO}/actions/artifacts/{run['id'] * 100}/zip" in state["calls"] + assert not any("native-feed-post.yml" in route for route in state["calls"]) + + +@pytest.mark.parametrize("missing", ["head", "run"]) +def test_candidate_gate_keeps_ci_priority_before_missing_marker( + candidate_marker_transport, missing +): + state = candidate_marker_transport + pr = state["candidate_pr"] + pr["body"] = None + if missing == "head": + pr["head"]["sha"] = None + else: + state["python_runs"] = [] + with pytest.raises(ValueError, match=r"^pre-merge CI evidence missing$"): + publishing.candidate_gate(REPO, pr) + assert len(state["candidate_reads"]) == int(missing == "run") + assert state["calls"] == [] + + +class PublicationFetchBoundary(RuntimeError): + """Stop immediately before prepare's first external git fetch.""" + + +@pytest.fixture +def main_callback_transport(main_proof_transport, tmp_path, monkeypatch): + """Real prepare/gate/validators; only observed API bytes and I/O boundaries vary.""" + state = main_proof_transport + source = state["python_run"]["head_sha"] + generated = contract.sha256(f"{source}:release-pr".encode())[:40] + event = contract.object_json((FIXTURES / "ready-event.json").read_bytes()) + pr = { + "number": 7, "merged": True, "merged_at": "2026-11-07T12:00:00Z", + "merge_commit_sha": source, "base": {"ref": "main", "sha": BASE}, + "head": {"sha": generated, "repo": {"full_name": REPO}}, + "user": {"login": "release[bot]"}, + "body": automation.body({ + "kind": "release", "version": "0.3.1", + "generated_head": generated, "base": BASE, "event_id": event["event_id"], + }), + } + state.update( + callback_pr=pr, api_calls=[], gate_attempts=[], validated_pairs=[], effects=[], + callback_root=tmp_path, output=tmp_path / "publication", + callback_report=receipts.receipt(None), + ) + state["report_before"] = copy.deepcopy(state["callback_report"]) + + def api(repo, endpoint, method="GET", data=None): + assert repo == REPO and method == "GET" and data is None + state["api_calls"].append(endpoint) + if endpoint.startswith("actions/runs/"): + return contract.object_json(evidence.gh_bytes(f"repos/{repo}/{endpoint}")) + if endpoint == f"commits/{source}/pulls?per_page=100": + return [copy.deepcopy(pr)] + if endpoint == "pulls/7": + return copy.deepcopy(pr) + raise AssertionError(f"unexpected callback API read: {endpoint}") + + real_gate = publishing.gate_for_run + + def observe_real_gate(repo, run_id, **kwargs): + state["gate_attempts"].append((repo, run_id, kwargs)) + result = real_gate(repo, run_id, **kwargs) + state["validated_pairs"].append(copy.deepcopy(result)) + return result + + def before_git_fetch(*args, **kwargs): + state["effects"].append(("git", args)) + assert not kwargs + assert args == ("fetch", "origin", source) + # This sentinel is after the actual gate, before the fetch is executed. + raise PublicationFetchBoundary("before git fetch origin exact main head") + + def reject_snapshot(*args, **kwargs): + state["effects"].append(("snapshot", args)) + raise AssertionError("prepare must not snapshot before complete proof") + + def reject_process(command, **kwargs): + state["effects"].append(("process", command)) + raise AssertionError("prepare must not download or publish before complete proof") + + def reject_publication_transport(*args, **kwargs): + state["effects"].append(("publication-transport", args)) + raise AssertionError("publication byte transport reached before the fetch boundary") + + monkeypatch.setenv("GITHUB_EVENT_NAME", "workflow_run") + monkeypatch.setenv("RELEASE_BOT_LOGIN", "release[bot]") + monkeypatch.setattr(publishing, "api", api) + monkeypatch.setattr(publishing, "gate_for_run", observe_real_gate) + monkeypatch.setattr(publishing, "git", before_git_fetch) + monkeypatch.setattr(publishing, "snapshot", reject_snapshot) + monkeypatch.setattr(publishing.subprocess, "run", reject_process) + monkeypatch.setattr(publishing, "fetch", reject_publication_transport) + monkeypatch.setattr(publishing, "gh_bytes", reject_publication_transport) + return state + + +def prepare_main_callback(state, source, *, run_id=None): + if run_id is None: + run_id = state["python_run" if source == "CI" else "native_run"]["id"] + return publishing.prepare( + state["callback_root"], REPO, run_id, state["output"], state["callback_report"] + ) + + +def assert_callback_created_nothing(state): + assert state["effects"] == [] + assert not state["output"].exists() + assert state["callback_report"] == state["report_before"] + assert state["validated_pairs"] == [] + + +def assert_callback_validated_current_pair(state, *, since=0): + python, native = state["python_run"], state["native_run"] + calls = state["calls"][since:] + result = state["validated_pairs"][-1] + assert result["run_id"] == python["id"] and result["attempt"] == python["run_attempt"] + assert result["main_proofs"]["network"] == state["python_evidence"]["main_proofs"]["network"] + assert result["main_proofs"]["native"] == { + "workflow_path": native["path"], "run": native, "jobs": state["native_jobs"], + "attestation": state["attestation"], + } + assert state["gate_attempts"][-1] == (REPO, python["id"], {"head": python["head_sha"]}) + assert "pulls/7" in state["api_calls"] + assert any("/actions/workflows/native-feed-post.yml/runs?" in route for route in calls) + prefix = f"repos/{REPO}/actions" + for run, artifact in ((python, "release-evidence"), (native, "native-evidence")): + assert ( + f"{prefix}/runs/{run['id']}/attempts/{run['run_attempt']}/jobs?per_page=100" + in calls + ) + assert f"{prefix}/artifacts/{run['id'] * 100}/zip" in calls + assert ( + run["id"], run["run_attempt"], f"{artifact}-{run['run_attempt']}" + ) in state["artifact_observations"] + assert not state["output"].exists() + assert state["callback_report"] == state["report_before"] + + +@pytest.fixture +def refreshed_release_pr(main_callback_transport, monkeypatch): + state = main_callback_transport + refreshed = copy.deepcopy(state["callback_pr"]) + real_api = publishing.api + + def api(repo, endpoint, method="GET", data=None): + observed = real_api(repo, endpoint, method, data) + return copy.deepcopy(refreshed) if endpoint == "pulls/7" else observed + + monkeypatch.setattr(publishing, "api", api) + return state, refreshed + + +def test_prepare_refuses_missing_refreshed_release_marker(refreshed_release_pr): + state, refreshed = refreshed_release_pr + refreshed["body"] = None + assert automation.marker(state["callback_pr"]) is not None + assert automation.marker(refreshed) is None + with pytest.raises(ValueError, match=r"^release PR marker missing$"): + prepare_main_callback(state, "CI") + source = state["python_run"]["head_sha"] + assert state["api_calls"] == [ + f"actions/runs/{state['python_run']['id']}", + f"commits/{source}/pulls?per_page=100", "pulls/7", + ] + assert state["calls"] == [f"repos/{REPO}/actions/runs/{state['python_run']['id']}"] + assert state["gate_attempts"] == [] + assert_callback_created_nothing(state) + + +def test_prepare_keeps_owner_priority_before_missing_marker(refreshed_release_pr): + state, refreshed = refreshed_release_pr + refreshed.update(body=None, user={"login": "other-owner"}) + with pytest.raises(ValueError, match=r"^unexpected release PR owner$"): + prepare_main_callback(state, "CI") + assert state["gate_attempts"] == [] + assert_callback_created_nothing(state) + + +def test_prepare_accepts_refreshed_release_marker(refreshed_release_pr): + state, _refreshed = refreshed_release_pr + with pytest.raises( + PublicationFetchBoundary, match=r"^before git fetch origin exact main head$" + ): + prepare_main_callback(state, "CI") + assert_callback_validated_current_pair(state) + assert state["effects"] == [("git", ("fetch", "origin", state["python_run"]["head_sha"]))] + + +@pytest.mark.parametrize("first_source", ["CI", "Native feed"]) +def test_main_callbacks_validate_the_same_pair_before_fetch(main_callback_transport, first_source): + state = main_callback_transport + second_source = "Native feed" if first_source == "CI" else "CI" + for source in (first_source, second_source): + since = len(state["calls"]) + with pytest.raises(PublicationFetchBoundary, match="before git fetch"): + prepare_main_callback(state, source) + assert_callback_validated_current_pair(state, since=since) + assert len(state["gate_attempts"]) == len(state["validated_pairs"]) == 2 + assert state["validated_pairs"][0] == state["validated_pairs"][1] + assert state["effects"] == [ + ("git", ("fetch", "origin", state["python_run"]["head_sha"])) + ] * 2 + # Two unit callbacks reach one validated pair. Publication/idempotency is not run. + + +@pytest.mark.parametrize("first_source", ["CI", "Native feed"]) +@pytest.mark.parametrize("counterpart_state", ["missing", "pending", "failure"]) +def test_main_callback_completion_order_waits_for_counterpart( + main_callback_transport, incident_transport, first_source, counterpart_state +): + state = main_callback_transport + counterpart = "native" if first_source == "CI" else "python" + run = state[f"{counterpart}_run"] + if counterpart_state == "missing": + state[f"{counterpart}_runs"] = [] + workflow = "native-feed-post.yml" if counterpart == "native" else "ci-post.yml" + reason = f"missing exact-head main {workflow} run" + else: + if counterpart_state == "pending": + run.update(status="in_progress", conclusion=None) + else: + run["conclusion"] = "failure" + reason = ( + "newest native proof has not completed successfully" if counterpart == "native" + else "CI has not completed successfully" + ) + if counterpart_state == "failure": + with pytest.raises(ValueError) as rejected: + prepare_main_callback(state, first_source) + assert str(rejected.value) == reason + else: + assert prepare_main_callback(state, first_source) == { + "eligible": "false", "waiting": "main-proofs-pending", + } + collect_callback_without_incident(state, incident_transport) + assert_callback_created_nothing(state) + if counterpart == "python" and counterpart_state == "missing": + assert state["gate_attempts"] == [] + else: + assert state["gate_attempts"] == [ + (REPO, state["python_run"]["id"], {"head": state["python_run"]["head_sha"]}) + ] + # The later counterpart completion changes the input; both real gates remain. + run.update(status="completed", conclusion="success") + state[f"{counterpart}_runs"] = [run] + second_source = "Native feed" if first_source == "CI" else "CI" + since = len(state["calls"]) + with pytest.raises(PublicationFetchBoundary, match="before git fetch"): + prepare_main_callback(state, second_source) + assert_callback_validated_current_pair(state, since=since) + assert state["effects"] == [("git", ("fetch", "origin", run["head_sha"]))] + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize("slot", ["python", "native"]) +@pytest.mark.parametrize("readiness", ["missing", "queued", "in_progress"]) +def test_main_callbacks_wait_without_incident_for_observed_pending_pair( + main_callback_transport, observed_main_parser, incident_transport, source, slot, readiness +): + state = main_callback_transport + run = state[f"{slot}_run"] + if readiness == "missing": + state[f"{slot}_runs"] = [] + else: + run.update(status=readiness, conclusion=None) + state[f"{slot}_artifact_missing"] = True + for job in state[f"{slot}_jobs"]: + job.update(status=readiness, conclusion=None) + assert prepare_main_callback(state, source) == { + "eligible": "false", "waiting": "main-proofs-pending", + } + assert_callback_created_nothing(state) + collect_callback_without_incident(state, incident_transport) + workflow = "ci-post.yml" if slot == "python" else "native-feed-post.yml" + assert any(f"/actions/workflows/{workflow}/runs?" in row for row in state["calls"]) + observed = {"workflow_runs": state[f"{slot}_runs"]} + assert observed in state["parsed_objects"], "the real parser did not consume readiness" + assert state["main_validations"] == [] + if slot == "python" and readiness == "missing": + assert state["gate_attempts"] == [] # There is no Python run ID to gate yet. + else: + python = state["python_run"] + assert state["gate_attempts"] == [(REPO, python["id"], {"head": python["head_sha"]})] + + # The same lifecycle consumes new completion input and reaches the real gate. + run.update(status="completed", conclusion="success") + state[f"{slot}_runs"] = [run] + state[f"{slot}_artifact_missing"] = False + for job in state[f"{slot}_jobs"]: + job.update(status="completed", conclusion="success") + with pytest.raises(PublicationFetchBoundary, match="before git fetch"): + prepare_main_callback(state, source) + assert_callback_validated_current_pair(state) + assert state["main_validations"][-1] == state["validated_pairs"][-1] + assert incident_transport["reads"] == incident_transport["writes"] == [] + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize("readiness", ["missing", "queued", "in_progress"]) +@pytest.mark.parametrize( + "fault,reason", + [ + ("stale-attempt", "stale CI attempt"), + ("network-non-object", "network: expected an object"), + ( + "network-job-red", + "required job 'Live PyPI hashes' missing, duplicated, failed, cancelled or skipped", + ), + ("wrong-tree", "network: wrong tree"), + ("e2e-incomplete", "incomplete E2E evidence"), + ], +) +@pytest.mark.parametrize("existing_issue", [False, True], ids=["new-issue", "existing-issue"]) +def test_main_callbacks_refuse_invalid_python_while_native_pending( + main_callback_transport, observed_main_parser, incident_transport, + source, readiness, fault, reason, existing_issue, +): + state = main_callback_transport + assert observed_main_parser is state + python, native = state["python_run"], state["native_run"] + current_python = copy.deepcopy(python) + proof = state["python_evidence"] + if readiness == "missing": + state["native_runs"] = [] + else: + native.update(status=readiness, conclusion=None) + state["native_artifact_missing"] = True + for job in state["native_jobs"]: + job.update(status=readiness, conclusion=None) + if fault == "stale-attempt": + proof["attempt"] -= 1 + elif fault == "network-non-object": + proof["main_proofs"]["network"] = [] + elif fault == "network-job-red": + next(job for job in state["python_jobs"] if job["name"] == state["network_name"])[ + "conclusion" + ] = "failure" + elif fault == "wrong-tree": + original_tree = proof["tree"] + proof["tree"] = contract.sha256(b"another completed Python evidence tree")[:40] + assert proof["tree"] != original_tree + assert proof["main_proofs"]["network"]["tree"] == original_tree + else: + assert fault == "e2e-incomplete" + for suite in ("docker", "database"): + proof["e2e"][suite].pop() + assert python == current_python + assert python["status"] == "completed" and python["conclusion"] == "success" + with pytest.raises(evidence.GateRejected) as rejected: + prepare_main_callback(state, source) + assert type(rejected.value) is evidence.GateRejected + assert str(rejected.value) == reason + assert_callback_created_nothing(state) + assert state["gate_attempts"] == [(REPO, python["id"], {"head": python["head_sha"]})] + assert_native_discovery(state) + assert {"workflow_runs": state["native_runs"]} in state["parsed_objects"] + if readiness != "missing": + assert native in state["parsed_objects"] + assert proof in state["parsed_objects"] + assert {"jobs": state["python_jobs"]} in state["parsed_objects"] + assert state["waiting_inputs"] == [ + (proof, python, state["python_jobs"], {"head": python["head_sha"], "base": None}), + ] + assert state["waiting_validations"] == state["gate_inputs"] == state["main_inputs"] == [] + assert state["artifact_observations"] == [ + (python["id"], python["run_attempt"], f"release-evidence-{python['run_attempt']}"), + ] + incidents = incident_transport + if existing_issue: + incidents["issues"].append({ + "number": 41, "state": "closed", + "body": "\nolder observation", + }) + report = reporting.collect( + state["callback_root"] / "downloaded-stages", failed=True, stage="prepare-publication" + ) + assert report["state"] == "failed" and report["event_id"] is None + assert report["attempt"] == 930 and report["sequence"] == 2 + reporting.incident(REPO, report, "release-owner", "prepare-publication") + assert len(incidents["reads"]) == 1 + assert [(route, method) for route, method, _ in incidents["writes"]] == ( + [("issues/41/comments", "POST"), ("issues/41", "PATCH")] if existing_issue else + [("issues", "POST"), ("issues/41", "PATCH")] + ) + assert len(incidents["issues"]) == 1 + issue = incidents["issues"][0] + assert issue["state"] == "open" and issue["title"] == "Data release failed: run-930" + assert "**failed**" in issue["body"] and "`prepare-publication`" in issue["body"] + assert_callback_created_nothing(state) + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize("readiness", ["missing", "queued", "in_progress"]) +def test_main_callbacks_wait_with_valid_python_while_native_pending_positive_twin( + main_callback_transport, observed_main_parser, incident_transport, source, readiness +): + state = main_callback_transport + assert observed_main_parser is state + python, native = state["python_run"], state["native_run"] + if readiness == "missing": + state["native_runs"] = [] + else: + native.update(status=readiness, conclusion=None) + state["native_artifact_missing"] = True + for job in state["native_jobs"]: + job.update(status=readiness, conclusion=None) + assert python["status"] == "completed" and python["conclusion"] == "success" + assert prepare_main_callback(state, source) == { + "eligible": "false", "waiting": "main-proofs-pending", + } + assert_callback_created_nothing(state) + collect_callback_without_incident(state, incident_transport) + assert incident_transport["issues"] == incident_transport["comments"] == [] + assert state["gate_attempts"] == [(REPO, python["id"], {"head": python["head_sha"]})] + assert_native_discovery(state) + assert {"workflow_runs": state["native_runs"]} in state["parsed_objects"] + if readiness != "missing": + assert native in state["parsed_objects"] + proof = state["python_evidence"] + assert proof in state["parsed_objects"] + assert {"jobs": state["python_jobs"]} in state["parsed_objects"] + assert state["waiting_inputs"] == [ + (proof, python, state["python_jobs"], {"head": python["head_sha"], "base": None}), + ] + assert state["waiting_validations"] == [proof] + assert state["gate_inputs"] == state["main_inputs"] == [] + assert state["artifact_observations"] == [ + (python["id"], python["run_attempt"], f"release-evidence-{python['run_attempt']}"), + ] + + +@pytest.mark.parametrize("existing_issue", [False, True], ids=["new-issue", "existing-issue"]) +def test_callback_reporting_failure_still_opens_or_updates_incident( + tmp_path, incident_transport, existing_issue +): + incidents = incident_transport + if existing_issue: + incidents["issues"].append({ + "number": 41, "state": "closed", + "body": "\nolder observation", + }) + report = reporting.collect(tmp_path, failed=True, stage="prepare-publication") + assert report["state"] == "failed" and report["event_id"] is None + assert report["attempt"] == 930 and report["sequence"] == 2 + reporting.incident(REPO, report, "release-owner", "prepare-publication") + assert len(incidents["reads"]) == 1 + assert [(route, method) for route, method, _ in incidents["writes"]] == ( + [("issues/41/comments", "POST"), ("issues/41", "PATCH")] if existing_issue else + [("issues", "POST"), ("issues/41", "PATCH")] + ) + issue = incidents["issues"][0] + assert issue["state"] == "open" + assert issue["title"] == "Data release failed: run-930" + assert "**failed**" in issue["body"] and "`prepare-publication`" in issue["body"] + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize("slot", ["python", "native"]) +@pytest.mark.parametrize( + "fault", ["failure", "cancelled", "wrong-head", "wrong-attempt", + "malformed-proof", "malformed-json", "missing-artifact"], +) +def test_main_callbacks_keep_completed_and_malformed_proofs_decisive( + main_callback_transport, observed_main_parser, incident_transport, source, slot, fault +): + state = main_callback_transport + run = state[f"{slot}_run"] + callback_id = None + if fault in {"failure", "cancelled"}: + run["conclusion"] = fault + reason = ( + "CI has not completed successfully" if slot == "python" + else "newest native proof has not completed successfully" + ) + elif fault == "wrong-head": + # The list still has the requested head; the current detail read contradicts it. + state[f"{slot}_runs"] = [copy.deepcopy(run)] + if source == ("CI" if slot == "python" else "Native feed"): + older_callback = {**run, "id": run["id"] - 1, "run_attempt": 1} + state[f"{slot}_runs"].append(older_callback) + callback_id = older_callback["id"] + run["head_sha"] = contract.sha256(b"contradictory current head")[:40] + reason = ( + "counterpart is not exact-head main CI" if slot == "python" + else "newest native proof has not completed successfully" + ) + elif fault == "wrong-attempt": + if slot == "python": + state["python_evidence"]["attempt"] -= 1 + reason = "stale CI attempt" + else: + state["attestation"]["attempt"] -= 1 + reason = "native: wrong attempt" + elif fault == "malformed-proof": + if slot == "python": + state["python_evidence"]["main_proofs"]["network"] = [] + reason = "network: expected an object" + else: + state["attestation"]["legs"] = [] + reason = "native legs: expected an object" + elif fault == "malformed-json": + filename = "evidence.json" if slot == "python" else "native-evidence.json" + state["artifact_raw"][filename] = b'{"run_id":1,"run_id":2}' + reason = "duplicate JSON key: run_id" + else: + state[f"{slot}_artifact_missing"] = True + artifact = "release-evidence" if slot == "python" else "native-evidence" + reason = f"missing unique {artifact}-{run['run_attempt']} artifact" + with pytest.raises(ValueError) as rejected: + prepare_main_callback(state, source, run_id=callback_id) + assert str(rejected.value) == reason + assert_callback_created_nothing(state) + assert state["parsed_objects"], "the real parser must observe the supplied refusal input" + if fault in {"wrong-attempt", "malformed-proof"}: + assert state["gate_inputs"], "the real validator must consume the artifact mutation" + if fault == "malformed-json": + assert b'{"run_id":1,"run_id":2}' in state["parsed_bytes"] + # A real refusal must still take the actionable failure path. + report = reporting.collect( + state["callback_root"], failed=True, stage="prepare-publication" + ) + reporting.incident(REPO, report, "release-owner", "prepare-publication") + assert [(route, method) for route, method, _ in incident_transport["writes"]] == [ + ("issues", "POST"), ("issues/41", "PATCH"), + ] + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize( + "fault", + ["native-tree", "native-attempt", "native-leg", "native-job-red", + "network-skipped", "network-job-red", "missing-native-artifact"], +) +def test_main_callbacks_refuse_invalid_proof_before_any_preparation( + main_callback_transport, source, fault +): + state = main_callback_transport + if fault == "native-tree": + state["attestation"]["tree"] = contract.sha256(b"other tree")[:40] + reason = "native: wrong tree" + elif fault == "native-attempt": + state["attestation"]["attempt"] -= 1 + reason = "native: wrong attempt" + elif fault == "native-leg": + state["attestation"]["legs"].pop("tsan") + reason = "native: missing or unexpected leg" + elif fault == "native-job-red": + next(job for job in state["native_jobs"] if job["name"] == "TSan")[ + "conclusion" + ] = "failure" + reason = "required job 'TSan' missing, duplicated, failed, cancelled or skipped" + elif fault == "network-skipped": + network = state["python_evidence"]["main_proofs"]["network"] + network.update(executed=[], skipped=[network["node_id"]]) + reason = "network executed: missing nonempty inventory" + elif fault == "network-job-red": + next(job for job in state["python_jobs"] if job["name"] == state["network_name"])[ + "conclusion" + ] = "failure" + reason = "required job 'Live PyPI hashes' missing, duplicated, failed, cancelled or skipped" + else: + state["native_artifact_missing"] = True + reason = f"missing unique native-evidence-{state['native_run']['run_attempt']} artifact" + with pytest.raises(evidence.GateRejected) as rejected: + prepare_main_callback(state, source) + assert str(rejected.value) == reason + assert state["gate_attempts"] == [ + (REPO, state["python_run"]["id"], {"head": state["python_run"]["head_sha"]}) + ] + assert_callback_created_nothing(state) + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize("fault", ["owner", "changed-pr-head", "ambiguous-marker"]) +def test_main_callbacks_keep_release_authorization_before_gate( + main_callback_transport, source, fault +): + state = main_callback_transport + pr = state["callback_pr"] + if fault == "owner": + pr["user"]["login"] = "other[bot]" + reason = "unexpected release PR owner" + elif fault == "changed-pr-head": + pr["head"]["sha"] = contract.sha256(b"changed PR head")[:40] + reason = "release PR head changed" + else: + pr["body"] += pr["body"] + reason = "ambiguous release PR marker" + with pytest.raises(ValueError) as rejected: + prepare_main_callback(state, source) + assert str(rejected.value) == reason + assert state["gate_attempts"] == [] + assert_callback_created_nothing(state) + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize( + "fault", ["pending", "failure", "cancelled", "stale-attempt", "network-red"] +) +def test_main_callbacks_never_replace_newer_bad_python_with_old_green( + main_callback_transport, source, fault +): + state = main_callback_transport + python = state["python_run"] + older = {**python, "id": python["id"] - 1, "run_attempt": 1} + state["python_runs"] = [python, older] + if fault == "pending": + python.update(status="in_progress", conclusion=None) + reason = "CI has not completed successfully" + elif fault in {"failure", "cancelled"}: + python["conclusion"] = fault + reason = "CI has not completed successfully" + elif fault == "stale-attempt": + state["python_evidence"]["attempt"] -= 1 + reason = "stale CI attempt" + else: + next(job for job in state["python_jobs"] if job["name"] == state["network_name"])[ + "conclusion" + ] = "failure" + reason = "required job 'Live PyPI hashes' missing, duplicated, failed, cancelled or skipped" + callback_id = older["id"] if source == "CI" else state["native_run"]["id"] + if fault == "pending": + assert prepare_main_callback(state, source, run_id=callback_id) == { + "eligible": "false", "waiting": "main-proofs-pending", + } + else: + with pytest.raises(evidence.GateRejected) as rejected: + prepare_main_callback(state, source, run_id=callback_id) + assert str(rejected.value) == reason + assert state["gate_attempts"] == [(REPO, python["id"], {"head": python["head_sha"]})] + assert not any( + f"/runs/{older['id']}/" in route or f"/artifacts/{older['id'] * 100}/" in route + for route in state["calls"] + ), "the old callback may be read; its jobs/artifacts cannot replace the newer Python run" + assert_callback_created_nothing(state) + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize( + "fault", ["pending", "failure", "cancelled", "missing-artifact", "wrong-tree"] +) +def test_main_callbacks_never_replace_newer_bad_native_with_old_green( + main_callback_transport, source, fault +): + state = main_callback_transport + native = state["native_run"] + older = {**native, "id": native["id"] - 1, "run_attempt": 1} + state["native_runs"] = [native, older] + if fault == "pending": + native.update(status="in_progress", conclusion=None) + reason = "newest native proof has not completed successfully" + elif fault in {"failure", "cancelled"}: + native["conclusion"] = fault + reason = "newest native proof has not completed successfully" + elif fault == "missing-artifact": + state["native_artifact_missing"] = True + reason = f"missing unique native-evidence-{native['run_attempt']} artifact" + else: + state["attestation"]["tree"] = contract.sha256(b"other native tree")[:40] + reason = "native: wrong tree" + callback_id = older["id"] if source == "Native feed" else state["python_run"]["id"] + if fault == "pending": + assert prepare_main_callback(state, source, run_id=callback_id) == { + "eligible": "false", "waiting": "main-proofs-pending", + } + else: + with pytest.raises(evidence.GateRejected) as rejected: + prepare_main_callback(state, source, run_id=callback_id) + assert str(rejected.value) == reason + assert state["gate_attempts"] == [ + (REPO, state["python_run"]["id"], {"head": state["python_run"]["head_sha"]}) + ] + assert_native_discovery(state) + assert not any( + f"/runs/{older['id']}/" in route or f"/artifacts/{older['id'] * 100}/" in route + for route in state["calls"] + ), "an old native callback cannot select old native jobs/artifacts" + assert_callback_created_nothing(state) + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +def test_main_callbacks_use_reobserved_current_attempt(main_callback_transport, source): + state = main_callback_transport + python, native = state["python_run"], state["native_run"] + state["python_runs"] = [copy.deepcopy(python)] + state["native_runs"] = [copy.deepcopy(native)] + for run in (python, native): + run["run_attempt"] += 1 + state["python_evidence"]["attempt"] = python["run_attempt"] + state["python_evidence"]["main_proofs"]["network"]["attempt"] = python["run_attempt"] + state["attestation"]["attempt"] = native["run_attempt"] + with pytest.raises(PublicationFetchBoundary, match="before git fetch"): + prepare_main_callback(state, source) + assert_callback_validated_current_pair(state) + for run in (python, native): + assert not any( + f"/runs/{run['id']}/attempts/{run['run_attempt'] - 1}/" in route + for route in state["calls"] + ) + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize("slot", ["python", "native"]) +def test_main_callbacks_refuse_artifact_from_previous_attempt( + main_callback_transport, source, slot +): + state = main_callback_transport + state[f"{slot}_artifact_attempt_delta"] = -1 + run = state[f"{slot}_run"] + artifact = "release-evidence" if slot == "python" else "native-evidence" + with pytest.raises(evidence.GateRejected) as rejected: + prepare_main_callback(state, source) + assert str(rejected.value) == f"missing unique {artifact}-{run['run_attempt']} artifact" + assert state["gate_attempts"] == [ + (REPO, state["python_run"]["id"], {"head": state["python_run"]["head_sha"]}) + ] + assert_callback_created_nothing(state) + + +def test_main_callback_old_green_transport_is_a_usable_positive_twin(main_callback_transport): + state = main_callback_transport + python, native = state["python_run"], state["native_run"] + older_python = {**python, "id": python["id"] - 1, "run_attempt": 1} + older_native = {**native, "id": native["id"] - 1, "run_attempt": 1} + state["python_runs"], state["native_runs"] = [older_python], [older_native] + with pytest.raises(PublicationFetchBoundary, match="before git fetch"): + prepare_main_callback(state, "CI", run_id=older_python["id"]) + result = state["validated_pairs"][-1] + assert result["run_id"] == older_python["id"] and result["attempt"] == 1 + assert result["main_proofs"]["network"]["run_id"] == older_python["id"] + assert result["main_proofs"]["native"]["run"]["id"] == older_native["id"] + assert result["main_proofs"]["native"]["attestation"]["run_id"] == older_native["id"] + assert state["effects"] == [("git", ("fetch", "origin", python["head_sha"]))] + assert not state["output"].exists() + assert state["callback_report"] == state["report_before"] + + +class PublicationCandidateBoundary(RuntimeError): + """Stop before the later, independent pre-merge candidate API inventory.""" + + +@pytest.fixture +def publication_main_transport( + main_callback_transport, observed_main_parser, bootstrap_lock, tmp_path, monkeypatch +): + """Save a real full gate, then re-observe it through publication's lower I/O.""" + state = main_callback_transport + python, native = state["python_run"], state["native_run"] + source = python["head_sha"] + event = contract.object_json((FIXTURES / "ready-event.json").read_bytes()) + manifest = (FIXTURES / "release-manifest.json").read_bytes() + pr = state["callback_pr"] + # Reuse the authored package-byte inputs of test_prepare_passes_adopter_lock_for_previous. + # These controls validate proof/hash handling; they do not install or serve a package. + package_directory = tmp_path / "observed-package-bytes" + package_directory.mkdir() + for name, raw in { + "pineforge_data-0.3.1-py3-none-any.whl": b"tested wheel", + "pineforge_data-0.3.1.tar.gz": b"tested sdist", + }.items(): + (package_directory / name).write_bytes(raw) + state["python_evidence"]["distributions"] = evidence.distribution_hashes(package_directory) + native.update( + updated_at="2026-11-07T12:00:00Z", display_title="Original completed run title", + ) + for run, jobs in ((python, state["python_jobs"]), (native, state["native_jobs"])): + for index, job in enumerate(jobs, 1): + job.update( + id=run["id"] * 1000 + index, run_id=run["id"], + run_attempt=run["run_attempt"], head_sha=source, + updated_at="2026-11-07T12:00:00Z", + ) + for leg, inventory in state["attestation"]["legs"].items(): + inventory["artifact"] = f"native-leg-{leg}-{native['run_attempt']}" + inventory["registration_sha256"] = contract.sha256(contract.json_bytes({ + "tests": [{"name": name} for name in inventory["registered"]], + })) + saved = reporting.gate_for_run(REPO, python["id"], head=source) + assert set(saved["main_proofs"]) == {"network", "native"} + assert state["main_validations"][-1] == saved + staged = contract.object_json((FIXTURES / "sample-release-receipt.json").read_bytes()) + staged.update( + pr=automation.pr_identity(pr), state="deploying", + deployed=receipts.served(event, source, "0.3.1", saved["distributions"]), + ) + lock = {**copy.deepcopy(bootstrap_lock), "event": event} + source_files = { + contract.RUNTIME: ( + f'DEFAULT_RELEASE_IMAGE = (\n "{contract.image_ref(event)}"\n)\n' + ).encode(), + contract.DOCKERFILE: ( + f"ARG PINEFORGE_RELEASE_IMAGE={contract.image_ref(event)}\n" + f"ARG PINEFORGE_HUB_COMMIT={event['hub_commit']}\n" + ).encode(), + "pyproject.toml": b'[project]\nname = "pineforge-data"\nversion = "0.3.1"\n', + contract.LOCK: contract.json_bytes(lock), contract.MANIFEST: manifest, + } + state.update( + saved_gate=copy.deepcopy(saved), staged=staged, package_directory=package_directory, + publication_reads=[], publication_writes=[], fresh_gates=[], local_validations=[], + candidate_boundaries=[], downloads=[], download_files=[], + ) + real_transport = evidence.gh_bytes + real_gate = reporting.gate_for_run + real_verify_local = reporting.verify_local + real_temporary_directory = reporting.tempfile.TemporaryDirectory + + def api(repo, endpoint, method="GET", data=None): + assert repo == REPO + if method != "GET": + state["publication_writes"].append((endpoint, method, copy.deepcopy(data))) + raise AssertionError("publication must not mutate before the candidate boundary") + assert data is None + state["publication_reads"].append(endpoint) + if endpoint == "actions/runs/200": + return { + "id": 200, "run_attempt": 1, "path": ".github/workflows/release.yml", + "event": "workflow_run", "head_sha": source, + "status": "completed", "conclusion": "success", + } + if endpoint == "actions/runs/200/artifacts?per_page=100&page=1": + return {"artifacts": [{ + "id": 200001, "name": "release-stage-1-50-verify", "expired": False, + }]} + if endpoint == "pulls/7": + return copy.deepcopy(pr) + if endpoint == f"commits/{source}": + return {"parents": [{"sha": BASE}]} + if endpoint == ( + "actions/workflows/ci.yml/runs?event=pull_request&head_sha=" + + pr["head"]["sha"] + "&per_page=100&page=1" + ): + state["candidate_boundaries"].append(endpoint) + assert len(state["fresh_gates"]) == len(state["local_validations"]) == 1 + assert state["local_validations"][0] == lock + raise PublicationCandidateBoundary("before pre-merge candidate inventory read") + raise AssertionError(f"unexpected publication API read: {endpoint}") + + def gh_bytes(route, **kwargs): + if route == f"repos/{REPO}/actions/runs/200/artifacts?per_page=100": + return contract.json_bytes({"artifacts": [{ + "id": 200001, "name": "release-stage-1-50-verify", "expired": False, + }]}) + if route == f"repos/{REPO}/actions/artifacts/200001/zip": + archive = io.BytesIO() + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("release-receipt.json", contract.json_bytes(staged)) + return archive.getvalue() + return real_transport(route, **kwargs) + + def hub_bytes(route, *, accept="application/vnd.github+json"): + state["publication_reads"].append(route) + if route == f"repos/{contract.HUB}/releases/tags/v{event['release_version']}": + assert accept == "application/vnd.github+json" + return contract.json_bytes({ + "draft": False, "prerelease": False, + "assets": [{"id": 81, "name": "release-manifest.json"}], + }) + assert route == f"repos/{contract.HUB}/releases/assets/81" + assert accept == "application/octet-stream" + return manifest + + def download(command, *, check): + assert check is True + bundle = Path(command[-1]) + assert command == [ + "gh", "run", "download", "200", "--repo", REPO, + "--name", "validated-release-bundle-1", "--dir", str(bundle), + ] + paths = [bundle / "evidence.json", *(bundle / "source" / name for name in source_files)] + for path in paths: + if ( + path.name.endswith(".env") + or not path.resolve().is_relative_to(tmp_path.resolve()) + ): + raise AssertionError(f"refused publication fixture file: {path}") + state["download_files"].append([str(path) for path in paths]) + state["downloads"].append(copy.deepcopy(command)) + (bundle / "evidence.json").write_bytes(contract.json_bytes(state["saved_gate"])) + for name, raw in source_files.items(): + path = bundle / "source" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(raw) + + def observe_gate(repo, run_id, **kwargs): + result = real_gate(repo, run_id, **kwargs) + state["fresh_gates"].append(copy.deepcopy(result)) + return result + + def verify_local(root): + result = real_verify_local(root) + state["local_validations"].append(copy.deepcopy(result)) + return result + + monkeypatch.setenv("GITHUB_RUN_ID", "300") + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "1") + monkeypatch.setattr(automation, "api", api) + monkeypatch.setattr(reporting, "api", api) + monkeypatch.setattr(evidence, "gh_bytes", gh_bytes) + monkeypatch.setattr(contract, "gh_bytes", hub_bytes) + monkeypatch.setattr(reporting.subprocess, "run", download) + monkeypatch.setattr(reporting, "gate_for_run", observe_gate) + monkeypatch.setattr(reporting, "verify_local", verify_local) + monkeypatch.setattr( + reporting.tempfile, "TemporaryDirectory", + lambda: real_temporary_directory(dir=tmp_path), + ) + return state + + +@pytest.mark.parametrize( + "metadata", + ["unchanged", "native-updated-at", "native-display-title", + "native-job-updated-at", "native-job-order"], +) +def test_publication_revalidates_full_main_proofs_despite_benign_metadata( + publication_main_transport, metadata +): + state = publication_main_transport + native = state["native_run"] + if metadata == "native-updated-at": + native["updated_at"] = "2026-11-08T00:00:00Z" + elif metadata == "native-display-title": + native["display_title"] = "Updated display title for the same completed run" + elif metadata == "native-job-updated-at": + state["native_jobs"][0]["updated_at"] = "2026-11-08T00:01:00Z" + elif metadata == "native-job-order": + state["native_jobs"].reverse() + report = receipts.receipt(None) + with pytest.raises(PublicationCandidateBoundary, match="before pre-merge candidate inventory"): + reporting.publication(REPO, 200, report) + fresh = state["fresh_gates"][0] + assert fresh["main_proofs"]["native"]["run"] == native + assert fresh["main_proofs"]["native"]["jobs"] == state["native_jobs"] + assert fresh == state["main_validations"][-1] + assert native in state["parsed_objects"] + assert {"jobs": state["native_jobs"]} in state["parsed_objects"] + assert state["python_evidence"] in state["parsed_objects"] + assert fresh["distributions"] == evidence.distribution_hashes(state["package_directory"]) + assert (fresh == state["saved_gate"]) is (metadata == "unchanged") + assert len(state["candidate_boundaries"]) == len(state["downloads"]) == 1 + assert len(state["download_files"][0]) == 6 + assert report["state"] == "deploying" and state["publication_writes"] == [] + # Partial lifecycle: real main proof and local metadata validation precede the sentinel. + # Pre-merge candidate validation, OCI/PyPI/previous identity and finalization do not run. + + +def test_publication_refuses_missing_release_marker(publication_main_transport): + state = publication_main_transport + state["callback_pr"]["body"] = None + assert automation.marker(state["callback_pr"]) is None + report = receipts.receipt(None) + with pytest.raises(ValueError, match=r"^publication PR identity moved$"): + reporting.publication(REPO, 200, report) + assert state["publication_reads"][-1] == "pulls/7" + assert state["fresh_gates"] == state["local_validations"] == [] + assert state["downloads"] == state["download_files"] == [] + assert state["candidate_boundaries"] == state["publication_writes"] == [] + assert report["state"] == "deploying" + + +def test_publication_accepts_release_marker_before_candidate_boundary( + publication_main_transport, +): + state = publication_main_transport + report = receipts.receipt(None) + with pytest.raises(PublicationCandidateBoundary, match="before pre-merge candidate inventory"): + reporting.publication(REPO, 200, report) + assert len(state["fresh_gates"]) == len(state["local_validations"]) == 1 + assert len(state["downloads"]) == len(state["candidate_boundaries"]) == 1 + assert state["publication_writes"] == [] + assert report["state"] == "deploying" + + +@pytest.mark.parametrize( + "fault,reason,gate_completes", + [ + ("native-id", "landed CI distributions/evidence moved", True), + ("native-attempt", "landed CI distributions/evidence moved", True), + ("tree", "landed CI distributions/evidence moved", True), + ("native-job-id", "landed CI distributions/evidence moved", True), + ("native-inventory", "landed CI distributions/evidence moved", True), + ("native-junit-digest", "landed CI distributions/evidence moved", True), + ("native-registration-digest", "landed CI distributions/evidence moved", True), + ("network-junit-digest", "landed CI distributions/evidence moved", True), + ("distribution", "landed CI distributions/evidence moved", True), + ("native-head", "newest native proof has not completed successfully", False), + ("native-workflow", "newest native proof has not completed successfully", False), + ("native-event", "newest native proof has not completed successfully", False), + ("native-conclusion", "newest native proof has not completed successfully", False), + ("native-tested-sha", "native: wrong tested_sha", False), + ("native-stale-attestation", "native: wrong attempt", False), + ("native-job-conclusion", + "required job 'TSan' missing, duplicated, failed, cancelled or skipped", False), + ("network-inventory", "network: missing unique required live-PyPI row", False), + ], +) +def test_publication_refuses_real_main_proof_identity_or_content_change( + publication_main_transport, fault, reason, gate_completes +): + state = publication_main_transport + native, attestation = state["native_run"], state["attestation"] + network = state["python_evidence"]["main_proofs"]["network"] + leg = attestation["legs"]["tsan"] + if fault in {"native-id", "native-attempt"}: + field = "id" if fault == "native-id" else "run_attempt" + native[field] += 1 + attestation.update(run_id=native["id"], attempt=native["run_attempt"]) + for job in state["native_jobs"]: + job.update(run_id=native["id"], run_attempt=native["run_attempt"]) + for name, inventory in attestation["legs"].items(): + inventory["artifact"] = f"native-leg-{name}-{native['run_attempt']}" + elif fault == "tree": + tree = contract.sha256(b"different landed tree")[:40] + state["python_evidence"]["tree"] = network["tree"] = attestation["tree"] = tree + elif fault == "native-job-id": + state["native_jobs"][0]["id"] += 1 + elif fault == "native-inventory": + leg["registered"].append("additional-native-row") + leg["executed"].append("additional-native-row") + junit = ElementTree.Element("testsuite") + for name in leg["executed"]: + ElementTree.SubElement(junit, "testcase", name=name) + leg["junit_sha256"] = contract.sha256(ElementTree.tostring(junit)) + leg["registration_sha256"] = contract.sha256(contract.json_bytes({ + "tests": [{"name": name} for name in leg["registered"]], + })) + elif fault in {"native-junit-digest", "native-registration-digest", "network-junit-digest"}: + target = network if fault == "network-junit-digest" else leg + field = "registration_sha256" if fault == "native-registration-digest" else "junit_sha256" + target[field] = contract.sha256(f"different report bytes: {fault}".encode()) + elif fault == "distribution": + wheel = state["package_directory"] / "pineforge_data-0.3.1-py3-none-any.whl" + wheel.write_bytes(wheel.read_bytes() + b"changed") + state["python_evidence"]["distributions"] = evidence.distribution_hashes( + state["package_directory"] + ) + elif fault in {"native-head", "native-workflow", "native-event"}: + state["native_runs"] = [copy.deepcopy(native)] + field, value = { + "native-head": ("head_sha", contract.sha256(b"different head")[:40]), + "native-workflow": ("path", ".github/workflows/native-feed.yml"), + "native-event": ("event", "workflow_dispatch"), + }[fault] + native[field] = value + elif fault == "native-conclusion": + native["conclusion"] = "failure" + elif fault == "native-tested-sha": + attestation["tested_sha"] = contract.sha256(b"different tested source")[:40] + elif fault == "native-stale-attestation": + attestation["attempt"] -= 1 + elif fault == "native-job-conclusion": + next(job for job in state["native_jobs"] if job["name"] == "TSan")[ + "conclusion" + ] = "failure" + else: + network["registered"].append("unrelated-network-row") + network["executed"].append("unrelated-network-row") + report = receipts.receipt(None) + with pytest.raises(ValueError) as rejected: + reporting.publication(REPO, 200, report) + assert str(rejected.value) == reason + assert len(state["fresh_gates"]) == int(gate_completes) + if gate_completes: + assert state["fresh_gates"][0] == state["main_validations"][-1] + assert state["fresh_gates"][0] != state["saved_gate"] + assert state["local_validations"] == state["candidate_boundaries"] == [] + assert len(state["downloads"]) == 1 and state["publication_writes"] == [] + assert report["state"] == "deploying" + + +@pytest.fixture +def retained_receipt_transport(tmp_path, monkeypatch): + report = contract.object_json((FIXTURES / "sample-release-receipt.json").read_bytes()) + pr = { + "number": report["pr"]["number"], "merged": True, "merge_commit_sha": MERGE, + "base": {"ref": "main", "sha": BASE}, + "head": {"sha": HEAD, "repo": {"full_name": REPO}}, + "body": automation.body({ + "kind": "release", "version": "0.3.1", "base": BASE, + "generated_head": HEAD, "event_id": report["event_id"], + }), + } + state = { + "report": report, "pr": pr, "saved": True, + "reads": [], "asset_reads": [], "uploads": [], "upload_files": [], + "temporary_directories": [], + } + + def api(repo, endpoint, method="GET", data=None): + state["reads"].append(endpoint) + assert repo == REPO and method == "GET" and data is None + if endpoint == "pulls/7": + return copy.deepcopy(pr) + assert endpoint == "releases?per_page=100&page=1" + return [{ + "tag_name": "v0.3.1", + "assets": [{"id": 98, "name": "release-receipt.json"}] if state["saved"] else [], + }] + + def gh_bytes(endpoint, *, accept): + state["asset_reads"].append(endpoint) + assert endpoint == f"repos/{REPO}/releases/assets/98" + assert accept == "application/octet-stream" + return receipts.canonical(report) + + def upload(command, *, check): + path = Path(command[4]) + if ( + path.name.endswith(".env") + or not path.resolve().is_relative_to(tmp_path.resolve()) + ): + raise AssertionError(f"refused retained receipt fixture file: {path}") + assert check is True + assert command == [ + "gh", "release", "upload", "v0.3.1", str(path), "--repo", REPO, + ] + state["upload_files"].append([path.name]) + state["uploads"].append(path.read_bytes()) + + real_temporary_directory = reporting.tempfile.TemporaryDirectory + + def temporary_directory(): + state["temporary_directories"].append(tmp_path) + return real_temporary_directory(dir=tmp_path) + + monkeypatch.setattr(automation, "api", api) + monkeypatch.setattr(reporting, "api", api) + monkeypatch.setattr(publishing, "gh_bytes", gh_bytes) + monkeypatch.setattr(reporting.subprocess, "run", upload) + monkeypatch.setattr(reporting.tempfile, "TemporaryDirectory", temporary_directory) + return state + + +def test_retain_refuses_missing_release_marker(retained_receipt_transport): + state = retained_receipt_transport + state["pr"]["body"] = None + report = state["report"] + before = copy.deepcopy(report) + assert automation.marker(state["pr"]) is None + with pytest.raises(ValueError, match=r"^retained release marker missing$"): + reporting.retain(REPO, report) + assert state["reads"] == ["pulls/7"] + assert state["asset_reads"] == state["uploads"] == state["upload_files"] == [] + assert state["temporary_directories"] == [] + assert report == before + + +@pytest.mark.parametrize("saved", [True, False]) +def test_retain_accepts_release_marker_and_preserves_receipt(retained_receipt_transport, saved): + state = retained_receipt_transport + state["saved"] = saved + report = state["report"] + before = copy.deepcopy(report) + reporting.retain(REPO, report) + assert state["reads"] == ["pulls/7", "releases?per_page=100&page=1"] + assert state["asset_reads"] == ([f"repos/{REPO}/releases/assets/98"] if saved else []) + assert state["upload_files"] == ([] if saved else [["release-receipt.json"]]) + assert state["uploads"] == ([] if saved else [receipts.canonical(report)]) + assert len(state["temporary_directories"]) == int(not saved) + assert report == before + + +@pytest.mark.parametrize("status", ["pending", "testing", "merged", "deploying", "failed"]) +def test_retain_other_states_do_not_require_a_marker(retained_receipt_transport, status): + state = retained_receipt_transport + state["pr"]["body"] = None + state["report"]["state"] = status + before = copy.deepcopy(state["report"]) + reporting.retain(REPO, state["report"]) + assert state["reads"] == state["asset_reads"] == state["uploads"] == state["upload_files"] == [] + assert state["temporary_directories"] == [] + assert state["report"] == before + + class ObservedGitHub: def __init__(self): self.writes = [] diff --git a/tests/test_main_proofs.py b/tests/test_main_proofs.py new file mode 100644 index 0000000..a6ef0df --- /dev/null +++ b/tests/test_main_proofs.py @@ -0,0 +1,892 @@ +"""Authored report fixtures; these controls do not execute CTest or live PyPI.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import subprocess +from pathlib import Path +from xml.etree import ElementTree + +import pytest +from release_chain import contract, evidence, main_proofs + +ROOT = Path(__file__).resolve().parents[1] +NETWORK_NODE = ( + "tests/test_consumer_receipt.py::test_bootstrap_recorded_hashes_match_live_pypi" +) +NATIVE_LEGS = ("release-amd64", "release-arm64", "release-macos", "asan-ubsan", "tsan") +NATIVE_WORKFLOW = ".github/workflows/native-feed-post.yml" +# Required subsets declared independently from the validators, from native/feed/CMakeLists.txt. +COMMON_NATIVE_ROWS = ("feed-unit", "feed-export-unit", "feed-mock-export", "feed-mock") +REQUIRED_NATIVE_ROWS = { + leg: COMMON_NATIVE_ROWS + (("feed-tsan-seeded-race",) if leg == "tsan" else ()) + for leg in NATIVE_LEGS +} +REQUIRED_NATIVE_OMISSIONS = [ + pytest.param(leg, row, id=f"{leg}-without-{row}") + for leg, rows in REQUIRED_NATIVE_ROWS.items() + for row in rows +] +NATIVE_JOB_NAMES = ( + "Release (ubuntu-24.04)", "Release (ubuntu-24.04-arm)", "Release (macos-15)", + "ASan + UBSan", "TSan", "Native proof", +) + + +def native_report_names(leg: str) -> list[str]: + # Distinct extra rows keep cross-leg report substitution observable. + return [*REQUIRED_NATIVE_ROWS[leg], f"{leg}-unit", f"{leg}-integration"] + + +def write_junit(path: Path, names: list[str], *, network: bool = False) -> bytes: + """Minimal pytest/CTest-shaped XML, with counters tied to actual fixture rows.""" + root = ElementTree.Element("testsuites", tests=str(len(names))) + suite = ElementTree.SubElement( + root, "testsuite", tests=str(len(names)), failures="0", errors="0", skipped="0" + ) + for name in names: + attributes = {"name": name} + if network: + attributes["classname"] = "tests.test_consumer_receipt" + ElementTree.SubElement(suite, "testcase", **attributes) + raw = ElementTree.tostring(root) + path.write_bytes(raw) + return raw + + +def write_registration(path: Path, names: list[str]) -> bytes: + raw = json.dumps( + {"kind": "ctestInfo", "version": {"major": 1, "minor": 0}, + "tests": [{"name": name} for name in names]}, + sort_keys=True, + ).encode() + path.write_bytes(raw) + return raw + + +def expected_inventory(names: list[str], raw: bytes) -> dict: + return { + "registered": names, "executed": list(names), + "skipped": [], "failures": [], "errors": [], + "junit_sha256": hashlib.sha256(raw).hexdigest(), + } + + +def expected_native_inventory(directory: Path, names: list[str]) -> dict: + return { + **expected_inventory(names, (directory / "native-junit.xml").read_bytes()), + "registration_sha256": hashlib.sha256( + (directory / "native-registration.json").read_bytes() + ).hexdigest(), + } + + +def write_native_report_pair(directory: Path, names: list[str]) -> dict: + registered_raw = write_registration(directory / "native-registration.json", names) + junit_raw = write_junit(directory / "native-junit.xml", names) + return { + **expected_inventory(names, junit_raw), + "registration_sha256": hashlib.sha256(registered_raw).hexdigest(), + } + + +def record_native_leg( + directory: Path, identity: dict, leg: str, inventory: dict +) -> None: + """Record authored report content, without asking a producer to accept it.""" + (directory / "native-leg.json").write_bytes(contract.json_bytes({ + **identity, "leg": leg, "inventory": inventory, + })) + + +@pytest.mark.parametrize("network", [False, True], ids=["ctest", "pytest-network"]) +@pytest.mark.parametrize("status", [None, "run", "passed"]) +def test_junit_inventory_uses_actual_rows_and_raw_digest(tmp_path, network, status): + names = ( + ["test_bootstrap_recorded_hashes_match_live_pypi"] if network + else ["native-unit-one", "native-unit-two"] + ) + path = tmp_path / "report.xml" + write_junit(path, names, network=network) + root = ElementTree.fromstring(path.read_bytes()) + if status is not None: + for case in root.iter("testcase"): + case.set("status", status) + raw = ElementTree.tostring(root) + path.write_bytes(raw) + actual_names = [NETWORK_NODE] if network else names + assert main_proofs.junit_inventory(path, network=network) == expected_inventory( + actual_names, raw + ) + # An input-dependent second report prevents a constant inventory/digest twin. + changed_names = names[:1] if not network else [names[0] + "_different"] + changed_raw = write_junit(path, changed_names, network=network) + changed = main_proofs.junit_inventory(path, network=network) + expected_names = [NETWORK_NODE + "_different"] if network else changed_names + assert changed == expected_inventory(expected_names, changed_raw) + assert changed["junit_sha256"] != hashlib.sha256(raw).hexdigest() + + +@pytest.mark.parametrize( + "fault,reason", + [ + ("empty", "missing nonempty inventory"), + ("empty-name", "missing nonempty inventory"), + ("duplicate", "duplicate inventory"), + ("skipped", "registration/execution disagree|required rows skipped"), + ("failure", "required rows failures"), + ("error", "required rows errors"), + ("notrun", "registration/execution disagree|required rows skipped"), + ("disabled", "registration/execution disagree|required rows skipped"), + ("unknown-status", "unsuccessful or unknown testcase status"), + ("omitted-row", "suite count disagrees"), + ("suite-overcount", "suite count disagrees"), + ("suite-undercount", "suite count disagrees"), + ("parent-overcount", "suite count disagrees"), + ("suite-skipped", "suite skipped"), + ("suite-failures", "suite failures"), + ("suite-errors", "suite errors"), + ("suite-disabled", "suite disabled"), + ("collection-error", "suite errors"), + ], +) +def test_junit_inventory_refuses_non_full_reports(tmp_path, fault, reason): + path = tmp_path / "report.xml" + names = [] if fault == "empty" else ["native-one", "native-two"] + write_junit(path, names) + root = ElementTree.fromstring(path.read_bytes()) + suite = root.find("testsuite") + cases = list(root.iter("testcase")) + if fault == "empty-name": + cases[0].set("name", "") + elif fault == "duplicate": + cases[1].set("name", cases[0].get("name")) + elif fault in {"skipped", "failure", "error"}: + ElementTree.SubElement(cases[0], fault, message="authored unsuccessful row") + elif fault in {"notrun", "disabled"}: + cases[0].set("status", fault) + elif fault == "unknown-status": + cases[0].set("status", "failed") + elif fault == "omitted-row": + suite.remove(cases[-1]) + elif fault in {"suite-overcount", "suite-undercount"}: + suite.set("tests", "3" if fault == "suite-overcount" else "1") + elif fault == "parent-overcount": + root.set("tests", "3") + elif fault.startswith("suite-"): + suite.set(fault.removeprefix("suite-"), "1") + elif fault == "collection-error": + suite.set("errors", "1") + ElementTree.SubElement(suite, "error", message="collection failed") + path.write_bytes(ElementTree.tostring(root)) + with pytest.raises(ValueError, match=reason): + main_proofs.junit_inventory(path) + + +@pytest.mark.parametrize("parent", ["testsuite", "testsuites"]) +@pytest.mark.parametrize("tag", ["error", "failure", "skipped"]) +def test_junit_inventory_refuses_suite_error_even_when_counter_claims_zero( + tmp_path, parent, tag +): + path = tmp_path / "collection.xml" + write_junit(path, ["collected-before-error"]) + root = ElementTree.fromstring(path.read_bytes()) + container = root if parent == "testsuites" else root.find("testsuite") + ElementTree.SubElement(container, tag, message="authored suite problem") + path.write_bytes(ElementTree.tostring(root)) + # A collection error is not made successful by an inconsistent zero counter. + with pytest.raises(ValueError) as rejected: + main_proofs.junit_inventory(path) + assert str(rejected.value) == f"{path}: suite contains {tag} element" + + +@pytest.mark.parametrize("parent", ["testsuite", "testsuites"]) +@pytest.mark.parametrize("content", ["system-out", "system-err", "text"]) +@pytest.mark.parametrize( + "names", [["unit-alpha"], ["unit-alpha", "unit-beta"]], ids=["one-row", "two-rows"] +) +def test_junit_inventory_accepts_harmless_suite_logs_and_text( + tmp_path, parent, content, names +): + path = tmp_path / "logs.xml" + original = write_junit(path, names) + root = ElementTree.fromstring(original) + container = root if parent == "testsuites" else root.find("testsuite") + message = "error/failure/skipped are harmless words in captured output" + if content == "text": + container.text = message + else: + ElementTree.SubElement(container, content).text = message + raw = ElementTree.tostring(root) + path.write_bytes(raw) + assert main_proofs.junit_inventory(path) == expected_inventory(names, raw) + assert raw != original + + +@pytest.mark.parametrize("raw", [b"", b"", b"not XML"]) +def test_junit_inventory_refuses_malformed_xml(tmp_path, raw): + path = tmp_path / "malformed.xml" + path.write_bytes(raw) + with pytest.raises(ElementTree.ParseError): + main_proofs.junit_inventory(path) + + +def test_junit_inventory_requires_a_report_and_junit_root(tmp_path): + # File absence and malformed syntax are I/O/parser refusals, not semantic results. + with pytest.raises(FileNotFoundError): + main_proofs.junit_inventory(tmp_path / "absent.xml") + path = tmp_path / "wrong-root.xml" + path.write_bytes(b'') + with pytest.raises(ValueError, match="not a JUnit report"): + main_proofs.junit_inventory(path) + + +@pytest.mark.parametrize("names", [["unit-alpha"], ["unit-alpha", "unit-beta"]]) +def test_native_inventory_consumes_registration_and_junit(tmp_path, names): + registration, junit = tmp_path / "registration.json", tmp_path / "junit.xml" + registered_raw = write_registration(registration, list(reversed(names))) + junit_raw = write_junit(junit, names) + expected = expected_inventory(names, junit_raw) + expected["registered"] = list(reversed(names)) + expected["registration_sha256"] = hashlib.sha256(registered_raw).hexdigest() + assert main_proofs.native_inventory(registration, junit) == expected + + +@pytest.mark.parametrize( + "fault,reason", + [ + ("empty-registration", "missing nonempty inventory"), + ("duplicate-registration", "duplicate inventory"), + ("omitted-registration", "registration/execution disagree"), + ("omitted-execution", "registration/execution disagree"), + ("extra-execution", "registration/execution disagree"), + ("empty-execution", "missing nonempty inventory"), + ], +) +def test_native_inventory_refuses_registration_execution_drift(tmp_path, fault, reason): + registered, executed = ["unit-alpha", "unit-beta"], ["unit-alpha", "unit-beta"] + if fault == "empty-registration": + registered = [] + elif fault == "duplicate-registration": + registered = ["unit-alpha", "unit-alpha"] + elif fault == "omitted-registration": + registered = registered[:1] + elif fault == "omitted-execution": + executed = executed[:1] + elif fault == "extra-execution": + executed.append("unregistered") + else: + executed = [] + registration, junit = tmp_path / "registration.json", tmp_path / "junit.xml" + write_registration(registration, registered) + write_junit(junit, executed) + with pytest.raises(ValueError, match=reason): + main_proofs.native_inventory(registration, junit) + + +@pytest.mark.parametrize( + "collected,reason", + [ + ({}, "CTest registration has no tests list"), + ({"tests": None}, "CTest registration has no tests list"), + ({"tests": {}}, "CTest registration has no tests list"), + ({"tests": [None]}, "CTest test: expected an object"), + ({"tests": [{}]}, "missing nonempty inventory"), + ], +) +def test_native_inventory_refuses_malformed_registration(tmp_path, collected, reason): + registration, junit = tmp_path / "registration.json", tmp_path / "junit.xml" + registration.write_text(json.dumps(collected)) + write_junit(junit, ["unit-alpha"]) + with pytest.raises(ValueError, match=reason): + main_proofs.native_inventory(registration, junit) + + +@pytest.mark.parametrize("missing", ["registration", "junit"]) +def test_native_inventory_requires_both_report_files(tmp_path, missing): + registration, junit = tmp_path / "registration.json", tmp_path / "junit.xml" + if missing != "registration": + write_registration(registration, ["unit-alpha"]) + if missing != "junit": + write_junit(junit, ["unit-alpha"]) + absent = registration if missing == "registration" else junit + with pytest.raises(FileNotFoundError) as rejected: + main_proofs.native_inventory(registration, junit) + assert rejected.value.filename == str(absent) + + +@pytest.fixture +def producer_repository(tmp_path, monkeypatch): + """A normal local test repository; no hook/config/signing overrides.""" + root = tmp_path / "repository" + workflows = root / ".github/workflows" + workflows.mkdir(parents=True) + for name in ("ci-post.yml", "native-feed-post.yml"): + (workflows / name).write_bytes((ROOT / ".github/workflows" / name).read_bytes()) + commands = [ + ["git", "init", "-b", "main"], + ["git", "add", ".github/workflows/ci-post.yml", NATIVE_WORKFLOW], + ["git", "-c", "user.name=Producer control", + "-c", "user.email=producer-control@example.invalid", + "commit", "-m", "Unit producer identity"], + ] + for command in commands: + subprocess.run(command, cwd=root, check=True, capture_output=True) + head = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=root, text=True + ).strip() + tree = subprocess.check_output( + ["git", "rev-parse", "HEAD^{tree}"], cwd=root, text=True + ).strip() + for name, value in { + "GITHUB_EVENT_NAME": "push", "GITHUB_REF": "refs/heads/main", + "GITHUB_REPOSITORY": "owner/data", + "GITHUB_WORKFLOW_REF": "owner/data/.github/workflows/ci-post.yml@refs/heads/main", + "GITHUB_SHA": head, "GITHUB_RUN_ID": "731", "GITHUB_RUN_ATTEMPT": "3", + }.items(): + monkeypatch.setenv(name, value) + return root, { + "run_id": 731, "attempt": 3, "head": head, "tested_sha": head, "tree": tree, + } + + +@pytest.fixture +def network_reports(producer_repository, tmp_path): + root, identity = producer_repository + directory = tmp_path / "network-reports" + directory.mkdir() + raw = write_junit( + directory / "network.xml", + ["test_bootstrap_recorded_hashes_match_live_pypi"], + network=True, + ) + proof = main_proofs.network_proof(root, directory) + (directory / "network-proof.json").write_bytes(contract.json_bytes(proof)) + return root, directory, identity, raw + + +def test_network_producer_and_consumer_bind_actual_report(network_reports): + root, directory, identity, raw = network_reports + expected = { + **identity, "node_id": NETWORK_NODE, + **expected_inventory([NETWORK_NODE], raw), + } + assert main_proofs.network_proof(root, directory) == expected + assert main_proofs.consume_network(root, directory) == expected + + +@pytest.mark.parametrize( + "fault,reason", + [ + ("missing-classname", "network JUnit has no source classname"), + ("wrong-classname", "wrong live-PyPI inventory"), + ("wrong-node", "wrong live-PyPI inventory"), + ("duplicate-node", "duplicate inventory"), + ("omitted-node", "missing nonempty inventory"), + ("skipped-node", "executed: missing nonempty inventory"), + ("failed-node", "required rows failures"), + ("errored-node", "required rows errors"), + ], +) +def test_network_proof_refuses_wrong_source_or_non_full_row(network_reports, fault, reason): + root, directory, _identity, _raw = network_reports + path = directory / "network.xml" + xml = ElementTree.fromstring(path.read_bytes()) + suite, case = xml.find("testsuite"), xml.find(".//testcase") + if fault == "missing-classname": + case.attrib.pop("classname") + elif fault == "wrong-classname": + case.set("classname", "tests.test_release_adapter") + elif fault == "wrong-node": + case.set("name", "test_an_unrelated_network_request") + elif fault == "duplicate-node": + suite.append(copy.deepcopy(case)) + suite.set("tests", "2") + xml.set("tests", "2") + elif fault == "omitted-node": + suite.remove(case) + suite.set("tests", "0") + xml.set("tests", "0") + else: + tag = {"skipped-node": "skipped", "failed-node": "failure", "errored-node": "error"}[fault] + ElementTree.SubElement(case, tag, message="authored network refusal") + path.write_bytes(ElementTree.tostring(xml)) + with pytest.raises(ValueError, match=reason): + main_proofs.network_proof(root, directory) + + +@pytest.mark.parametrize( + "name,value,reason", + [ + ("GITHUB_EVENT_NAME", "pull_request", "required push/main workflow"), + ("GITHUB_REF", "refs/heads/topic", "required push/main workflow"), + ("GITHUB_REPOSITORY", "other/data", "required push/main workflow"), + ("GITHUB_WORKFLOW_REF", "owner/data/.github/workflows/ci.yml@refs/heads/main", + "required push/main workflow"), + ("GITHUB_SHA", "e" * 40, "checkout differs from event head"), + ], +) +def test_network_proof_refuses_wrong_producer_source( + network_reports, monkeypatch, name, value, reason +): + root, directory, _identity, _raw = network_reports + monkeypatch.setenv(name, value) + with pytest.raises(ValueError, match=reason): + main_proofs.network_proof(root, directory) + + +@pytest.mark.parametrize( + "field", ["run_id", "attempt", "head", "tested_sha", "tree", "junit_sha256", "node_id"] +) +def test_consume_network_refuses_recorded_identity_or_digest_tampering(network_reports, field): + root, directory, _identity, _raw = network_reports + path = directory / "network-proof.json" + proof = contract.object_json(path.read_bytes()) + original = proof[field] + proof[field] = original + 1 if type(original) is int else ( + hashlib.sha256(f"tampered:{field}".encode()).hexdigest()[:len(original)] + ) + path.write_bytes(contract.json_bytes(proof)) + with pytest.raises(ValueError, match="network report bytes or producer identity changed"): + main_proofs.consume_network(root, directory) + + +def test_consume_network_refuses_changed_raw_bytes_with_unchanged_rows(network_reports): + root, directory, _identity, _raw = network_reports + path = directory / "network.xml" + path.write_bytes(path.read_bytes() + b"\n") + with pytest.raises(ValueError, match="network report bytes or producer identity changed"): + main_proofs.consume_network(root, directory) + + +@pytest.mark.parametrize("field", ["registered", "executed", "skipped", "failures", "errors"]) +def test_consume_network_refuses_recorded_inventory_tampering(network_reports, field): + root, directory, _identity, _raw = network_reports + path = directory / "network-proof.json" + proof = contract.object_json(path.read_bytes()) + proof[field] = [] if field in {"registered", "executed"} else [NETWORK_NODE] + path.write_bytes(contract.json_bytes(proof)) + with pytest.raises(ValueError, match="network report bytes or producer identity changed"): + main_proofs.consume_network(root, directory) + + +@pytest.mark.parametrize("missing", ["network.xml", "network-proof.json"]) +def test_consume_network_requires_both_report_files(network_reports, tmp_path, missing): + root, directory, _identity, _raw = network_reports + incomplete = tmp_path / "incomplete-network" + incomplete.mkdir() + for name in ("network.xml", "network-proof.json"): + if name != missing: + (incomplete / name).write_bytes((directory / name).read_bytes()) + with pytest.raises(FileNotFoundError) as rejected: + main_proofs.consume_network(root, incomplete) + assert rejected.value.filename == str(incomplete / missing) + + +@pytest.mark.parametrize("name", ["GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT"]) +def test_consume_network_refuses_previous_run_or_attempt(network_reports, monkeypatch, name): + root, directory, identity, _raw = network_reports + previous = identity["run_id" if name == "GITHUB_RUN_ID" else "attempt"] + monkeypatch.setenv(name, str(previous + 1)) + with pytest.raises(ValueError, match="network report bytes or producer identity changed"): + main_proofs.consume_network(root, directory) + + +@pytest.fixture +def native_reports(producer_repository, tmp_path, monkeypatch): + root, identity = producer_repository + monkeypatch.setenv( + "GITHUB_WORKFLOW_REF", f"owner/data/{NATIVE_WORKFLOW}@refs/heads/main" + ) + directory = tmp_path / "native-reports" + directory.mkdir() + for leg in NATIVE_LEGS: + reports = directory / leg + reports.mkdir() + # Different reports in each leg make accidental report substitution observable. + names = native_report_names(leg) + write_registration(reports / "native-registration.json", names) + write_junit(reports / "native-junit.xml", names) + (reports / "native-leg.json").write_bytes( + contract.json_bytes(main_proofs.native_leg(root, leg, reports)) + ) + needs = {name: {"result": "success"} for name in ("authz", "main-ref", *NATIVE_LEGS)} + return root, directory, identity, needs + + +def test_native_leg_and_aggregate_consume_real_files_and_authoritative_needs( + native_reports, monkeypatch +): + root, directory, identity, needs = native_reports + api_calls = [] + + def reject_job_api(*args, **kwargs): + api_calls.append((args, kwargs)) + raise AssertionError("a running native aggregate must not query its own job") + + monkeypatch.setattr(contract, "gh_bytes", reject_job_api) + monkeypatch.setattr(evidence, "gh_bytes", reject_job_api) + git_reads = [] + real_check_output = subprocess.check_output + + def observe_identity_read(command, **kwargs): + git_reads.append(command) + assert command in ( + ["git", "rev-parse", "HEAD"], ["git", "rev-parse", "HEAD^{tree}"] + ), "native aggregation may read local identity, never query a completed job" + assert kwargs == {"cwd": root, "text": True} + return real_check_output(command, **kwargs) + + monkeypatch.setattr(main_proofs.subprocess, "check_output", observe_identity_read) + result = main_proofs.native_aggregate(root, directory, needs) + expected_legs = {} + for leg in NATIVE_LEGS: + reports = directory / leg + inventory = expected_native_inventory(reports, native_report_names(leg)) + assert main_proofs.native_leg(root, leg, reports) == { + **identity, "leg": leg, "inventory": inventory, + } + expected_legs[leg] = { + **inventory, "artifact": f"native-leg-{leg}-{identity['attempt']}", + } + assert result == {**identity, "legs": expected_legs} + assert api_calls == [] + assert git_reads + assert "Native proof" not in needs + + +@pytest.mark.parametrize("leg", NATIVE_LEGS) +def test_native_required_subset_accepts_extra_rows_and_binds_their_bytes(native_reports, leg): + root, directory, identity, needs = native_reports + reports = directory / leg + required = list(REQUIRED_NATIVE_ROWS[leg]) + minimum = write_native_report_pair(reports, required) + record_native_leg(reports, identity, leg, minimum) + assert main_proofs.native_leg(root, leg, reports) == { + **identity, "leg": leg, "inventory": minimum, + } + before = main_proofs.native_aggregate(root, directory, needs) + artifact = f"native-leg-{leg}-{identity['attempt']}" + assert before["legs"][leg] == {**minimum, "artifact": artifact} + + # Represent CTest's WILL_FAIL success with a normal, unskipped fixture row. + if leg == "tsan": + assert "feed-tsan-seeded-race" in minimum["executed"] + else: + assert "feed-tsan-seeded-race" not in minimum["registered"] + extended = write_native_report_pair(reports, native_report_names(leg)) + record_native_leg(reports, identity, leg, extended) + assert main_proofs.native_leg(root, leg, reports) == { + **identity, "leg": leg, "inventory": extended, + } + after = main_proofs.native_aggregate(root, directory, needs) + assert after == { + **before, "legs": {**before["legs"], leg: {**extended, "artifact": artifact}}, + } + assert extended["executed"] != minimum["executed"] + for field in ("registration_sha256", "junit_sha256"): + assert extended[field] != minimum[field] + + +@pytest.mark.parametrize("leg,missing", REQUIRED_NATIVE_OMISSIONS) +@pytest.mark.parametrize("producer", ["native-leg", "native-aggregate"]) +def test_native_producers_refuse_mutually_reduced_required_coverage( + native_reports, leg, missing, producer +): + root, directory, identity, needs = native_reports + reports = directory / leg + complete = expected_native_inventory(reports, native_report_names(leg)) + assert main_proofs.native_leg(root, leg, reports) == { + **identity, "leg": leg, "inventory": complete, + } + positive = main_proofs.native_aggregate(root, directory, needs) + assert positive["legs"][leg] == { + **complete, "artifact": f"native-leg-{leg}-{identity['attempt']}", + } + + names = [name for name in native_report_names(leg) if name != missing] + reduced = write_native_report_pair(reports, names) + record_native_leg(reports, identity, leg, reduced) + # Both inventories agree, counters are honest, and the recorded hashes are current. + assert names and missing not in names + assert main_proofs.native_inventory( + reports / "native-registration.json", reports / "native-junit.xml" + ) == reduced + for field in ("registration_sha256", "junit_sha256"): + assert reduced[field] != complete[field] + with pytest.raises(ValueError) as rejected: + if producer == "native-leg": + main_proofs.native_leg(root, leg, reports) + else: + main_proofs.native_aggregate(root, directory, needs) + assert str(rejected.value) == f"native {leg}: missing required native rows: {missing}" + + +@pytest.fixture +def complete_main_reports(native_reports, tmp_path, monkeypatch): + """Real local producers supply both proofs; job/run observations are offline inputs.""" + root, directory, identity, needs = native_reports + network_directory = tmp_path / "main-network" + network_directory.mkdir() + network_raw = write_junit( + network_directory / "network.xml", + ["test_bootstrap_recorded_hashes_match_live_pypi"], + network=True, + ) + python_identity = { + **identity, "run_id": identity["run_id"] + 10, "attempt": identity["attempt"] + 1, + } + with monkeypatch.context() as network_environment: + network_environment.setenv( + "GITHUB_WORKFLOW_REF", "owner/data/.github/workflows/ci-post.yml@refs/heads/main" + ) + network_environment.setenv("GITHUB_RUN_ID", str(python_identity["run_id"])) + network_environment.setenv("GITHUB_RUN_ATTEMPT", str(python_identity["attempt"])) + network = main_proofs.network_proof(root, network_directory) + expected_network = { + **python_identity, "node_id": NETWORK_NODE, + **expected_inventory([NETWORK_NODE], network_raw), + } + assert network == expected_network + (network_directory / "network-proof.json").write_bytes(contract.json_bytes(network)) + assert main_proofs.consume_network(root, network_directory) == expected_network + + native = main_proofs.native_aggregate(root, directory, needs) + expected_legs = { + leg: { + **expected_native_inventory(directory / leg, native_report_names(leg)), + "artifact": f"native-leg-{leg}-{identity['attempt']}", + } + for leg in NATIVE_LEGS + } + assert native == {**identity, "legs": expected_legs} + run = { + "id": python_identity["run_id"], "run_attempt": python_identity["attempt"], + "head_sha": identity["head"], "head_branch": "main", "event": "push", + "path": ".github/workflows/ci-post.yml", + "status": "completed", "conclusion": "success", + } + native_run = { + **run, "id": identity["run_id"], "run_attempt": identity["attempt"], + "path": NATIVE_WORKFLOW, + } + + def observed_jobs(names, observation): + return [ + { + "name": name, "status": "completed", "conclusion": "success", + "run_id": observation["id"], "run_attempt": observation["run_attempt"], + "head_sha": observation["head_sha"], + } + for name in names + ] + + jobs = observed_jobs((*evidence.JOBS, "release/ready", "Live PyPI hashes"), run) + proof = { + **python_identity, "event": "push", "base": None, "pr": None, + "jobs": dict.fromkeys(evidence.JOBS, "success"), + "e2e": {suite: sorted(rows) for suite, rows in evidence.TESTS.items()}, + "main_proofs": { + "network": network, + "native": { + "workflow_path": NATIVE_WORKFLOW, "run": native_run, + "jobs": observed_jobs(NATIVE_JOB_NAMES, native_run), "attestation": native, + }, + }, + } + return root, directory, identity, proof, run, jobs + + +def test_validate_main_accepts_complete_local_proofs_and_observed_jobs(complete_main_reports): + _root, _directory, _identity, proof, run, jobs = complete_main_reports + main_proofs.validate_main(proof, run, jobs) + evidence.validate_gate(proof, run, jobs, head=proof["head"], base=None, pr=None) + + +@pytest.mark.parametrize("leg,missing", REQUIRED_NATIVE_OMISSIONS) +def test_validate_main_refuses_mutually_reduced_required_coverage( + complete_main_reports, leg, missing +): + _root, directory, identity, proof, run, jobs = complete_main_reports + # Validate the complete twin before changing any report or proof content. + main_proofs.validate_main(proof, run, jobs) + evidence.validate_gate(proof, run, jobs, head=proof["head"], base=None, pr=None) + native = proof["main_proofs"]["native"]["attestation"] + before = copy.deepcopy(native["legs"][leg]) + reports = directory / leg + names = [name for name in native_report_names(leg) if name != missing] + reduced = write_native_report_pair(reports, names) + record_native_leg(reports, identity, leg, reduced) + assert names and missing not in names + assert main_proofs.native_inventory( + reports / "native-registration.json", reports / "native-junit.xml" + ) == reduced + native["legs"][leg] = {**reduced, "artifact": before["artifact"]} + for field in ("registration_sha256", "junit_sha256"): + assert native["legs"][leg][field] != before[field] + # The consumer sees an internally consistent reduced proof, not stale evidence. + with pytest.raises(ValueError) as rejected: + main_proofs.validate_main(proof, run, jobs) + assert str(rejected.value) == f"native {leg}: missing required native rows: {missing}" + + +@pytest.mark.parametrize("dependency", ["authz", "main-ref", *NATIVE_LEGS]) +@pytest.mark.parametrize("result", ["missing", "failure", "cancelled", "skipped", "in_progress"]) +def test_native_aggregate_refuses_missing_or_unsuccessful_needs(native_reports, dependency, result): + root, directory, _identity, needs = native_reports + if result == "missing": + needs.pop(dependency) + reason = "native needs inventory changed" + else: + needs[dependency]["result"] = result + reason = "native dependency failed, cancelled or skipped" + with pytest.raises(ValueError, match=reason): + main_proofs.native_aggregate(root, directory, needs) + + +def test_native_aggregate_refuses_unexpected_dependency(native_reports): + root, directory, _identity, needs = native_reports + needs["unreviewed-leg"] = {"result": "success"} + with pytest.raises(ValueError, match="native needs inventory changed"): + main_proofs.native_aggregate(root, directory, needs) + + +@pytest.mark.parametrize("leg", NATIVE_LEGS) +@pytest.mark.parametrize( + "field", ["run_id", "attempt", "head", "tested_sha", "tree", "leg", + "junit_sha256", "registration_sha256"] +) +def test_native_aggregate_refuses_one_stale_leg_identity_or_digest(native_reports, leg, field): + root, directory, _identity, needs = native_reports + path = directory / leg / "native-leg.json" + row = contract.object_json(path.read_bytes()) + target = row["inventory"] if field.endswith("sha256") else row + old = target[field] + target[field] = old + 1 if type(old) is int else ( + "other-leg" if field == "leg" + else hashlib.sha256(f"tampered:{field}".encode()).hexdigest()[:len(old)] + ) + path.write_bytes(contract.json_bytes(row)) + with pytest.raises(ValueError, match=f"{leg}: report bytes or producer identity changed"): + main_proofs.native_aggregate(root, directory, needs) + + +@pytest.mark.parametrize("leg", NATIVE_LEGS) +@pytest.mark.parametrize("field", ["registered", "executed", "skipped", "failures", "errors"]) +def test_native_aggregate_refuses_recorded_inventory_tampering(native_reports, leg, field): + root, directory, _identity, needs = native_reports + path = directory / leg / "native-leg.json" + row = contract.object_json(path.read_bytes()) + row["inventory"][field] = ( + [] if field in {"registered", "executed"} else [f"{leg}-unit"] + ) + path.write_bytes(contract.json_bytes(row)) + with pytest.raises(ValueError, match=f"{leg}: report bytes or producer identity changed"): + main_proofs.native_aggregate(root, directory, needs) + + +@pytest.mark.parametrize("leg", NATIVE_LEGS) +@pytest.mark.parametrize( + "missing", ["leg-directory", "native-leg.json", "native-registration.json", "native-junit.xml"] +) +def test_native_aggregate_requires_every_current_leg_report(native_reports, tmp_path, leg, missing): + root, directory, _identity, needs = native_reports + incomplete = tmp_path / "incomplete-native" + incomplete.mkdir() + for selected in NATIVE_LEGS: + if selected == leg and missing == "leg-directory": + continue + reports = incomplete / selected + reports.mkdir() + for name in ("native-leg.json", "native-registration.json", "native-junit.xml"): + if selected != leg or name != missing: + (reports / name).write_bytes((directory / selected / name).read_bytes()) + absent = incomplete / leg / ("native-leg.json" if missing == "leg-directory" else missing) + with pytest.raises(FileNotFoundError) as rejected: + main_proofs.native_aggregate(root, incomplete, needs) + assert rejected.value.filename == str(absent) + + +@pytest.mark.parametrize("leg", NATIVE_LEGS) +@pytest.mark.parametrize("filename", ["native-registration.json", "native-junit.xml"]) +def test_native_aggregate_reparses_raw_report_bytes(native_reports, leg, filename): + root, directory, _identity, needs = native_reports + path = directory / leg / filename + path.write_bytes(path.read_bytes() + b"\n") + with pytest.raises(ValueError, match=f"{leg}: report bytes or producer identity changed"): + main_proofs.native_aggregate(root, directory, needs) + + +@pytest.mark.parametrize("leg", NATIVE_LEGS) +@pytest.mark.parametrize( + "fault,reason", + [ + ("empty", "missing nonempty inventory"), + ("omitted", "registration/execution disagree"), + ("duplicate", "duplicate inventory"), + ("skipped", "registration/execution disagree|required rows skipped"), + ("failure", "required rows failures"), + ("error", "required rows errors"), + ], +) +def test_native_aggregate_refuses_non_full_leg_reports(native_reports, leg, fault, reason): + root, directory, _identity, needs = native_reports + reports = directory / leg + names = native_report_names(leg) + if fault == "empty": + names = [] + elif fault == "omitted": + names = names[:1] + elif fault == "duplicate": + names = [names[0], names[0]] + path = reports / "native-junit.xml" + write_junit(path, names) + if fault in {"skipped", "failure", "error"}: + xml = ElementTree.fromstring(path.read_bytes()) + ElementTree.SubElement(xml.find(".//testcase"), fault) + path.write_bytes(ElementTree.tostring(xml)) + # Even a digest refreshed to match the bad bytes cannot bypass report parsing. + saved = reports / "native-leg.json" + recorded = contract.object_json(saved.read_bytes()) + recorded["inventory"]["junit_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + saved.write_bytes(contract.json_bytes(recorded)) + with pytest.raises(ValueError, match=reason): + main_proofs.native_aggregate(root, directory, needs) + + +@pytest.mark.parametrize("name", ["GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT"]) +def test_native_aggregate_refuses_previous_run_or_attempt(native_reports, monkeypatch, name): + root, directory, identity, needs = native_reports + previous = identity["run_id" if name == "GITHUB_RUN_ID" else "attempt"] + monkeypatch.setenv(name, str(previous + 1)) + with pytest.raises(ValueError, match="report bytes or producer identity changed"): + main_proofs.native_aggregate(root, directory, needs) + + +@pytest.mark.parametrize( + "name,value,reason", + [ + ("GITHUB_EVENT_NAME", "workflow_dispatch", "required push/main workflow"), + ("GITHUB_REF", "refs/heads/topic", "required push/main workflow"), + ("GITHUB_WORKFLOW_REF", "owner/data/.github/workflows/ci-post.yml@refs/heads/main", + "required push/main workflow"), + ("GITHUB_SHA", "e" * 40, "checkout differs from event head"), + ], +) +def test_native_leg_refuses_wrong_producer_source(native_reports, monkeypatch, name, value, reason): + root, directory, _identity, _needs = native_reports + monkeypatch.setenv(name, value) + with pytest.raises(ValueError, match=reason): + main_proofs.native_leg(root, "tsan", directory / "tsan") + + +def test_native_producers_refuse_changed_workflow_bytes(native_reports): + root, directory, _identity, needs = native_reports + path = root / NATIVE_WORKFLOW + path.write_bytes(path.read_bytes() + b"\n") + with pytest.raises(ValueError, match="authoritative needs require independent workflow review"): + main_proofs.native_leg(root, "tsan", directory / "tsan") + with pytest.raises(ValueError, match="authoritative needs require independent workflow review"): + main_proofs.native_aggregate(root, directory, needs) diff --git a/tests/test_post_merge_workflows.py b/tests/test_post_merge_workflows.py index 2e02201..35042a7 100644 --- a/tests/test_post_merge_workflows.py +++ b/tests/test_post_merge_workflows.py @@ -5,7 +5,10 @@ import copy import json import os +import re +import shlex import subprocess +from fnmatch import fnmatchcase from pathlib import Path from typing import Any @@ -16,6 +19,9 @@ ROOT = Path(__file__).resolve().parents[1] WORKFLOWS = ROOT / ".github/workflows" PAIRS = ("ci", "native-feed") +NETWORK_NODE = ( + "tests/test_consumer_receipt.py::test_bootstrap_recorded_hashes_match_live_pypi" +) ROUTES = { "workflow-lint": "ubuntu-24.04", "test": "ubuntu-24.04", @@ -62,6 +68,196 @@ def assert_post_events(workflow: dict[str, Any]) -> None: assert workflow["concurrency"]["cancel-in-progress"] == "false" +def workflow_uses(value: Any, location: str = "$") -> list[tuple[str, Any]]: + """Observe step and reusable-job references through every mapping/list.""" + references = [] + if isinstance(value, dict): + for key, child in value.items(): + path = f"{location}.{key}" + if key == "uses": + references.append((path, child)) + references.extend(workflow_uses(child, path)) + elif isinstance(value, list): + for index, child in enumerate(value): + references.extend(workflow_uses(child, f"{location}[{index}]")) + return references + + +def assert_action_pins(workflow: dict[str, Any]) -> None: + references = workflow_uses(workflow) + assert references, "post workflow has no uses references" + offending = [ + (path, reference) for path, reference in references + if not isinstance(reference, str) + or re.fullmatch(r"[^@\s]+@[0-9a-fA-F]{40}", reference) is None + ] + assert not offending, ( + "uses references require an @ suffix of exactly 40 hexadecimal characters: " + + "; ".join(f"{path}={reference!r}" for path, reference in offending) + ) + + +@pytest.mark.parametrize("stem", PAIRS) +def test_post_proof_workflows_pin_every_actual_uses_reference(stem: str) -> None: + path = WORKFLOWS / f"{stem}-post.yml" + workflow = yaml.load(path.read_text(), Loader=yaml.BaseLoader) + assert_action_pins(workflow) + + +@pytest.fixture(params=PAIRS) +def pinned_uses_control(request) -> dict[str, Any]: + path = WORKFLOWS / f"{request.param}-post.yml" + workflow = yaml.load(path.read_text(), Loader=yaml.BaseLoader) + observed = workflow_uses(workflow) + assert observed, f"{path.name}: missing actual uses inventory for the control" + pinned = [ + reference for _path, reference in observed + if isinstance(reference, str) + and re.fullmatch(r"[^@\s]+@[0-9a-fA-F]{40}", reference) + ] + assert pinned, f"{path.name}: no existing pinned reference for the positive control" + # Fixture-only mappings exercise traversal; they are not runnable workflows. + return { + "jobs": { + "reusable-control": {"uses": pinned[0]}, + "step-control": {"steps": [{"uses": reference} for reference in pinned]}, + }, + "nested-control": [{"mapping": {"uses": pinned[-1]}}], + } + + +def test_action_pin_control_accepts_references_from_the_actual_files(pinned_uses_control) -> None: + assert_action_pins(pinned_uses_control) + + +@pytest.mark.parametrize("location", ["step", "reusable-job", "nested"]) +@pytest.mark.parametrize("fault", ["floating-tag", "short-sha", "long-sha", "nonhex-sha"]) +def test_action_pin_control_refuses_a_changed_reference( + pinned_uses_control, location: str, fault: str +) -> None: + assert_action_pins(pinned_uses_control) + changed = copy.deepcopy(pinned_uses_control) + owners = { + "step": ("$.jobs.step-control.steps[0].uses", + changed["jobs"]["step-control"]["steps"][0]), + "reusable-job": ("$.jobs.reusable-control.uses", + changed["jobs"]["reusable-control"]), + "nested": ("$.nested-control[0].mapping.uses", + changed["nested-control"][0]["mapping"]), + } + path, owner = owners[location] + action, revision = owner["uses"].rsplit("@", 1) + suffix = { + "floating-tag": "v6", "short-sha": revision[:7], + "long-sha": revision + "0", "nonhex-sha": "g" + revision[1:], + }[fault] + owner["uses"] = offending = f"{action}@{suffix}" + with pytest.raises(AssertionError) as rejected: + assert_action_pins(changed) + message = str(rejected.value) + assert "uses references require an @ suffix of exactly 40 hexadecimal characters" in message + assert f"{path}={offending!r}" in message + + +def test_action_pin_control_refuses_an_empty_reference_inventory() -> None: + with pytest.raises(AssertionError) as rejected: + assert_action_pins({"jobs": {"run-only": {"steps": [{"run": "true"}]}}}) + assert str(rejected.value).splitlines()[0] == "post workflow has no uses references" + + +def native_ctest_step(workflow: dict[str, Any], job_id: str) -> dict[str, Any]: + steps = [ + step for step in workflow["jobs"][job_id]["steps"] + if re.search(r"(?m)^[ \t]*ctest(?:[ \t]|$)", step.get("run", "")) + ] + assert len(steps) == 1, f"{job_id}: expected one actual CTest-producing step" + return steps[0] + + +def assert_native_ctest_serial(workflow: dict[str, Any], job_id: str) -> None: + step = native_ctest_step(workflow, job_id) + commands = [ + shlex.split(line, comments=True) + for line in step["run"].replace("\\\n", " ").splitlines() + ] + ctest = [command for command in commands if command and command[0] == "ctest"] + assert len(ctest) == 2, f"{job_id}: require registration and execution commands" + for command in ctest: + parallel = [ + token for token in command[1:] + if token.startswith("-j") or token == "--parallel" or token.startswith("--parallel=") + ] + assert not parallel, ( + f"{job_id}: CTest must remain serial; parallel request {parallel!r} " + f"in {shlex.join(command)!r}" + ) + assert ctest[0] == [ + "ctest", "--test-dir", "build-feed", "--show-only=json-v1", + ">", "native-registration.json", + ], f"{job_id}: registration command changed" + assert ctest[1] == [ + "ctest", "--test-dir", "build-feed", "--output-on-failure", + "--output-junit", "$GITHUB_WORKSPACE/native-junit.xml", + ], f"{job_id}: JUnit execution command changed" + assert ["cmake", "--build", "build-feed", "-j4"] in commands, ( + f"{job_id}: preserve existing build parallelism" + ) + + +@pytest.mark.parametrize("job_id", NATIVE_ROUTES) +def test_native_post_ctest_execution_remains_serial(job_id: str) -> None: + path = WORKFLOWS / "native-feed-post.yml" + workflow = yaml.load(path.read_text(), Loader=yaml.BaseLoader) + assert_native_ctest_serial(workflow, job_id) + + +@pytest.fixture(params=NATIVE_ROUTES) +def serial_ctest_control(request) -> tuple[dict[str, Any], str]: + path = WORKFLOWS / "native-feed-post.yml" + workflow = yaml.load(path.read_text(), Loader=yaml.BaseLoader) + job_id = request.param + step = native_ctest_step(workflow, job_id) + # Fixture-only serial copy of the actual step, never a workflow-file repair. + step["run"] = re.sub( + r"(?m)^([ \t]*ctest[^\n]*?) -j4[ \t]*$", r"\1", step["run"] + ) + return workflow, job_id + + +def test_native_ctest_control_accepts_serial_tests_with_parallel_builds( + serial_ctest_control, +) -> None: + workflow, job_id = serial_ctest_control + assert_native_ctest_serial(workflow, job_id) + + +@pytest.mark.parametrize("command", ["registration", "execution"]) +@pytest.mark.parametrize("parallel", ["-j4", "-j 4", "--parallel 4", "--parallel=4"]) +def test_native_ctest_control_refuses_inserted_parallelism( + serial_ctest_control, command: str, parallel: str +) -> None: + workflow, job_id = serial_ctest_control + assert_native_ctest_serial(workflow, job_id) + changed = copy.deepcopy(workflow) + step = native_ctest_step(changed, job_id) + lines = step["run"].splitlines(keepends=True) + targets = [ + index for index, line in enumerate(lines) + if line.lstrip().startswith("ctest ") + and ("--show-only=json-v1" in line) == (command == "registration") + ] + assert len(targets) == 1 + index = targets[0] + # Put the option on the CTest invocation, before any registration redirection. + lines[index] = lines[index].replace("ctest ", f"ctest {parallel} ", 1) + step["run"] = "".join(lines) + with pytest.raises(AssertionError) as rejected: + assert_native_ctest_serial(changed, job_id) + message = str(rejected.value) + assert f"{job_id}: CTest must remain serial; parallel request" in message + assert parallel in message + + @pytest.mark.parametrize("stem", PAIRS) def test_pr_workflows_have_only_the_pull_request_trigger(stem: str) -> None: assert set(load(stem)["on"]) == {"pull_request"} @@ -83,27 +279,117 @@ def test_post_event_control_rejects_every_pr_event(stem: str, event: str) -> Non assert_post_events(changed) -def test_native_paths_cover_both_workflow_definitions() -> None: +def test_native_pr_paths_cover_both_workflow_definitions() -> None: paths = [ "native/feed/**", ".github/workflows/native-feed.yml", ".github/workflows/native-feed-post.yml", ] assert load("native-feed")["on"]["pull_request"]["paths"] == paths - assert load("native-feed", post=True)["on"]["push"]["paths"] == paths + + +@pytest.mark.parametrize( + "changed_path", + [ + "scripts/release_chain/evidence.py", + "tests/test_consumer_receipt.py", + ".github/workflows/ci-post.yml", + ], +) +def test_main_native_proof_runs_for_changes_outside_native_paths(changed_path: str) -> None: + workflow = load("native-feed", post=True) + assert_post_events(workflow) + assert (ROOT / changed_path).is_file() + assert not changed_path.startswith("native/feed/") + push = workflow["on"]["push"] + paths = push.get("paths") + assert paths is None or any(fnmatchcase(changed_path, pattern) for pattern in paths), ( + f"the actual main push path list omits a merge changing only {changed_path}" + ) + assert not any(fnmatchcase(changed_path, pattern) for pattern in push.get("paths-ignore", [])) + assert not {"paths", "paths-ignore"} & set(push), ( + f"a main merge changing only {changed_path} must still schedule all native legs" + ) + assert_native_post(workflow) + + +def assert_main_network_job(workflow: dict[str, Any]) -> str: + jobs = workflow["jobs"] + selected = [ + (job_id, job, step) + for job_id, job in jobs.items() + for step in job.get("steps", []) + if NETWORK_NODE in step.get("run", "") + ] + assert len(selected) == 1, "main CI is missing its single dedicated live-PyPI row" + job_id, job, step = selected[0] + assert job_id != "test" and "matrix" not in job.get("strategy", {}) + assert "if" not in job and "continue-on-error" not in job + assert needs(job) == {"authz", "main-ref"} + assert step.get("if") in { + None, "needs.authz.result == 'success'", "${{ needs.authz.result == 'success' }}" + } + assert "continue-on-error" not in step + tokens = shlex.split(step["run"]) + assert NETWORK_NODE in tokens + assert "pytest" in tokens + assert [token for token in tokens if token.startswith("tests/")] == [NETWORK_NODE] + assert not {"--collect-only", "--co"} & set(tokens) + enabled = { + **workflow.get("env", {}), **job.get("env", {}), **step.get("env", {}) + }.get("PINEFORGE_NETWORK_TEST") + assert enabled == "1" or "PINEFORGE_NETWORK_TEST=1" in tokens, ( + "the required row is skipped unless its real opt-in is enabled" + ) + assert any(token.startswith("--junitxml=") for token in tokens) or "--junitxml" in tokens + assert job_id in needs(jobs["ready"]) + assert job_id in needs(jobs["check"]) + return job_id + + +def test_main_ci_explicitly_executes_the_required_live_pypi_row() -> None: + assert_main_network_job(load("ci", post=True)) + + +def test_pr_ci_keeps_the_live_pypi_proof_in_the_main_tier() -> None: + workflow = load("ci") + assert workflow.get("env", {}).get("PINEFORGE_NETWORK_TEST") != "1" + for job in workflow["jobs"].values(): + assert job.get("env", {}).get("PINEFORGE_NETWORK_TEST") != "1" + for step in job.get("steps", []): + assert step.get("env", {}).get("PINEFORGE_NETWORK_TEST") != "1" + command = step.get("run", "") + assert NETWORK_NODE not in command + assert "PINEFORGE_NETWORK_TEST=1" not in command + + +def assert_native_aggregate(jobs: dict[str, Any]) -> str: + aggregates = [ + (job_id, job) for job_id, job in jobs.items() if job.get("name") == "Native proof" + ] + assert len(aggregates) == 1, "native legs alone do not produce immutable native evidence" + job_id, aggregate = aggregates[0] + assert needs(aggregate) == {"authz", "main-ref", *NATIVE_ROUTES} + assert "continue-on-error" not in aggregate + return job_id + + +def test_main_native_workflow_requires_the_producing_aggregate() -> None: + assert_native_aggregate(load("native-feed", post=True)["jobs"]) def assert_post_ci(workflow: dict[str, Any]) -> None: jobs = workflow["jobs"] + network_id = assert_main_network_job(workflow) assert "changes" not in jobs - assert set(jobs) == {"authz", "main-ref", "check", *ROUTES} + assert set(jobs) == {"authz", "main-ref", "check", network_id, *ROUTES} for job_id, route in ROUTES.items(): job = jobs[job_id] assert job["runs-on"] == route, job_id assert "if" not in job, job_id expected = {"authz", "main-ref"} if job_id == "ready": - expected |= evidence.NEEDS + expected |= evidence.NEEDS | {network_id} assert needs(job) == expected, job_id check = jobs["check"] assert check["if"] == "${{ always() }}" @@ -157,7 +443,45 @@ def assert_post_ci(workflow: dict[str, Any]) -> None: '"python-package-distributions-$ATTEMPT" --dir dist' ), } - assert jobs[job_id]["steps"] == expected, job_id + expected.insert(3, { + "name": "Download this attempt's actual live PyPI report", + "if": "needs.authz.result == 'success'", + "env": { + "GH_TOKEN": "${{ github.token }}", + "GH_REPO": "${{ github.repository }}", + "RUN_ID": "${{ github.run_id }}", + "ATTEMPT": "${{ github.run_attempt }}", + }, + "run": ( + 'gh run download "$RUN_ID" --name ' + '"network-proof-$ATTEMPT" --dir reports/network' + ), + }) + observed = copy.deepcopy(jobs[job_id]["steps"]) + aggregate_steps = [ + step for step in observed if "NEEDS_JSON" in step.get("env", {}) + ] + assert len(aggregate_steps) == 1 + encoded_needs = aggregate_steps[0]["env"].pop("NEEDS_JSON") + # Check every dependency-to-result binding, including the new main-only job. + # Keep all other existing step bytes under the original comparison. + expanded = re.sub( + r"\$\{\{\s*toJSON\(needs\.([A-Za-z0-9_-]+)\)\s*\}\}", + r'"\1"', + encoded_needs, + ) + pairs = json.loads(expanded, object_pairs_hook=list) + required = evidence.NEEDS | {network_id} + assert len(pairs) == len(required) + assert dict(pairs) == {name: name for name in required} + expected_aggregate = next( + step for step in expected if "NEEDS_JSON" in step.get("env", {}) + ) + expected_aggregate["env"].pop("NEEDS_JSON") + expected_aggregate["run"] += " --network-proof reports/network" + else: + observed = jobs[job_id]["steps"] + assert observed == expected, job_id def test_post_ci_has_every_proof_without_pr_classification() -> None: @@ -185,9 +509,47 @@ def test_post_ci_control_rejects_reverting_a_proof(fault: str) -> None: assert_post_ci(changed) +def with_native_reports(steps: list[dict[str, Any]], leg: str) -> list[dict[str, Any]]: + """Only augment the original commands with registration/JUnit production.""" + test_command = "ctest --test-dir build-feed --output-on-failure\n" + producing = [step for step in steps if test_command in step.get("run", "")] + assert len(producing) == 1 + producing[0]["run"] = "set -euo pipefail\n" + producing[0]["run"].replace( + test_command, + "ctest --test-dir build-feed --show-only=json-v1 > native-registration.json\n" + 'ctest --test-dir build-feed --output-on-failure --output-junit ' + '"$GITHUB_WORKSPACE/native-junit.xml"\n', + ) + return [ + *steps, + { + "uses": PINS["actions/setup-python@v6"], + "with": {"python-version": "3.13"}, + }, + { + "name": "Bind the actual registration and JUnit to this leg", + "env": {"PYTHONPATH": "scripts"}, + "run": ( + f"python -m release_chain.main_proofs native-leg --leg {leg} " + "--directory . --output native-leg.json" + ), + }, + { + "uses": PINS["actions/upload-artifact@v5"], + "if": "always()", + "with": { + "name": f"native-leg-{leg}-${{{{ github.run_attempt }}}}", + "path": "native-registration.json\nnative-junit.xml\nnative-leg.json\n", + "if-no-files-found": "error", + }, + }, + ] + + def assert_native_post(workflow: dict[str, Any]) -> None: jobs = workflow["jobs"] - assert set(jobs) == {"authz", "main-ref", *NATIVE_ROUTES} + aggregate_id = assert_native_aggregate(jobs) + assert set(jobs) == {"authz", "main-ref", aggregate_id, *NATIVE_ROUTES} for job_id, (name, route) in NATIVE_ROUTES.items(): job = jobs[job_id] assert job["name"] == name @@ -203,11 +565,11 @@ def assert_native_post(workflow: dict[str, Any]) -> None: expected = copy.deepcopy(original["steps"]) expected[0]["uses"] = PINS[expected[0]["uses"]] expected[1]["run"] += " libdigest-sha-perl" - assert jobs[job_id]["steps"] == expected + assert jobs[job_id]["steps"] == with_native_reports(expected, job_id) for job_id in ("asan-ubsan", "tsan"): expected = copy.deepcopy(load("native-feed")["jobs"][job_id]["steps"][1:]) expected[0]["run"] += " libdigest-sha-perl" - assert jobs[job_id]["steps"][1:] == expected + assert jobs[job_id]["steps"][1:] == with_native_reports(expected, job_id) def test_native_post_keeps_linux_proofs_and_the_hosted_macos_leg() -> None: @@ -216,9 +578,10 @@ def test_native_post_keeps_linux_proofs_and_the_hosted_macos_leg() -> None: def test_native_pods_install_the_curl_scripts_checksum_tool() -> None: jobs = load("native-feed", post=True)["jobs"] - for job_id, job in jobs.items(): - if job_id in {"authz", "main-ref", "release-macos"}: + for job_id in NATIVE_ROUTES: + if job_id == "release-macos": continue + job = jobs[job_id] commands = "\n".join(step.get("run", "") for step in job["steps"]) assert "libdigest-sha-perl" in commands assert commands.index("libdigest-sha-perl") < commands.index("build_ci_curl.sh") diff --git a/tests/test_release_adapter.py b/tests/test_release_adapter.py index 3d00849..7d43bb3 100644 --- a/tests/test_release_adapter.py +++ b/tests/test_release_adapter.py @@ -1,9 +1,11 @@ from __future__ import annotations import copy +import io import shutil import subprocess import time +import zipfile from pathlib import Path from typing import Any from xml.etree import ElementTree @@ -376,19 +378,231 @@ def green_gate() -> tuple[dict[str, Any], dict[str, Any], list[dict[str, Any]]]: return gate, run, jobs +MAIN_NETWORK_NODE = ( + "tests/test_consumer_receipt.py::test_bootstrap_recorded_hashes_match_live_pypi" +) +MAIN_NATIVE_JOBS = { + "release-amd64": "Release (ubuntu-24.04)", + "release-arm64": "Release (ubuntu-24.04-arm)", + "release-macos": "Release (macos-15)", + "asan-ubsan": "ASan + UBSan", + "tsan": "TSan", +} + + +@pytest.fixture(params=[("landed-one", 80, 2), ("landed-two", 180, 3)]) +def main_proof_inputs(request): + """Decided schema as unit input; these are not observed hosted results.""" + label, run_id, attempt = request.param + gate, run, jobs = green_gate() + head = contract.sha256(label.encode())[:40] + tree = contract.sha256(f"{label}:tree".encode())[:40] + gate.update( + run_id=run_id, attempt=attempt, head=head, tested_sha=head, tree=tree, + event="push", pr=None, base=None, + ) + run.update( + id=run_id, run_attempt=attempt, head_sha=head, head_branch="main", + event="push", path=".github/workflows/ci-post.yml", + ) + # The job label is fixture data, not a new public API or product constant. + # Once present, use the actual workflow's display name rather than pinning a choice here. + workflow = yaml.load( + (ROOT / ".github/workflows/ci-post.yml").read_text(), Loader=yaml.BaseLoader + ) + network_names = [ + job.get("name", job_id) + for job_id, job in workflow["jobs"].items() + if any(MAIN_NETWORK_NODE in step.get("run", "") for step in job.get("steps", [])) + ] + network_name = network_names[0] if len(network_names) == 1 else "Live PyPI hashes" + if not any(job["name"] == network_name for job in jobs): + jobs.append({"name": network_name, "status": "completed", "conclusion": "success"}) + native_run = { + **run, "id": run_id + 10, "run_attempt": attempt + 1, + "path": ".github/workflows/native-feed-post.yml", + } + legs = {} + for leg in MAIN_NATIVE_JOBS: + names = ["feed-unit", "feed-export-unit", "feed-mock-export", "feed-mock"] + if leg == "tsan": + names.append("feed-tsan-seeded-race") + junit = ElementTree.Element("testsuite", name=f"{label}:{leg}") + for name in names: + ElementTree.SubElement(junit, "testcase", name=name) + legs[leg] = { + "registered": names, "executed": list(names), "skipped": [], + "failures": [], "errors": [], + "junit_sha256": contract.sha256(ElementTree.tostring(junit)), + } + network_junit = ElementTree.Element("testsuite", name=f"{label}:network") + ElementTree.SubElement(network_junit, "testcase", name=MAIN_NETWORK_NODE) + gate["main_proofs"] = { + "native": { + "workflow_path": native_run["path"], + "run": native_run, + "jobs": [ + {"name": name, "status": "completed", "conclusion": "success"} + for name in (*MAIN_NATIVE_JOBS.values(), "Native proof") + ], + "attestation": { + "run_id": native_run["id"], "attempt": native_run["run_attempt"], + "head": head, "tested_sha": head, "tree": tree, "legs": legs, + }, + }, + "network": { + "run_id": run_id, "attempt": attempt, "head": head, "tested_sha": head, + "tree": tree, "node_id": MAIN_NETWORK_NODE, + "registered": [MAIN_NETWORK_NODE], "executed": [MAIN_NETWORK_NODE], + "skipped": [], "failures": [], "errors": [], + "junit_sha256": contract.sha256(ElementTree.tostring(network_junit)), + }, + } + return gate, run, jobs + + +def test_main_complete_matching_proofs_positive_twin(main_proof_inputs) -> None: + gate, run, jobs = main_proof_inputs + # Acceptance alone is uncredited until the negative twins and I/O controls work. + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) + + +@pytest.mark.parametrize("missing", ["main_proofs", "native", "network"]) +def test_main_python_green_requires_both_proof_slots(main_proof_inputs, missing) -> None: + gate, run, jobs = main_proof_inputs + if missing == "main_proofs": + gate.pop("main_proofs") + else: + gate["main_proofs"].pop(missing) + with pytest.raises(ValueError): + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) + + +@pytest.mark.parametrize("shape", ["null", "empty", "extra"]) +def test_main_proof_object_has_exactly_the_decided_slots(main_proof_inputs, shape) -> None: + gate, run, jobs = main_proof_inputs + if shape == "null": + gate["main_proofs"] = None + elif shape == "empty": + gate["main_proofs"] = {} + else: + gate["main_proofs"]["unobserved"] = {"conclusion": "success"} + with pytest.raises(ValueError): + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) + + +@pytest.mark.parametrize( + "path,value", + [ + (("workflow_path",), ".github/workflows/native-feed.yml"), + (("run", "path"), ".github/workflows/ci-post.yml"), + (("run", "event"), "pull_request"), + (("run", "head_branch"), "topic"), + (("run", "head_sha"), "f" * 40), + (("run", "id"), 999), + (("run", "run_attempt"), 1), + (("run", "status"), "in_progress"), + (("run", "conclusion"), "failure"), + (("attestation", "run_id"), 999), + (("attestation", "attempt"), 1), + (("attestation", "head"), "e" * 40), + (("attestation", "tested_sha"), "e" * 40), + (("attestation", "tree"), "e" * 40), + ], +) +def test_main_refuses_native_identity_or_run_drift(main_proof_inputs, path, value) -> None: + gate, run, jobs = main_proof_inputs + target = gate["main_proofs"]["native"] + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + with pytest.raises(ValueError): + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) + + +@pytest.mark.parametrize("name", [*MAIN_NATIVE_JOBS.values(), "Native proof"]) +@pytest.mark.parametrize( + "fault", ["missing", "duplicate", "in_progress", "failure", "cancelled", "skipped"] +) +def test_main_refuses_incomplete_native_jobs(main_proof_inputs, name, fault) -> None: + gate, run, jobs = main_proof_inputs + native_jobs = gate["main_proofs"]["native"]["jobs"] + job = next(row for row in native_jobs if row["name"] == name) + if fault == "missing": + native_jobs.remove(job) + elif fault == "duplicate": + native_jobs.append(copy.deepcopy(job)) + elif fault == "in_progress": + job.update(status="in_progress", conclusion=None) + else: + job["conclusion"] = fault + with pytest.raises(ValueError): + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) + + +@pytest.mark.parametrize("leg", MAIN_NATIVE_JOBS) +@pytest.mark.parametrize( + "fault", + ["missing", "empty", "omitted", "duplicate", "skipped", "failures", "errors", "digest"], +) +def test_main_refuses_vacuous_or_incomplete_native_inventory( + main_proof_inputs, leg, fault +) -> None: + gate, run, jobs = main_proof_inputs + legs = gate["main_proofs"]["native"]["attestation"]["legs"] + row = legs[leg] + if fault == "missing": + legs.pop(leg) + elif fault == "empty": + row.update(registered=[], executed=[]) + elif fault == "omitted": + row["executed"].pop() + elif fault == "duplicate": + row["registered"].append(row["registered"][0]) + row["executed"].append(row["executed"][0]) + elif fault == "digest": + row["junit_sha256"] = "" + else: + row[fault] = [row["registered"][0]] + with pytest.raises(ValueError): + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) + + +@pytest.mark.parametrize( + "field,value", + [ + ("run_id", 999), ("attempt", 1), ("head", "e" * 40), + ("tested_sha", "e" * 40), ("tree", "e" * 40), + ("node_id", "tests/test_consumer_receipt.py::other"), + ("registered", []), ("executed", []), + ("registered", ["tests/test_consumer_receipt.py::other"]), + ("executed", ["tests/test_consumer_receipt.py::other"]), + ("registered", [MAIN_NETWORK_NODE, MAIN_NETWORK_NODE]), + ("executed", [MAIN_NETWORK_NODE, MAIN_NETWORK_NODE]), + ("skipped", [MAIN_NETWORK_NODE]), ("failures", [MAIN_NETWORK_NODE]), + ("errors", [MAIN_NETWORK_NODE]), ("junit_sha256", ""), + ], +) +def test_main_disabled_or_wrong_network_row_is_non_full( + main_proof_inputs, field, value +) -> None: + gate, run, jobs = main_proof_inputs + gate["main_proofs"]["network"][field] = value + with pytest.raises(ValueError): + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) + + def test_gate_accepts_only_the_complete_current_attempt() -> None: gate, run, jobs = green_gate() evidence.validate_gate(gate, run, jobs, head="a" * 40, base="b" * 40, pr=7) -def test_landed_evidence_requires_the_post_workflow() -> None: - gate, run, jobs = green_gate() - gate.update(event="push", pr=None, base=None) - run.update(event="push", path=".github/workflows/ci-post.yml") - evidence.validate_gate(gate, run, jobs, head="a" * 40, base=None, pr=None) +def test_landed_evidence_requires_the_post_workflow(main_proof_inputs) -> None: + gate, run, jobs = main_proof_inputs + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) run["path"] = ".github/workflows/ci.yml" with pytest.raises(ValueError, match="unexpected CI workflow"): - evidence.validate_gate(gate, run, jobs, head="a" * 40, base=None, pr=None) + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) def test_pr_evidence_cannot_use_the_post_workflow() -> None: @@ -429,6 +643,349 @@ def api(repo, endpoint): assert report["pr"]["merge"] == "a" * 40 +class RecoveryPublishersBoundary(RuntimeError): + """A validated recovery is about to read the later publisher-run inventory.""" + + +@pytest.fixture +def recovery_main_transport(baseline, main_proof_inputs, monkeypatch): + """Keep recovery, markers, local metadata, artifact parsing and gates real.""" + gate, python, jobs = main_proof_inputs + native = gate["main_proofs"]["native"] + python_proof = copy.deepcopy(gate) + python_proof["main_proofs"] = {"network": gate["main_proofs"]["network"]} + pr = { + "number": 7, "merged_at": "2026-11-07T12:00:00Z", + "merge_commit_sha": python["head_sha"], + "head": {"sha": "b" * 40}, "base": {"ref": "main", "sha": "c" * 40}, + "body": automation.body({ + "version": contract.package_version(baseline), "base": "c" * 40, + }), + } + state = { + "root": baseline, "pr": pr, "python": python, "python_runs": [python], + "python_proof": python_proof, "python_jobs": jobs, + "native": native, "native_runs": [native["run"]], + "native_artifact_missing": False, "reads": [], "writes": [], + "parsed": [], "gate_calls": [], "validated": [], "local_checks": [], + "publisher_reads": [], "local_attempts": [], + "waiting_inputs": [], "waiting_validations": [], + } + + def gh_bytes(route, **kwargs): + assert not kwargs + state["reads"].append(route) + prefix = "repos/owner/data/" + assert route.startswith(prefix) + endpoint = route.removeprefix(prefix) + path = endpoint.split("?", 1)[0] + if path == "actions/workflows/native-feed-post.yml/runs": + assert endpoint == ( + "actions/workflows/native-feed-post.yml/runs" + f"?event=push&head_sha={python['head_sha']}&per_page=100" + ) + return contract.json_bytes({"workflow_runs": state["native_runs"]}) + for run, observed_jobs, payload, artifact, filename in ( + (python, jobs, python_proof, "release-evidence", "evidence.json"), + (native["run"], native["jobs"], native["attestation"], + "native-evidence", "native-evidence.json"), + ): + run_id, attempt = run["id"], run["run_attempt"] + if path == f"actions/runs/{run_id}": + return contract.json_bytes(run) + if path == f"actions/runs/{run_id}/attempts/{attempt}/jobs": + return contract.json_bytes({"jobs": observed_jobs}) + if path == f"actions/runs/{run_id}/artifacts": + missing = artifact == "native-evidence" and state["native_artifact_missing"] + return contract.json_bytes({"artifacts": [] if missing else [{ + "id": run_id * 100, "name": f"{artifact}-{attempt}", "expired": False, + }]}) + if path == f"actions/artifacts/{run_id * 100}/zip": + archive = io.BytesIO() + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr(filename, contract.json_bytes(payload)) + return archive.getvalue() + raise AssertionError(f"unexpected recovery proof transport: {route}") + + def api(repo, endpoint, method="GET", data=None): + assert repo == "owner/data" + if method != "GET": + state["writes"].append((endpoint, method, data)) + raise AssertionError("recovery dispatched or mutated before its proof boundary") + assert data is None + state["reads"].append(endpoint) + if endpoint == "releases?per_page=100&page=1": + return [] + if endpoint == ( + "pulls?state=closed&base=main&sort=updated&direction=desc&per_page=100&page=1" + ): + return [copy.deepcopy(pr)] + if endpoint == ( + "actions/workflows/ci-post.yml/runs?event=push&per_page=100&head_sha=" + + pr["merge_commit_sha"] + ): + return {"workflow_runs": copy.deepcopy(state["python_runs"])} + if endpoint == "actions/workflows/release.yml/runs?per_page=100": + state["publisher_reads"].append(endpoint) + assert len(state["validated"]) == 1 + assert state["local_checks"] == [(baseline, None)] + raise RecoveryPublishersBoundary("before publisher-run inventory read") + raise AssertionError(f"unexpected recovery API transport: {endpoint}") + + real_parse = evidence.object_json + real_gate = publishing.gate_for_run + real_verify_local = publishing.verify_local + real_validate_waiting_python = evidence._validate_waiting_python + + def parse(raw): + result = real_parse(raw) + state["parsed"].append(copy.deepcopy(result)) + return result + + def observe_gate(repo, run_id, **kwargs): + state["gate_calls"].append((repo, run_id, kwargs)) + result = real_gate(repo, run_id, **kwargs) + state["validated"].append(copy.deepcopy(result)) + return result + + def verify_local(root): + state["local_attempts"].append(root) + result = real_verify_local(root) + state["local_checks"].append((root, copy.deepcopy(result))) + return result + + def validate_waiting_python(proof, run, observed_jobs, **kwargs): + state["waiting_inputs"].append(copy.deepcopy((proof, run, observed_jobs, kwargs))) + result = real_validate_waiting_python(proof, run, observed_jobs, **kwargs) + state["waiting_validations"].append(copy.deepcopy(proof)) + return result + + monkeypatch.setattr(automation, "api", api) + monkeypatch.setattr(publishing, "api", api) + monkeypatch.setattr(reporting, "api", api) + monkeypatch.setattr(evidence, "gh_bytes", gh_bytes) + monkeypatch.setattr(evidence, "object_json", parse) + monkeypatch.setattr(evidence, "_validate_waiting_python", validate_waiting_python) + monkeypatch.setattr(publishing, "gate_for_run", observe_gate) + monkeypatch.setattr(publishing, "verify_local", verify_local) + return state + + +@pytest.mark.parametrize("readiness", ["missing", "queued", "in_progress"]) +def test_recovery_waits_for_missing_or_pending_native_proof(recovery_main_transport, readiness): + state = recovery_main_transport + if readiness == "missing": + state["native_runs"] = [] + else: + state["native"]["run"].update(status=readiness, conclusion=None) + state["native_artifact_missing"] = True + for job in state["native"]["jobs"]: + job.update(status=readiness, conclusion=None) + report = receipts.receipt(None) + publishing.recover_applied("owner/data", state["root"], report) + assert report["state"] == "testing" + assert report["pr"] == automation.pr_identity({**state["pr"], "merged": True}) + assert state["gate_calls"] == [ + ("owner/data", state["python"]["id"], {"head": state["python"]["head_sha"]}), + ] + assert {"workflow_runs": state["native_runs"]} in state["parsed"] + assert state["python_proof"] in state["parsed"], "the Python artifact must be parsed" + assert state["validated"] == state["local_checks"] == state["publisher_reads"] == [] + reporting.incident("owner/data", report, "release-owner", "recover-applied") + assert state["writes"] == [] + + +@pytest.mark.parametrize("readiness", ["missing", "queued", "in_progress"]) +@pytest.mark.parametrize( + "fault,reason", + [ + ("stale-attempt", "stale CI attempt"), + ("network-non-object", "network: expected an object"), + ( + "network-job-red", + "required job 'Live PyPI hashes' missing, duplicated, failed, cancelled or skipped", + ), + ("wrong-tree", "network: wrong tree"), + ("e2e-incomplete", "incomplete E2E evidence"), + ], +) +def test_recovery_refuses_invalid_python_while_native_pending( + recovery_main_transport, readiness, fault, reason +): + state = recovery_main_transport + python, native = state["python"], state["native"] + current_python = copy.deepcopy(python) + proof = state["python_proof"] + if readiness == "missing": + state["native_runs"] = [] + else: + native["run"].update(status=readiness, conclusion=None) + state["native_artifact_missing"] = True + for job in native["jobs"]: + job.update(status=readiness, conclusion=None) + if fault == "stale-attempt": + proof["attempt"] -= 1 + elif fault == "network-non-object": + proof["main_proofs"]["network"] = [] + elif fault == "network-job-red": + next(job for job in state["python_jobs"] if job["name"] == "Live PyPI hashes")[ + "conclusion" + ] = "failure" + elif fault == "wrong-tree": + original_tree = proof["tree"] + proof["tree"] = contract.sha256(b"another completed recovery evidence tree")[:40] + assert proof["tree"] != original_tree + assert proof["main_proofs"]["network"]["tree"] == original_tree + else: + assert fault == "e2e-incomplete" + for suite in ("docker", "database"): + proof["e2e"][suite].pop() + assert python == current_python + assert python["status"] == "completed" and python["conclusion"] == "success" + report = receipts.receipt(None) + before = copy.deepcopy(report) + with pytest.raises(evidence.GateRejected) as rejected: + publishing.recover_applied("owner/data", state["root"], report) + assert type(rejected.value) is evidence.GateRejected + assert str(rejected.value) == reason + assert report["state"] == before["state"] == "pending" + assert report == { + **before, "pr": automation.pr_identity({**state["pr"], "merged": True}), + } + assert state["gate_calls"] == [ + ("owner/data", python["id"], {"head": python["head_sha"]}), + ] + assert {"workflow_runs": state["native_runs"]} in state["parsed"] + if readiness != "missing": + assert native["run"] in state["parsed"] + assert proof in state["parsed"] + assert {"jobs": state["python_jobs"]} in state["parsed"] + assert state["waiting_inputs"] == [ + (proof, python, state["python_jobs"], {"head": python["head_sha"], "base": None}), + ] + assert state["waiting_validations"] == state["validated"] == [] + assert state["local_attempts"] == state["local_checks"] == state["publisher_reads"] == [] + assert state["writes"] == [] + native_id = native["run"]["id"] + assert not any( + f"/runs/{native_id}/attempts/" in route + or f"/runs/{native_id}/artifacts" in route + or f"/artifacts/{native_id * 100}/" in route + for route in state["reads"] + ) + + +@pytest.mark.parametrize("readiness", ["missing", "queued", "in_progress"]) +def test_recovery_waits_with_valid_python_while_native_pending_positive_twin( + recovery_main_transport, readiness +): + state = recovery_main_transport + python, native = state["python"], state["native"] + if readiness == "missing": + state["native_runs"] = [] + else: + native["run"].update(status=readiness, conclusion=None) + state["native_artifact_missing"] = True + for job in native["jobs"]: + job.update(status=readiness, conclusion=None) + assert python["status"] == "completed" and python["conclusion"] == "success" + report = receipts.receipt(None) + publishing.recover_applied("owner/data", state["root"], report) + assert report["state"] == "testing" + assert report["pr"] == automation.pr_identity({**state["pr"], "merged": True}) + assert state["gate_calls"] == [ + ("owner/data", python["id"], {"head": python["head_sha"]}), + ] + assert {"workflow_runs": state["native_runs"]} in state["parsed"] + if readiness != "missing": + assert native["run"] in state["parsed"] + proof = state["python_proof"] + assert proof in state["parsed"] + assert {"jobs": state["python_jobs"]} in state["parsed"] + assert state["waiting_inputs"] == [ + (proof, python, state["python_jobs"], {"head": python["head_sha"], "base": None}), + ] + assert state["waiting_validations"] == [proof] + assert state["validated"] == state["local_attempts"] == state["local_checks"] == [] + assert state["publisher_reads"] == state["writes"] == [] + reporting.incident("owner/data", report, "release-owner", "recover-applied") + assert state["writes"] == [] + + +def test_recovery_without_python_observation_keeps_testing(recovery_main_transport): + state = recovery_main_transport + state["python_runs"] = [] + report = receipts.receipt(None) + publishing.recover_applied("owner/data", state["root"], report) + assert report["state"] == "testing" and report["pr"]["merge"] == state["python"]["head_sha"] + assert state["gate_calls"] == state["parsed"] == state["writes"] == [] + assert state["publisher_reads"] == state["local_checks"] == [] + assert not any("native-feed-post.yml" in route for route in state["reads"]) + + +@pytest.mark.parametrize( + "fault", + ["native-failure", "native-cancelled", "native-attempt", "native-malformed", + "native-missing-artifact", "python-failure", "python-cancelled"], +) +def test_recovery_completed_red_or_malformed_proof_is_not_waiting( + recovery_main_transport, fault +): + state = recovery_main_transport + native = state["native"] + if fault in {"native-failure", "native-cancelled"}: + native["run"]["conclusion"] = fault.removeprefix("native-") + reason = "newest native proof has not completed successfully" + elif fault in {"python-failure", "python-cancelled"}: + state["python"]["conclusion"] = fault.removeprefix("python-") + reason = "CI has not completed successfully" + elif fault == "native-attempt": + native["attestation"]["attempt"] -= 1 + reason = "native: wrong attempt" + elif fault == "native-malformed": + native["attestation"]["legs"] = [] + reason = "native legs: expected an object" + else: + state["native_artifact_missing"] = True + reason = f"missing unique native-evidence-{native['run']['run_attempt']} artifact" + report = receipts.receipt(None) + with pytest.raises(evidence.GateRejected) as rejected: + publishing.recover_applied("owner/data", state["root"], report) + assert str(rejected.value) == reason + assert report["state"] == "pending", "a completed refusal must not become testing" + assert state["gate_calls"] and state["parsed"] + assert state["validated"] == state["local_checks"] == state["publisher_reads"] == [] + assert state["writes"] == [] + + +def test_recovery_complete_pair_reaches_publisher_inventory_boundary(recovery_main_transport): + state = recovery_main_transport + report = receipts.receipt(None) + with pytest.raises(RecoveryPublishersBoundary, match="before publisher-run inventory read"): + publishing.recover_applied("owner/data", state["root"], report) + observed = state["validated"][0] + assert observed["main_proofs"]["native"] == state["native"] + assert observed["main_proofs"]["network"] == state["python_proof"]["main_proofs"]["network"] + assert state["native"]["attestation"] in state["parsed"] + assert state["python_proof"] in state["parsed"] + assert state["local_checks"] == [(state["root"], None)] + assert len(state["publisher_reads"]) == 1 and state["writes"] == [] + # Partial lifecycle only: publisher selection, dispatch, publication and replay are unproved. + + +def test_recovery_complete_pair_still_reads_local_metadata(recovery_main_transport): + state = recovery_main_transport + (state["root"] / contract.DOCKERFILE).write_text( + "ARG PINEFORGE_RELEASE_IMAGE=wrong-runtime-pin\n" + ) + report = receipts.receipt(None) + with pytest.raises(ValueError) as rejected: + publishing.recover_applied("owner/data", state["root"], report) + assert str(rejected.value) == "runtime/server pin mismatch" + assert len(state["validated"]) == 1 + assert state["local_checks"] == state["publisher_reads"] == state["writes"] == [] + + @pytest.mark.parametrize("job_name", [*evidence.JOBS, "release/ready"]) @pytest.mark.parametrize("result", ["failure", "cancelled", "skipped", "missing"]) def test_green_workflow_cannot_hide_a_missing_or_failed_gate(job_name, result) -> None: @@ -443,6 +1000,8 @@ def test_green_workflow_cannot_hide_a_missing_or_failed_gate(job_name, result) - @pytest.fixture def evidence_inputs(tmp_path: Path, monkeypatch): + from release_chain import main_proofs + root = tmp_path / "repository" workflow = root / ".github" / "workflows" / "ci.yml" workflow.parent.mkdir(parents=True) @@ -450,16 +1009,14 @@ def evidence_inputs(tmp_path: Path, monkeypatch): (workflow.parent / "ci-post.yml").write_bytes( (ROOT / ".github/workflows/ci-post.yml").read_bytes() ) - hooks = tmp_path / "empty-hooks" - hooks.mkdir() subprocess.run( - ["git", "-c", "commit.gpgsign=false", "init"], + ["git", "init", "-b", "main"], cwd=root, check=True, capture_output=True, ) subprocess.run( - ["git", "-c", "commit.gpgsign=false", "add", ".github/workflows/ci.yml"], + ["git", "add", ".github/workflows/ci.yml", ".github/workflows/ci-post.yml"], cwd=root, check=True, capture_output=True, @@ -471,10 +1028,6 @@ def evidence_inputs(tmp_path: Path, monkeypatch): "user.name=Release evidence test", "-c", "user.email=release-evidence@example.invalid", - "-c", - "commit.gpgsign=false", - "-c", - f"core.hooksPath={hooks}", "commit", "-m", "Test release evidence", @@ -494,64 +1047,133 @@ def evidence_inputs(tmp_path: Path, monkeypatch): monkeypatch.setenv("GITHUB_RUN_ID", "70") monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "2") monkeypatch.setenv("GITHUB_EVENT_NAME", "push") - needs = {name: {"result": "success"} for name in evidence.NEEDS} - jobs = [{"name": name} for name in evidence.JOBS] + monkeypatch.setenv("GITHUB_REF", "refs/heads/main") + monkeypatch.setenv( + "GITHUB_WORKFLOW_REF", + "pineforge-4pass/pineforge-data/.github/workflows/ci-post.yml@refs/heads/main", + ) + head = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=root, text=True + ).strip() + monkeypatch.setenv("GITHUB_SHA", head) + # Authored unit report bytes, outside the exact two-distribution inventory. + network = tmp_path / "network" + network.mkdir() + suite = ElementTree.Element( + "testsuite", tests="1", failures="0", errors="0", skipped="0" + ) + ElementTree.SubElement( + suite, + "testcase", + classname="tests.test_consumer_receipt", + name="test_bootstrap_recorded_hashes_match_live_pypi", + ) + (network / "network.xml").write_bytes(ElementTree.tostring(suite)) + (network / "network-proof.json").write_bytes( + contract.json_bytes(main_proofs.network_proof(root, network)) + ) + needs = {name: {"result": "success"} for name in evidence.NEEDS | {"network-proof"}} + jobs = [{"name": name} for name in (*evidence.JOBS, "Live PyPI hashes")] def attempt_jobs(repo, run_id, attempt): assert (repo, run_id, attempt) == ("pineforge-4pass/pineforge-data", 70, 2) return jobs monkeypatch.setattr(evidence, "attempt_jobs", attempt_jobs) - return root, directory, needs, jobs + return root, directory, network, needs, jobs @pytest.mark.parametrize("stale_statuses", [False, True], ids=["names-only", "stale-statuses"]) def test_create_evidence_accepts_api_inventory_without_waiting( evidence_inputs, monkeypatch, stale_statuses ) -> None: - root, directory, needs, jobs = evidence_inputs + root, directory, network, needs, jobs = evidence_inputs if stale_statuses: for index, job in enumerate(jobs): job["status"] = "in_progress" if index % 2 == 0 else "queued" + job["conclusion"] = None if index % 2 == 0 else "failure" def reject_sleep(_seconds: float) -> None: raise AssertionError("sleep") monkeypatch.setattr(time, "sleep", reject_sleep) - result = evidence.create_evidence(root, directory, needs) + result = evidence.create_evidence(root, directory, needs, network=network) assert result["jobs"] == dict.fromkeys(evidence.JOBS, "success") + assert result["event"] == "push" and result["pr"] is None and result["base"] is None + assert result["head"] == result["tested_sha"] == subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=root, text=True + ).strip() + assert result["tree"] == subprocess.check_output( + ["git", "rev-parse", "HEAD^{tree}"], cwd=root, text=True + ).strip() + assert result["main_proofs"] == { + "network": contract.object_json((network / "network-proof.json").read_bytes()) + } + assert result["main_proofs"]["network"]["junit_sha256"] == contract.sha256( + (network / "network.xml").read_bytes() + ) + assert result["distributions"] == { + "pineforge_data-0.0.0-py3-none-any.whl": contract.sha256(b"wheel"), + "pineforge_data-0.0.0.tar.gz": contract.sha256(b"sdist"), + } + assert network.parent == directory.parent and network != directory assert not hasattr(evidence, "JOBS_CLOSE_TIMEOUT") assert not hasattr(evidence, "closed_jobs") -@pytest.mark.parametrize("job_id", sorted(evidence.NEEDS)) +@pytest.mark.parametrize("job_id", sorted(evidence.NEEDS | {"network-proof"})) @pytest.mark.parametrize("result", ["failure", "cancelled", "skipped"]) def test_create_evidence_rejects_unsuccessful_needs(evidence_inputs, job_id, result) -> None: - root, directory, needs, _jobs = evidence_inputs + root, directory, network, needs, _jobs = evidence_inputs needs[job_id]["result"] = result with pytest.raises(ValueError, match="a required CI job failed"): - evidence.create_evidence(root, directory, needs) + evidence.create_evidence(root, directory, needs, network=network) -def test_create_evidence_rejects_missing_need(evidence_inputs) -> None: - root, directory, needs, _jobs = evidence_inputs - needs.pop("test") +@pytest.mark.parametrize("job_id", ["test", "network-proof"]) +def test_create_evidence_rejects_missing_need(evidence_inputs, job_id) -> None: + root, directory, network, needs, _jobs = evidence_inputs + needs.pop(job_id) with pytest.raises(ValueError, match="required CI job inventory changed"): - evidence.create_evidence(root, directory, needs) + evidence.create_evidence(root, directory, needs, network=network) -@pytest.mark.parametrize("job_name", evidence.JOBS) +@pytest.mark.parametrize("job_name", [*evidence.JOBS, "Live PyPI hashes"]) @pytest.mark.parametrize("defect", ["missing", "duplicated"]) def test_create_evidence_rejects_missing_or_duplicated_api_names( evidence_inputs, job_name, defect ) -> None: - root, directory, needs, jobs = evidence_inputs + root, directory, network, needs, jobs = evidence_inputs if defect == "missing": jobs[:] = [job for job in jobs if job["name"] != job_name] else: jobs.append({"name": job_name}) with pytest.raises(ValueError, match="missing or duplicated"): - evidence.create_evidence(root, directory, needs) + evidence.create_evidence(root, directory, needs, network=network) + + +def test_create_evidence_main_requires_network_input(evidence_inputs) -> None: + root, directory, _network, needs, _jobs = evidence_inputs + with pytest.raises(ValueError, match="main requires the actual live-PyPI report"): + evidence.create_evidence(root, directory, needs, network=None) + + +@pytest.mark.parametrize("fault", ["raw-report", "recorded-digest", "recorded-attempt"]) +def test_create_evidence_consumes_network_report_bytes(evidence_inputs, fault) -> None: + root, directory, network, needs, _jobs = evidence_inputs + if fault == "raw-report": + report = network / "network.xml" + report.write_bytes(report.read_bytes() + b"\n") + else: + path = network / "network-proof.json" + proof = contract.object_json(path.read_bytes()) + if fault == "recorded-digest": + proof["junit_sha256"] = contract.sha256(b"different raw report") + else: + proof["attempt"] -= 1 + path.write_bytes(contract.json_bytes(proof)) + with pytest.raises(ValueError, match="network report bytes or producer identity changed"): + evidence.create_evidence(root, directory, needs, network=network) @pytest.mark.parametrize( @@ -569,10 +1191,10 @@ def test_create_evidence_rejects_missing_or_duplicated_api_names( ids=["enabled", "comment", "escaped-key", "whitespace-only"], ) def test_create_evidence_rejects_continue_on_error(evidence_inputs, workflow) -> None: - root, directory, needs, _jobs = evidence_inputs + root, directory, network, needs, _jobs = evidence_inputs (root / ".github" / "workflows" / "ci.yml").write_text(workflow) with pytest.raises(ValueError, match=r"ci\.yml changed or uses continue-on-error"): - evidence.create_evidence(root, directory, needs) + evidence.create_evidence(root, directory, needs, network=network) @pytest.mark.parametrize( @@ -593,10 +1215,10 @@ def test_reviewed_ci_workflow_pin_matches_the_file(filename: str, pin: str) -> N def test_create_evidence_rejects_missing_workflow(evidence_inputs) -> None: - root, directory, needs, _jobs = evidence_inputs + root, directory, network, needs, _jobs = evidence_inputs (root / ".github" / "workflows" / "ci.yml").unlink() with pytest.raises(ValueError, match=r"cannot read ci\.yml"): - evidence.create_evidence(root, directory, needs) + evidence.create_evidence(root, directory, needs, network=network) @pytest.mark.parametrize( @@ -1167,6 +1789,223 @@ def test_completed_receipt_replay_does_not_dispatch_or_publish( assert report["deployed"] == saved["deployed"] +@pytest.fixture +def saved_recovery_transport(ready, baseline, monkeypatch): + """Real saved-package/lock/OCI verification over immutable fixture bytes.""" + event, manifest = ready + tagged = baseline / "tagged-source" + baseline_files = { + name: (baseline / name).read_bytes() + for name in (contract.RUNTIME, contract.DOCKERFILE, "pyproject.toml") + } + apply(tagged, baseline_files) + apply(tagged, contract.update_files(tagged, event, manifest)) + source_files = {name: (tagged / name).read_bytes() for name in contract.UPDATE_FILES} + # The installed original pin intentionally has no lock or manifest. + # Its package version identifies the stored receipt without claiming a new pin. + (baseline / "pyproject.toml").write_bytes(source_files["pyproject.toml"]) + archive = io.BytesIO() + with zipfile.ZipFile(archive, "w") as zipped: + for name in (contract.LOCK, contract.MANIFEST, contract.RUNTIME): + zipped.writestr(name.removeprefix("src/"), source_files[name]) + downloads = { + "pineforge_data-0.3.1-py3-none-any.whl": archive.getvalue(), + "pineforge_data-0.3.1.tar.gz": b"retained source distribution fixture", + } + hashes = {name: contract.sha256(raw) for name, raw in downloads.items()} + metadata = {"urls": [ + { + "filename": name, "digests": {"sha256": digest}, + "url": f"https://files.pythonhosted.org/{name}", + } + for name, digest in hashes.items() + ]} + saved = contract.object_json((FIXTURES / "sample-release-receipt.json").read_bytes()) + source = saved["pr"]["merge"] + saved.update( + event_id=event["event_id"], release_version=event["release_version"], + deployed=receipts.served(event, source, "0.3.1", hashes), + ) + assets = [ + {"id": index, "name": name} + for index, name in enumerate([*downloads, "release-receipt.json"], 1) + ] + release = {"tag_name": "v0.3.1", "assets": assets} + state = { + "root": baseline, "baseline_files": baseline_files, + "source_files": source_files, "saved": saved, + "event": event, "downloads": downloads, "reads": [], "git": [], + "steps": [], "package_calls": [], "local_reads": [], "local_results": [], + "processes": [], "snapshot_files": [], + } + registry_fixture(monkeypatch, event) + + def api(repo, endpoint, method="GET", data=None): + state["reads"].append(endpoint) + assert repo == "owner/data" and method == "GET" and data is None + if endpoint == "releases?per_page=100&page=1": + return [copy.deepcopy(release)] + assert endpoint == "releases/tags/v0.3.1" + return copy.deepcopy(release) + + def gh_bytes(endpoint, *, accept): + state["reads"].append(endpoint) + assert accept == "application/octet-stream" + asset = next( + item for item in assets + if endpoint == f"repos/owner/data/releases/assets/{item['id']}" + ) + if asset["name"] == "release-receipt.json": + return receipts.canonical(saved) + return downloads[asset["name"]] + + def fetch(url, headers=None): + state["reads"].append(url) + assert headers is None + if url == "https://pypi.org/pypi/pineforge-data/0.3.1/json": + return contract.json_bytes(metadata) + assert url.startswith("https://files.pythonhosted.org/") + return downloads[url.rsplit("/", 1)[1]] + + def git(*args): + state["git"].append(args) + if args == ("fetch", "origin", "refs/tags/v0.3.1"): + return "" + assert args == ("rev-parse", "v0.3.1^{commit}") + return source + + def snapshot(revision, root): + assert revision == source + paths = [root / name for name in sorted(source_files)] + for path in paths: + if ( + path.name.endswith(".env") + or not path.resolve().is_relative_to(baseline.resolve()) + ): + raise AssertionError(f"refused saved publication fixture file: {path}") + state["snapshot_files"].append([path.relative_to(root).as_posix() for path in paths]) + apply(root, source_files) + + real_verify_local = publishing.verify_local + real_verified_package = publishing.verified_package + real_temporary_directory = publishing.tempfile.TemporaryDirectory + + def observe_local(root): + state["steps"].append(("verify-local", root)) + state["local_reads"].append(root) + result = real_verify_local(root) + state["local_results"].append(copy.deepcopy(result)) + return result + + def observe_package(repo, version, stored): + state["package_calls"].append((repo, version, copy.deepcopy(stored))) + state["steps"].append(("verify-package-start", version)) + result = real_verified_package(repo, version, stored) + state["steps"].append(("verify-package-complete", version)) + return result + + def reject_process(command, **kwargs): + state["processes"].append(command) + raise AssertionError("saved receipt replay must not dispatch or publish") + + monkeypatch.setattr(automation, "api", api) + monkeypatch.setattr(publishing, "api", api) + monkeypatch.setattr(publishing, "gh_bytes", gh_bytes) + monkeypatch.setattr(publishing, "fetch", fetch) + monkeypatch.setattr(publishing, "git", git) + monkeypatch.setattr(publishing, "snapshot", snapshot) + # These observers return only the result of the complete original validators. + monkeypatch.setattr(publishing, "verify_local", observe_local) + monkeypatch.setattr(publishing, "verified_package", observe_package) + monkeypatch.setattr(publishing.subprocess, "run", reject_process) + monkeypatch.setattr( + publishing.tempfile, "TemporaryDirectory", + lambda: real_temporary_directory(dir=baseline), + ) + return state + + +def assert_saved_recovery_verification_order(state): + assert state["package_calls"] == [("owner/data", "0.3.1", state["saved"])] + assert len(state["local_reads"]) == len(state["local_results"]) == 2 + assert state["local_reads"][0] != state["root"] + assert state["local_reads"][1] == state["root"] + assert state["local_results"][0]["event"] == state["event"] + assert state["steps"][-2:] == [ + ("verify-package-complete", "0.3.1"), ("verify-local", state["root"]), + ] + assert state["git"] == [ + ("fetch", "origin", "refs/tags/v0.3.1"), ("rev-parse", "v0.3.1^{commit}"), + ] + assert state["reads"].count("https://pypi.org/pypi/pineforge-data/0.3.1/json") == 2 + assert state["snapshot_files"] == [sorted(contract.UPDATE_FILES)] + assert state["processes"] == [] + + +def test_recovery_saved_receipt_refuses_missing_installed_lock(saved_recovery_transport): + state = saved_recovery_transport + assert contract.verify_local(state["root"]) is None + report = receipts.receipt(state["event"], "reconcile") + before = copy.deepcopy(report) + with pytest.raises(ValueError, match=r"^stored publication source lock missing$"): + publishing.recover_applied("owner/data", state["root"], report) + assert_saved_recovery_verification_order(state) + assert state["local_results"][1] is None + assert report == before + + +def test_recovery_saved_receipt_replays_after_real_verification(saved_recovery_transport): + state = saved_recovery_transport + apply(state["root"], state["source_files"]) + report = receipts.receipt(state["event"], "reconcile") + publishing.recover_applied("owner/data", state["root"], report) + assert_saved_recovery_verification_order(state) + assert state["local_results"][1] == state["local_results"][0] + assert report == state["saved"] + + +def test_recovery_saved_receipt_keeps_installed_image_mismatch_refusal(saved_recovery_transport): + state = saved_recovery_transport + manifest = contract.object_json(state["source_files"][contract.MANIFEST]) + manifest["image"]["digest"] = "sha256:" + "0" * 64 + raw = contract.json_bytes(manifest) + installed_event = contract.event_from_manifest(raw) + apply(state["root"], state["baseline_files"]) + apply(state["root"], contract.update_files(state["root"], installed_event, raw)) + report = receipts.receipt(state["event"], "reconcile") + before = copy.deepcopy(report) + with pytest.raises(ValueError, match=r"^stored publication does not match installed defaults$"): + publishing.recover_applied("owner/data", state["root"], report) + assert_saved_recovery_verification_order(state) + assert state["local_results"][1]["event"] == installed_event + assert report == before + + +def test_recovery_keeps_stored_identity_priority_before_local_lock(saved_recovery_transport): + state = saved_recovery_transport + state["saved"]["event_id"] = "sha256:" + "0" * 64 + report = receipts.receipt(state["event"], "reconcile") + before = copy.deepcopy(report) + with pytest.raises(ValueError, match=r"^stored publication identity differs$"): + publishing.recover_applied("owner/data", state["root"], report) + assert state["package_calls"] == state["local_reads"] == state["git"] == [] + assert state["processes"] == [] + assert report == before + + +def test_recovery_verifies_package_before_requiring_local_lock(saved_recovery_transport): + state = saved_recovery_transport + name = "pineforge_data-0.3.1-py3-none-any.whl" + state["downloads"][name] += b"changed" + report = receipts.receipt(state["event"], "reconcile") + before = copy.deepcopy(report) + with pytest.raises(ValueError, match=r"^PyPI bytes differ$"): + publishing.recover_applied("owner/data", state["root"], report) + assert len(state["package_calls"]) == 1 + assert state["local_reads"] == state["git"] == state["processes"] == [] + assert report == before + + def assert_pull_request_filter_policy(workflow: dict[str, Any], name: str) -> None: assert "pull_request" in workflow["on"] assert "pull_request_target" not in workflow["on"] @@ -1293,7 +2132,7 @@ def test_workflows_keep_every_gate_ahead_of_oidc_publication() -> None: else: assert ci["jobs"][job]["env"]["PINEFORGE_DATABASE_E2E"] == "1" release = yaml.load((workflows / "release.yml").read_text(), Loader=yaml.BaseLoader) - assert release["on"]["workflow_run"]["workflows"] == ["CI"] + assert release["on"]["workflow_run"]["workflows"] == ["CI", "Native feed"] # old: assert set(release["jobs"]["publish"]["needs"]) == {"prepare", "build", "github-release"} assert set(release["jobs"]["publish"]["needs"]) == { "authz", From ad126749da67c4b62c6a4e756f0003a0794756f9 Mon Sep 17 00:00:00 2001 From: luisleo526 Date: Sun, 8 Nov 2026 11:46:05 +0800 Subject: [PATCH 2/2] fix: synchronize the native feed client cursor Use an atomic cursor for concurrent producer observations and client-worker advances while preserving send-before-advance behavior and existing queue and retention locking. --- native/feed/src/serve.cpp | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/native/feed/src/serve.cpp b/native/feed/src/serve.cpp index ec1ed7e..cd66e9d 100644 --- a/native/feed/src/serve.cpp +++ b/native/feed/src/serve.cpp @@ -28,7 +28,7 @@ constexpr auto keepalive = std::chrono::seconds(5); // the runner's WebSoc Json number(std::uint64_t value) { return Json::number(std::to_string(value)); } struct Client { - std::uint64_t next = 0; // the next message index this client receives + std::atomic next{0}; // the next message index this client receives std::deque> queue; std::size_t queue_bytes = 0; bool live = false, slow = false; @@ -57,7 +57,7 @@ struct Hub { // A client that cannot keep up is cut off; it never slows the producer or another client. if (shared->size() > config.client_queue_bytes - std::min(client->queue_bytes, config.client_queue_bytes)) { client->slow = true; - log("warn", "client_slow", Json::object({{"next", number(client->next)}, {"queued_bytes", number(client->queue_bytes)}})); + log("warn", "client_slow", Json::object({{"next", number(client->next.load())}, {"queued_bytes", number(client->queue_bytes)}})); continue; } client->queue_bytes += shared->size(); @@ -178,7 +178,7 @@ int admit(const mg_connection* constant, void* data) { else if (hub.connected >= hub.config.max_clients) status = 503, reason = "client limit reached (--max-clients)"; else { auto* client = new Client; - client->next = index; + client->next.store(index); hub.clients.insert(client); ++hub.connected; mg_set_user_connection_data(constant, client); @@ -219,7 +219,7 @@ void serve_client(mg_connection* connection, Hub& hub, Client* client) { const auto send = [&](const std::string& bytes, int opcode) { const auto written = mg_websocket_write(connection, opcode, bytes.data(), bytes.size()); if (written <= 0 || (!bytes.empty() && static_cast(written) != bytes.size())) { - log("warn", "client_write_failed", Json::object({{"next", number(client->next)}})); + log("warn", "client_write_failed", Json::object({{"next", number(client->next.load())}})); return false; } last_send = std::chrono::steady_clock::now(); @@ -227,7 +227,7 @@ void serve_client(mg_connection* connection, Hub& hub, Client* client) { }; const auto expired = [&] { std::lock_guard guard(hub.mutex); - return client->next < hub.first_retained; + return client->next.load() < hub.first_retained; }; // The journal is read only while catching up: a client at the head holds no segment open. std::optional reader; @@ -238,18 +238,18 @@ void serve_client(mg_connection* connection, Hub& hub, Client* client) { std::lock_guard guard(hub.mutex); stopping = hub.stopping; head = hub.published; - if (!stopping && client->next == head) client->live = true; + if (!stopping && client->next.load() == head) client->live = true; } if (stopping) { send_close(connection, 1001, "producer stopping"); return; } if (client->live) break; if (!reader) { reader.emplace(hub.journal); - if (!reader->seek(client->next)) { + if (!reader->seek(client->next.load())) { send_close(connection, 4410, "cursor expired: older than the retained journal"); return; } } - while (client->next < head) { + while (client->next.load() < head) { if (hub.halting) { send_close(connection, 1001, "producer stopping"); return; } auto line = reader->next(); if (!line) { @@ -270,7 +270,7 @@ void serve_client(mg_connection* connection, Hub& hub, Client* client) { } } if (!send(*line, MG_WEBSOCKET_OPCODE_TEXT)) return; - ++client->next; + client->next.fetch_add(1); } } reader.reset(); @@ -299,7 +299,7 @@ void serve_client(mg_connection* connection, Hub& hub, Client* client) { } if (line) { if (!send(*line, MG_WEBSOCKET_OPCODE_TEXT)) return; - ++client->next; + client->next.fetch_add(1); } else if (std::chrono::steady_clock::now() - last_send >= keepalive && !send("", MG_WEBSOCKET_OPCODE_PONG)) return; } } @@ -344,7 +344,7 @@ void on_connection_close(const mg_connection* connection) { hub->clients.erase(client); --hub->connected; } - const auto next = client->next; + const auto next = client->next.load(); delete client; mg_set_user_connection_data(connection, nullptr); try { log("info", "client_disconnected", Json::object({{"next", number(next)}})); } catch (...) {}