diff --git a/.github/workflows/ci-post.yml b/.github/workflows/ci-post.yml index 0b8ec50..92c7b51 100644 --- a/.github/workflows/ci-post.yml +++ b/.github/workflows/ci-post.yml @@ -182,6 +182,37 @@ jobs: - if: matrix.python-version == '3.13' run: python -m twine check --strict dist/* + network-proof: + name: Live PyPI hashes + needs: [authz, main-ref] + runs-on: ubuntu-24.04 + env: + PF_CI_LABELS: '["ubuntu-24.04"]' + PINEFORGE_NETWORK_TEST: "1" + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.13' + cache: pip + - run: python -m pip install -e '.[dev]' pytest-xdist + - name: Execute the required live PyPI hash row with one actual worker + run: pytest -n 1 tests/test_consumer_receipt.py::test_bootstrap_recorded_hashes_match_live_pypi --junitxml=network.xml + - name: Derive the required zero-skip inventory from the actual JUnit + env: + PYTHONPATH: scripts + run: python -m release_chain.main_proofs network --directory . --output network-proof.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: network-proof-${{ github.run_attempt }} + path: | + network.xml + network-proof.json + if-no-files-found: error + docker: name: Docker runtime needs: [authz, main-ref] @@ -399,7 +430,7 @@ jobs: ready: name: release/ready - needs: [authz, main-ref, test, contract, docker, database-e2e, distribution] + needs: [authz, main-ref, test, contract, docker, database-e2e, distribution, network-proof] runs-on: ubuntu-24.04 timeout-minutes: 15 env: @@ -421,6 +452,14 @@ jobs: RUN_ID: ${{ github.run_id }} ATTEMPT: ${{ github.run_attempt }} run: gh run download "$RUN_ID" --name "python-package-distributions-$ATTEMPT" --dir dist + - name: Download this attempt's actual live PyPI report + if: needs.authz.result == 'success' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + ATTEMPT: ${{ github.run_attempt }} + run: gh run download "$RUN_ID" --name "network-proof-$ATTEMPT" --dir reports/network - name: Require the complete job inventory and bind evidence to this head/base/tree if: needs.authz.result == 'success' env: @@ -428,9 +467,10 @@ jobs: NEEDS_JSON: >- {"test":${{ toJSON(needs.test) }},"contract":${{ toJSON(needs.contract) }}, "docker":${{ toJSON(needs.docker) }},"database-e2e":${{ toJSON(needs.database-e2e) }}, - "distribution":${{ toJSON(needs.distribution) }}} + "distribution":${{ toJSON(needs.distribution) }}, + "network-proof":${{ toJSON(needs.network-proof) }}} GH_TOKEN: ${{ github.token }} - run: python -m release_chain.evidence aggregate --path dist + run: python -m release_chain.evidence aggregate --path dist --network-proof reports/network - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 if: needs.authz.result == 'success' with: @@ -439,7 +479,7 @@ jobs: if-no-files-found: error check: - needs: [authz, main-ref, workflow-lint, test, docker, database-e2e, contract, distribution, docs-tests, ready] + needs: [authz, main-ref, workflow-lint, test, docker, database-e2e, contract, distribution, docs-tests, network-proof, ready] if: ${{ always() }} runs-on: ubuntu-24.04 timeout-minutes: 10 diff --git a/.github/workflows/native-feed-post.yml b/.github/workflows/native-feed-post.yml index 837100a..19b4ec6 100644 --- a/.github/workflows/native-feed-post.yml +++ b/.github/workflows/native-feed-post.yml @@ -4,7 +4,6 @@ name: Native feed on: push: branches: [main] - paths: ['native/feed/**', '.github/workflows/native-feed.yml', '.github/workflows/native-feed-post.yml'] workflow_dispatch: permissions: @@ -111,12 +110,30 @@ jobs: run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl" - name: Configure, build and test run: | + set -euo pipefail export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" export DYLD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${DYLD_LIBRARY_PATH:-}" cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \ -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" cmake --build build-feed -j4 - ctest --test-dir build-feed --output-on-failure + ctest --test-dir build-feed --show-only=json-v1 > native-registration.json + ctest --test-dir build-feed --output-on-failure --output-junit "$GITHUB_WORKSPACE/native-junit.xml" + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.13' + - name: Bind the actual registration and JUnit to this leg + env: + PYTHONPATH: scripts + run: python -m release_chain.main_proofs native-leg --leg release-amd64 --directory . --output native-leg.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: native-leg-release-amd64-${{ github.run_attempt }} + path: | + native-registration.json + native-junit.xml + native-leg.json + if-no-files-found: error release-arm64: name: Release (ubuntu-24.04-arm) @@ -139,12 +156,30 @@ jobs: run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl" - name: Configure, build and test run: | + set -euo pipefail export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" export DYLD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${DYLD_LIBRARY_PATH:-}" cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \ -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" cmake --build build-feed -j4 - ctest --test-dir build-feed --output-on-failure + ctest --test-dir build-feed --show-only=json-v1 > native-registration.json + ctest --test-dir build-feed --output-on-failure --output-junit "$GITHUB_WORKSPACE/native-junit.xml" + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.13' + - name: Bind the actual registration and JUnit to this leg + env: + PYTHONPATH: scripts + run: python -m release_chain.main_proofs native-leg --leg release-arm64 --directory . --output native-leg.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: native-leg-release-arm64-${{ github.run_attempt }} + path: | + native-registration.json + native-junit.xml + native-leg.json + if-no-files-found: error release-macos: name: Release (macos-15) @@ -167,12 +202,30 @@ jobs: run: bash native/feed/tests/build_ci_curl.sh "$RUNNER_TEMP/curl" - name: Configure, build and test run: | + set -euo pipefail export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" export DYLD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${DYLD_LIBRARY_PATH:-}" cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=Release \ -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" cmake --build build-feed -j4 - ctest --test-dir build-feed --output-on-failure + ctest --test-dir build-feed --show-only=json-v1 > native-registration.json + ctest --test-dir build-feed --output-on-failure --output-junit "$GITHUB_WORKSPACE/native-junit.xml" + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.13' + - name: Bind the actual registration and JUnit to this leg + env: + PYTHONPATH: scripts + run: python -m release_chain.main_proofs native-leg --leg release-macos --directory . --output native-leg.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: native-leg-release-macos-${{ github.run_attempt }} + path: | + native-registration.json + native-junit.xml + native-leg.json + if-no-files-found: error asan-ubsan: name: ASan + UBSan @@ -190,12 +243,30 @@ jobs: ASAN_OPTIONS: detect_leaks=1:abort_on_error=1 UBSAN_OPTIONS: halt_on_error=1:print_stacktrace=1 run: | + set -euo pipefail export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_C_COMPILER=clang-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \ -DPINEFORGE_FEED_SANITIZER=address,undefined cmake --build build-feed -j4 - ctest --test-dir build-feed --output-on-failure + ctest --test-dir build-feed --show-only=json-v1 > native-registration.json + ctest --test-dir build-feed --output-on-failure --output-junit "$GITHUB_WORKSPACE/native-junit.xml" + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.13' + - name: Bind the actual registration and JUnit to this leg + env: + PYTHONPATH: scripts + run: python -m release_chain.main_proofs native-leg --leg asan-ubsan --directory . --output native-leg.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: native-leg-asan-ubsan-${{ github.run_attempt }} + path: | + native-registration.json + native-junit.xml + native-leg.json + if-no-files-found: error tsan: name: TSan @@ -220,10 +291,70 @@ jobs: env: TSAN_OPTIONS: halt_on_error=1 run: | + set -euo pipefail export LD_LIBRARY_PATH="$RUNNER_TEMP/curl/lib:${LD_LIBRARY_PATH:-}" cmake -S native/feed -B build-feed -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo \ -DCMAKE_CXX_COMPILER=clang++-18 -DCMAKE_C_COMPILER=clang-18 -DCURL_DIR="$RUNNER_TEMP/curl/lib/cmake/CURL" \ -DPINEFORGE_FEED_SANITIZER=thread \ -DPINEFORGE_FEED_TSAN_OPTIONS_EXTRA=external_symbolizer_path=/usr/lib/llvm-18/bin/llvm-symbolizer cmake --build build-feed -j4 - ctest --test-dir build-feed --output-on-failure + ctest --test-dir build-feed --show-only=json-v1 > native-registration.json + ctest --test-dir build-feed --output-on-failure --output-junit "$GITHUB_WORKSPACE/native-junit.xml" + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.13' + - name: Bind the actual registration and JUnit to this leg + env: + PYTHONPATH: scripts + run: python -m release_chain.main_proofs native-leg --leg tsan --directory . --output native-leg.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + if: always() + with: + name: native-leg-tsan-${{ github.run_attempt }} + path: | + native-registration.json + native-junit.xml + native-leg.json + if-no-files-found: error + + native-proof: + name: Native proof + needs: [authz, main-ref, release-amd64, release-arm64, release-macos, asan-ubsan, tsan] + runs-on: ubuntu-24.04 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: '3.13' + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: native-leg-release-amd64-${{ github.run_attempt }} + path: native-reports/release-amd64 + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: native-leg-release-arm64-${{ github.run_attempt }} + path: native-reports/release-arm64 + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: native-leg-release-macos-${{ github.run_attempt }} + path: native-reports/release-macos + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: native-leg-asan-ubsan-${{ github.run_attempt }} + path: native-reports/asan-ubsan + - uses: actions/download-artifact@018cc2cf5baa6db3ef3c5f8a56943fffe632ef53 # v6.0.0 + with: + name: native-leg-tsan-${{ github.run_attempt }} + path: native-reports/tsan + - name: Require authoritative leg results and reparse their actual reports + env: + PYTHONPATH: scripts + NEEDS_JSON: ${{ toJSON(needs) }} + run: python -m release_chain.main_proofs native-aggregate --directory native-reports --output native-evidence.json + - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: native-evidence-${{ github.run_attempt }} + path: native-evidence.json + if-no-files-found: error diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 49fac1e..2f766b0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -23,7 +23,7 @@ on: required: false type: boolean workflow_run: - workflows: [CI] + workflows: [CI, Native feed] types: [completed] branches: [main] diff --git a/native/feed/src/serve.cpp b/native/feed/src/serve.cpp index ec1ed7e..cd66e9d 100644 --- a/native/feed/src/serve.cpp +++ b/native/feed/src/serve.cpp @@ -28,7 +28,7 @@ constexpr auto keepalive = std::chrono::seconds(5); // the runner's WebSoc Json number(std::uint64_t value) { return Json::number(std::to_string(value)); } struct Client { - std::uint64_t next = 0; // the next message index this client receives + std::atomic next{0}; // the next message index this client receives std::deque> queue; std::size_t queue_bytes = 0; bool live = false, slow = false; @@ -57,7 +57,7 @@ struct Hub { // A client that cannot keep up is cut off; it never slows the producer or another client. if (shared->size() > config.client_queue_bytes - std::min(client->queue_bytes, config.client_queue_bytes)) { client->slow = true; - log("warn", "client_slow", Json::object({{"next", number(client->next)}, {"queued_bytes", number(client->queue_bytes)}})); + log("warn", "client_slow", Json::object({{"next", number(client->next.load())}, {"queued_bytes", number(client->queue_bytes)}})); continue; } client->queue_bytes += shared->size(); @@ -178,7 +178,7 @@ int admit(const mg_connection* constant, void* data) { else if (hub.connected >= hub.config.max_clients) status = 503, reason = "client limit reached (--max-clients)"; else { auto* client = new Client; - client->next = index; + client->next.store(index); hub.clients.insert(client); ++hub.connected; mg_set_user_connection_data(constant, client); @@ -219,7 +219,7 @@ void serve_client(mg_connection* connection, Hub& hub, Client* client) { const auto send = [&](const std::string& bytes, int opcode) { const auto written = mg_websocket_write(connection, opcode, bytes.data(), bytes.size()); if (written <= 0 || (!bytes.empty() && static_cast(written) != bytes.size())) { - log("warn", "client_write_failed", Json::object({{"next", number(client->next)}})); + log("warn", "client_write_failed", Json::object({{"next", number(client->next.load())}})); return false; } last_send = std::chrono::steady_clock::now(); @@ -227,7 +227,7 @@ void serve_client(mg_connection* connection, Hub& hub, Client* client) { }; const auto expired = [&] { std::lock_guard guard(hub.mutex); - return client->next < hub.first_retained; + return client->next.load() < hub.first_retained; }; // The journal is read only while catching up: a client at the head holds no segment open. std::optional reader; @@ -238,18 +238,18 @@ void serve_client(mg_connection* connection, Hub& hub, Client* client) { std::lock_guard guard(hub.mutex); stopping = hub.stopping; head = hub.published; - if (!stopping && client->next == head) client->live = true; + if (!stopping && client->next.load() == head) client->live = true; } if (stopping) { send_close(connection, 1001, "producer stopping"); return; } if (client->live) break; if (!reader) { reader.emplace(hub.journal); - if (!reader->seek(client->next)) { + if (!reader->seek(client->next.load())) { send_close(connection, 4410, "cursor expired: older than the retained journal"); return; } } - while (client->next < head) { + while (client->next.load() < head) { if (hub.halting) { send_close(connection, 1001, "producer stopping"); return; } auto line = reader->next(); if (!line) { @@ -270,7 +270,7 @@ void serve_client(mg_connection* connection, Hub& hub, Client* client) { } } if (!send(*line, MG_WEBSOCKET_OPCODE_TEXT)) return; - ++client->next; + client->next.fetch_add(1); } } reader.reset(); @@ -299,7 +299,7 @@ void serve_client(mg_connection* connection, Hub& hub, Client* client) { } if (line) { if (!send(*line, MG_WEBSOCKET_OPCODE_TEXT)) return; - ++client->next; + client->next.fetch_add(1); } else if (std::chrono::steady_clock::now() - last_send >= keepalive && !send("", MG_WEBSOCKET_OPCODE_PONG)) return; } } @@ -344,7 +344,7 @@ void on_connection_close(const mg_connection* connection) { hub->clients.erase(client); --hub->connected; } - const auto next = client->next; + const auto next = client->next.load(); delete client; mg_set_user_connection_data(connection, nullptr); try { log("info", "client_disconnected", Json::object({{"next", number(next)}})); } catch (...) {} diff --git a/scripts/release_chain/evidence.py b/scripts/release_chain/evidence.py index 062d0e2..03c8607 100644 --- a/scripts/release_chain/evidence.py +++ b/scripts/release_chain/evidence.py @@ -2,6 +2,7 @@ from __future__ import annotations +import copy import io import json import os @@ -11,14 +12,24 @@ from typing import Any from xml.etree import ElementTree -from .contract import gh_bytes, json_bytes, object_json, require, sha256 +from .contract import HEX40, gh_bytes, json_bytes, object_json, require, sha256 +from .main_proofs import ( + NATIVE_WORKFLOW, + NETWORK_JOB, + NETWORK_NODE, + consume_network, + successful_jobs, + validate_identity, + validate_main, +) +from .main_proofs import validate_inventory as validate_proof_inventory # SHA256 of both reviewed CI definitions: no continue-on-error, matrix leg names = JOBS. # Edits must update the pin in the same PR after re-review: needs..result == "success" proves # every leg succeeded (2026-10-23, VP-145 review P0: a lexical test alone misses escaped YAML keys). REVIEWED_CI_WORKFLOW_SHA256 = "e2b41563fbb8ab32c3c16675fc3d411b9dade719593ab0157fd47d641062ea2a" REVIEWED_CI_POST_WORKFLOW_SHA256 = ( - "4551f97c4d3f2590dc62cb7ce5652991d7ea17f3ba92f6af880af8269efe58ad" + "b8d3d92bb4b106f8985f8bbe4d96f12bff67d4d2b71b5bbf07bb46c76f9a8d49" ) JOBS = ( @@ -31,6 +42,7 @@ "Distribution", ) NEEDS = {"test", "contract", "docker", "database-e2e", "distribution"} +PENDING_STATUSES = {"queued", "in_progress", "requested", "waiting", "pending"} TESTS = { "docker": { "test_local_runtime_uses_published_release_image", @@ -46,20 +58,104 @@ class GateRejected(ValueError): - def __init__(self, message: str, run: dict[str, Any], jobs: list[dict[str, Any]]) -> None: + def __init__( + self, message: str, run: dict[str, Any] | None, jobs: list[dict[str, Any]] | None + ) -> None: super().__init__(message) + self.run = run + self.jobs = jobs self.evidence = { - "run_id": run["id"], - "attempt": run["run_attempt"], - "head": run["head_sha"], + "run_id": run.get("id") if run is not None else None, + "attempt": run.get("run_attempt") if run is not None else None, + "head": run.get("head_sha") if run is not None else None, "jobs": { name: next( - (job["conclusion"] or job["status"] for job in jobs if job["name"] == name), + ( + job.get("conclusion") or job.get("status") + for job in jobs if job.get("name") == name + ), "missing", ) for name in (*JOBS, "release/ready") - }, + } if jobs is not None else None, + } + + +class ProofPending(GateRejected): + """An observed main counterpart is absent or has not completed yet.""" + + def __init__( + self, + message: str, + run: dict[str, Any] | None, + jobs: list[dict[str, Any]] | None, + *, + workflow: str, + ) -> None: + super().__init__(message, run, jobs) + self.workflow = workflow + self.observations = {workflow: {"run": run, "jobs": jobs}} + + +def require_main_ready( + run: dict[str, Any], run_id: int, *, head: str, workflow: str, message: str +) -> None: + require( + type(run_id) is int + and run_id > 0 + and type(run.get("id")) is int + and run["id"] == run_id + and run.get("path") == workflow + and run.get("event") == "push" + and run.get("head_branch") == "main" + and isinstance(head, str) + and HEX40.fullmatch(head) + and run.get("head_sha") == head + and type(run.get("run_attempt")) is int + and run["run_attempt"] > 0, + message, + ) + require(isinstance(run.get("status"), str) and "conclusion" in run, message) + if run["status"] in PENDING_STATUSES and run["conclusion"] is None: + raise ProofPending(message, run, None, workflow=workflow) + require(run["status"] == "completed" and run["conclusion"] == "success", message) + + +def semantic_evidence_identity(evidence: dict[str, Any]) -> dict[str, Any]: + """Preserve proof content while projecting raw API run/job identity.""" + identity = copy.deepcopy(evidence) + proofs = identity.get("main_proofs") + if isinstance(proofs, dict) and isinstance(proofs.get("native"), dict): + native = proofs["native"] + run = native.get("run") + require(isinstance(run, dict), "native run: expected an object") + native["run"] = { + field: run[field] + for field in ( + "id", "workflow_id", "path", "event", "head_branch", "head_sha", + "run_attempt", "status", "conclusion", + ) + if field in run } + jobs = native.get("jobs") + require( + isinstance(jobs, list) and all(isinstance(job, dict) for job in jobs), + "missing observed jobs", + ) + native["jobs"] = sorted( + ( + { + field: job[field] + for field in ( + "id", "name", "run_id", "run_attempt", "head_sha", "status", "conclusion", + ) + if field in job + } + for job in jobs + ), + key=json_bytes, + ) + return identity def check_junit(path: Path, suite: str) -> None: @@ -87,16 +183,22 @@ def distribution_hashes(directory: Path) -> dict[str, str]: def attempt_jobs(repo: str, run_id: int, attempt: int) -> list[dict[str, Any]]: - return object_json( + jobs = object_json( gh_bytes(f"repos/{repo}/actions/runs/{run_id}/attempts/{attempt}/jobs?per_page=100") - )["jobs"] + ).get("jobs") + require( + isinstance(jobs, list) and all(isinstance(job, dict) for job in jobs), + "missing observed jobs", + ) + assert isinstance(jobs, list) + return jobs # Completion and success come from needs (authoritative). # The jobs API supplies names only: it can lag a finished leg by minutes (2026-10-22). # No timer by owner rule. -def validate_inventory(jobs: list[dict[str, Any]]) -> None: - for name in JOBS: +def validate_inventory(jobs: list[dict[str, Any]], *, main: bool = False) -> None: + for name in (*JOBS, NETWORK_JOB) if main else JOBS: matching = [job for job in jobs if job["name"] == name] require( len(matching) == 1, @@ -120,8 +222,14 @@ def require_no_continue_on_error(root: Path) -> None: ) -def create_evidence(root: Path, directory: Path, needs: dict[str, Any]) -> dict[str, Any]: - require(set(needs) == NEEDS, "required CI job inventory changed") +def create_evidence( + root: Path, directory: Path, needs: dict[str, Any], *, network: Path | None = None +) -> dict[str, Any]: + main = os.environ["GITHUB_EVENT_NAME"] == "push" + require( + set(needs) == NEEDS | ({"network-proof"} if main else set()), + "required CI job inventory changed", + ) require( all(value.get("result") == "success" for value in needs.values()), "a required CI job failed, was cancelled, missing or skipped", @@ -135,8 +243,8 @@ def create_evidence(root: Path, directory: Path, needs: dict[str, Any]) -> dict[ run_id = int(os.environ["GITHUB_RUN_ID"]) attempt = int(os.environ["GITHUB_RUN_ATTEMPT"]) jobs = attempt_jobs(repo, run_id, attempt) - validate_inventory(jobs) - return { + validate_inventory(jobs, main=main) + result: dict[str, Any] = { "run_id": run_id, "attempt": attempt, "event": os.environ["GITHUB_EVENT_NAME"], @@ -153,9 +261,14 @@ def create_evidence(root: Path, directory: Path, needs: dict[str, Any]) -> dict[ ], "distributions": distribution_hashes(directory), } + if main: + require(network is not None, "main requires the actual live-PyPI report") + assert network is not None + result["main_proofs"] = {"network": consume_network(root, network)} + return result -def validate_gate( +def _validate_ci_evidence( evidence: dict[str, Any], run: dict[str, Any], jobs: list[dict[str, Any]], @@ -186,6 +299,45 @@ def validate_gate( validate_jobs(jobs, aggregate=True) +def validate_gate( + evidence: dict[str, Any], + run: dict[str, Any], + jobs: list[dict[str, Any]], + *, + head: str, + base: str | None, + pr: int | None, +) -> None: + _validate_ci_evidence(evidence, run, jobs, head=head, base=base, pr=pr) + if pr is None: + validate_main(evidence, run, jobs) + + +def _validate_waiting_python( + evidence: dict[str, Any], + run: dict[str, Any], + jobs: list[dict[str, Any]], + *, + head: str, + base: str | None, +) -> None: + _validate_ci_evidence(evidence, run, jobs, head=head, base=base, pr=None) + require( + evidence.get("tested_sha") == evidence.get("head"), + "main proof must test the landed main head", + ) + network = evidence["main_proofs"]["network"] + require(isinstance(network, dict), "network: expected an object") + validate_identity(network, run, evidence, "network") + validate_proof_inventory(network, "network") + require( + network.get("node_id") == NETWORK_NODE + and network["registered"] == network["executed"] == [NETWORK_NODE], + "network: missing unique required live-PyPI row", + ) + successful_jobs(jobs, (NETWORK_JOB,), run=run) + + def validate_jobs(jobs: list[dict[str, Any]], *, aggregate: bool) -> None: for name in (*JOBS, "release/ready") if aggregate else JOBS: matching = [job for job in jobs if job["name"] == name] @@ -209,21 +361,121 @@ def artifact_json(repo: str, run_id: int, name: str, filename: str) -> dict[str, return object_json(zipped.read(filename)) +def matching_main_runs(observed: Any, workflow: str, *, head: str) -> list[dict[str, Any]]: + require(isinstance(head, str) and HEX40.fullmatch(head), "invalid main proof head") + require(isinstance(observed, list), "missing workflow run observations") + runs = [] + for row in observed: + require( + isinstance(row, dict) and type(row.get("id")) is int and row["id"] > 0, + "workflow observation has no valid run identity", + ) + require( + type(row.get("run_attempt")) is int + and row["run_attempt"] > 0 + and all( + isinstance(row.get(field), str) and row[field] + for field in ("event", "head_branch", "head_sha", "path", "status") + ) + and HEX40.fullmatch(row["head_sha"]) + and row["status"] in PENDING_STATUSES | {"completed"} + and "conclusion" in row + and ( + row["conclusion"] is None + or (isinstance(row["conclusion"], str) and bool(row["conclusion"])) + ), + "malformed workflow run observation", + ) + if ( + row["event"] == "push" + and row["head_branch"] == "main" + and row["head_sha"] == head + and row["path"] == f".github/workflows/{workflow}" + ): + runs.append(row) + return runs + + +def latest_main_run(repo: str, workflow: str, *, head: str) -> dict[str, Any]: + """Choose by identity/id before examining status; never fall back to green.""" + runs = [] + page = 1 + while True: + suffix = "" if page == 1 else f"&page={page}" + observed = object_json( + gh_bytes( + f"repos/{repo}/actions/workflows/{workflow}/runs" + f"?event=push&head_sha={head}&per_page=100{suffix}" + ) + ).get("workflow_runs") + runs.extend(matching_main_runs(observed, workflow, head=head)) + assert isinstance(observed, list) + if len(observed) < 100: + break + page += 1 + if not runs: + raise ProofPending( + f"missing exact-head main {workflow} run", None, None, + workflow=f".github/workflows/{workflow}", + ) + return max(runs, key=lambda row: row["id"]) + + def gate_for_run( repo: str, run_id: int, *, head: str, base: str | None = None, pr: int | None = None ) -> dict[str, Any]: run = object_json(gh_bytes(f"repos/{repo}/actions/runs/{run_id}")) - jobs = object_json( - gh_bytes( - f"repos/{repo}/actions/runs/{run_id}/attempts/{run['run_attempt']}/jobs?per_page=100" - ) - )["jobs"] + jobs: list[dict[str, Any]] | None = None try: + if pr is None: + require_main_ready( + run, run_id, head=head, workflow=".github/workflows/ci-post.yml", + message="CI has not completed successfully", + ) + jobs = attempt_jobs(repo, run_id, run["run_attempt"]) validate_jobs(jobs, aggregate=True) evidence = artifact_json( repo, run_id, f"release-evidence-{run['run_attempt']}", "evidence.json" ) + if pr is None: + proofs = evidence.get("main_proofs") + require( + isinstance(proofs, dict) and set(proofs) == {"network"}, + "Python producer must supply its network proof only", + ) + assert isinstance(proofs, dict) + try: + native = latest_main_run(repo, "native-feed-post.yml", head=head) + # Bind the current attempt rather than the cached list observation. + current = object_json(gh_bytes(f"repos/{repo}/actions/runs/{native['id']}")) + require_main_ready( + current, native["id"], head=head, workflow=NATIVE_WORKFLOW, + message="newest native proof has not completed successfully", + ) + except ProofPending: + # A native wait cannot hide invalid completed Python evidence. + _validate_waiting_python(evidence, run, jobs, head=head, base=base) + raise + native_jobs = attempt_jobs(repo, current["id"], current["run_attempt"]) + attestation = artifact_json( + repo, current["id"], f"native-evidence-{current['run_attempt']}", + "native-evidence.json", + ) + evidence["main_proofs"] = { + **proofs, + "native": { + "workflow_path": NATIVE_WORKFLOW, + "run": current, + "jobs": native_jobs, + "attestation": attestation, + }, + } validate_gate(evidence, run, jobs, head=head, base=base, pr=pr) + except ProofPending as exc: + exc.observations.setdefault( + ".github/workflows/ci-post.yml", {"run": run, "jobs": jobs} + ) + raise except ValueError as exc: raise GateRejected(str(exc), run, jobs) from exc return evidence @@ -236,11 +488,15 @@ def main() -> None: parser.add_argument("command", choices=("junit", "aggregate")) parser.add_argument("--path", type=Path, required=True) parser.add_argument("--suite", choices=TESTS) + parser.add_argument("--network-proof", type=Path) args = parser.parse_args() if args.command == "junit": check_junit(args.path, args.suite) else: - evidence = create_evidence(Path.cwd(), args.path, json.loads(os.environ["NEEDS_JSON"])) + evidence = create_evidence( + Path.cwd(), args.path, json.loads(os.environ["NEEDS_JSON"]), + network=args.network_proof, + ) Path("evidence.json").write_bytes(json_bytes(evidence)) diff --git a/scripts/release_chain/main_proofs.py b/scripts/release_chain/main_proofs.py new file mode 100644 index 0000000..b2636ba --- /dev/null +++ b/scripts/release_chain/main_proofs.py @@ -0,0 +1,333 @@ +"""Produce and validate the actual native and live-PyPI main inventories.""" + +from __future__ import annotations + +import argparse +import os +import subprocess +from pathlib import Path +from typing import Any, cast +from xml.etree import ElementTree + +from .contract import HEX40, HEX64, json_bytes, object_json, require, sha256 + +NATIVE_WORKFLOW = ".github/workflows/native-feed-post.yml" +NATIVE_LEGS = { + "release-amd64": "Release (ubuntu-24.04)", + "release-arm64": "Release (ubuntu-24.04-arm)", + "release-macos": "Release (macos-15)", + "asan-ubsan": "ASan + UBSan", + "tsan": "TSan", +} +NETWORK_JOB = "Live PyPI hashes" +NETWORK_NODE = ( + "tests/test_consumer_receipt.py::test_bootstrap_recorded_hashes_match_live_pypi" +) +NATIVE_WORKFLOW_SHA256 = "2f720000c43809ac80e98e6ab7a96f58dff461691bb2c68f0f2fd149e167ffaa" + + +def object_value(value: Any, label: str) -> dict[str, Any]: + require(isinstance(value, dict), f"{label}: expected an object") + return cast(dict[str, Any], value) + + +def names(value: Any, label: str) -> list[str]: + require( + isinstance(value, list) + and bool(value) + and all(isinstance(name, str) and name for name in value), + f"{label}: missing nonempty inventory", + ) + require(len(value) == len(set(value)), f"{label}: duplicate inventory") + return cast(list[str], value) + + +def validate_inventory(value: Any, label: str) -> None: + row = object_value(value, label) + registered = names(row.get("registered"), f"{label} registered") + executed = names(row.get("executed"), f"{label} executed") + require(set(registered) == set(executed), f"{label}: registration/execution disagree") + for field in ("skipped", "failures", "errors"): + require(row.get(field) == [], f"{label}: required rows {field}") + digest = row.get("junit_sha256") + require( + isinstance(digest, str) and HEX64.fullmatch(digest), + f"{label}: missing raw JUnit SHA256", + ) + + +def _validate_native_inventory(value: Any, leg: str) -> None: + label = f"native {leg}" + require(leg in NATIVE_LEGS, f"{label}: unknown leg") + validate_inventory(value, label) + required = {"feed-unit", "feed-export-unit", "feed-mock-export", "feed-mock"} + if leg == "tsan": + required.add("feed-tsan-seeded-race") + observed = set(value["registered"]) & set(value["executed"]) + missing = sorted(required - observed) + require( + not missing, + f"{label}: missing required native rows: {', '.join(missing)}", + ) + + +def successful_jobs( + jobs: Any, required: tuple[str, ...], *, run: dict[str, Any] +) -> None: + require(isinstance(jobs, list), "missing observed jobs") + for name in required: + matches = [row for row in jobs if isinstance(row, dict) and row.get("name") == name] + require( + len(matches) == 1 + and matches[0].get("status") == "completed" + and matches[0].get("conclusion") == "success", + f"required job {name!r} missing, duplicated, failed, cancelled or skipped", + ) + for field, expected in ( + ("run_id", run.get("id")), + ("run_attempt", run.get("run_attempt")), + ("head_sha", run.get("head_sha")), + ): + if field in matches[0]: + require(matches[0][field] == expected, f"{name}: wrong observed {field}") + + +def validate_identity( + value: Any, run: dict[str, Any], evidence: dict[str, Any], label: str +) -> None: + row = object_value(value, label) + for field, expected in ( + ("run_id", run.get("id")), + ("attempt", run.get("run_attempt")), + ("head", evidence.get("head")), + ("tested_sha", evidence.get("tested_sha")), + ("tree", evidence.get("tree")), + ): + require(expected is not None and row.get(field) == expected, f"{label}: wrong {field}") + require( + type(run.get("id")) is int + and type(run.get("run_attempt")) is int + and type(row["run_id"]) is int + and row["run_id"] > 0 + and type(row["attempt"]) is int + and row["attempt"] > 0, + f"{label}: invalid run/attempt", + ) + for field in ("head", "tested_sha", "tree"): + require( + isinstance(row[field], str) and HEX40.fullmatch(row[field]), + f"{label}: invalid {field}", + ) + + +def validate_main( + evidence: dict[str, Any], run: dict[str, Any], jobs: list[dict[str, Any]] +) -> None: + proofs = object_value(evidence.get("main_proofs"), "main_proofs") + require(set(proofs) == {"native", "network"}, "main requires exactly native/network proofs") + require( + run.get("head_branch") == "main" + and evidence.get("tested_sha") == evidence.get("head"), + "main proof must test the landed main head", + ) + network = object_value(proofs["network"], "network") + validate_identity(network, run, evidence, "network") + validate_inventory(network, "network") + require( + network.get("node_id") == NETWORK_NODE + and network["registered"] == network["executed"] == [NETWORK_NODE], + "network: missing unique required live-PyPI row", + ) + successful_jobs(jobs, (NETWORK_JOB,), run=run) + + native = object_value(proofs["native"], "native") + observed = object_value(native.get("run"), "native run") + require( + native.get("workflow_path") == observed.get("path") == NATIVE_WORKFLOW + and observed.get("event") == "push" + and observed.get("head_branch") == "main" + and observed.get("head_sha") == evidence["head"] + and observed.get("status") == "completed" + and observed.get("conclusion") == "success", + "native: wrong source/workflow or incomplete run", + ) + successful_jobs(native.get("jobs"), (*NATIVE_LEGS.values(), "Native proof"), run=observed) + attestation = object_value(native.get("attestation"), "native attestation") + validate_identity(attestation, observed, evidence, "native") + legs = object_value(attestation.get("legs"), "native legs") + require(set(legs) == set(NATIVE_LEGS), "native: missing or unexpected leg") + for leg in NATIVE_LEGS: + _validate_native_inventory(legs[leg], leg) + + +def junit_inventory(path: Path, *, network: bool = False) -> dict[str, Any]: + raw = path.read_bytes() + root = ElementTree.fromstring(raw) + require(root.tag in {"testsuite", "testsuites"}, f"{path}: not a JUnit report") + cases = list(root.iter("testcase")) + registered, executed = [], [] + problems: dict[str, list[str]] = {"skipped": [], "failures": [], "errors": []} + for case in cases: + name = case.attrib.get("name", "") + if network: + classname = case.attrib.get("classname", "") + require(bool(classname), "network JUnit has no source classname") + name = f"{classname.replace('.', '/')}.py::{name}" + registered.append(name) + require( + case.get("status") in {None, "run", "passed", "notrun", "disabled"}, + f"{path}: unsuccessful or unknown testcase status", + ) + if case.find("skipped") is not None or case.get("status") in {"notrun", "disabled"}: + problems["skipped"].append(name) + else: + executed.append(name) + for tag, field in (("failure", "failures"), ("error", "errors")): + if case.find(tag) is not None: + problems[field].append(name) + row = { + "registered": registered, + "executed": executed, + **problems, + "junit_sha256": sha256(raw), + } + validate_inventory(row, str(path)) + # Collection/setup errors may be reported on a suite rather than a testcase. + for suite in root.iter(): + if suite.tag in {"testsuite", "testsuites"}: + if "tests" in suite.attrib: + require( + int(suite.attrib["tests"]) == len(list(suite.iter("testcase"))), + f"{path}: suite count disagrees with actual testcase rows", + ) + for field in ("failures", "errors", "skipped", "disabled"): + require(int(suite.get(field, "0")) == 0, f"{path}: suite {field}") + for tag in ("failure", "error", "skipped"): + require( + suite.find(tag) is None, f"{path}: suite contains {tag} element" + ) + return row + + +def native_inventory(registration: Path, junit: Path) -> dict[str, Any]: + raw = registration.read_bytes() + collected = object_json(raw) + tests = collected.get("tests") + require(isinstance(tests, list), "CTest registration has no tests list") + assert isinstance(tests, list) + registered = names( + [object_value(test, "CTest test").get("name") for test in tests], "CTest registration" + ) + row = junit_inventory(junit) + row["registered"] = registered + row["registration_sha256"] = sha256(raw) + validate_inventory(row, "CTest") + return row + + +def local_identity(root: Path, workflow: str) -> dict[str, Any]: + require( + os.environ["GITHUB_EVENT_NAME"] == "push" + and os.environ["GITHUB_REF"] == "refs/heads/main" + and os.environ["GITHUB_WORKFLOW_REF"].split("@", 1)[0] + == f"{os.environ['GITHUB_REPOSITORY']}/{workflow}", + "producer is not the required push/main workflow", + ) + tested = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=root, text=True).strip() + tree = subprocess.check_output( + ["git", "rev-parse", "HEAD^{tree}"], cwd=root, text=True + ).strip() + require(tested == os.environ["GITHUB_SHA"], "producer checkout differs from event head") + return { + "run_id": int(os.environ["GITHUB_RUN_ID"]), + "attempt": int(os.environ["GITHUB_RUN_ATTEMPT"]), + "head": os.environ["GITHUB_SHA"], + "tested_sha": tested, + "tree": tree, + } + + +def require_native_workflow(root: Path) -> None: + raw = (root / NATIVE_WORKFLOW).read_bytes() + require( + sha256(raw) == NATIVE_WORKFLOW_SHA256 and b"continue-on-error" not in raw, + "native workflow changed: authoritative needs require independent workflow review", + ) + + +def network_proof(root: Path, directory: Path) -> dict[str, Any]: + identity = local_identity(root, ".github/workflows/ci-post.yml") + row = junit_inventory(directory / "network.xml", network=True) + require(row["registered"] == row["executed"] == [NETWORK_NODE], "wrong live-PyPI inventory") + return {**identity, "node_id": NETWORK_NODE, **row} + + +def consume_network(root: Path, directory: Path) -> dict[str, Any]: + recorded = object_json((directory / "network-proof.json").read_bytes()) + actual = network_proof(root, directory) + require(recorded == actual, "network report bytes or producer identity changed") + return actual + + +def native_leg(root: Path, leg: str, directory: Path) -> dict[str, Any]: + require_native_workflow(root) + identity = local_identity(root, NATIVE_WORKFLOW) + inventory = native_inventory( + directory / "native-registration.json", directory / "native-junit.xml" + ) + _validate_native_inventory(inventory, leg) + return { + **identity, + "leg": leg, + "inventory": inventory, + } + + +def native_aggregate(root: Path, directory: Path, needs: dict[str, Any]) -> dict[str, Any]: + require(set(needs) == {"authz", "main-ref", *NATIVE_LEGS}, "native needs inventory changed") + require( + all(isinstance(row, dict) and row.get("result") == "success" for row in needs.values()), + "native dependency failed, cancelled or skipped", + ) + require_native_workflow(root) + identity = local_identity(root, NATIVE_WORKFLOW) + legs = {} + for leg in NATIVE_LEGS: + reports = directory / leg + recorded = object_json((reports / "native-leg.json").read_bytes()) + actual = native_leg(root, leg, reports) + require(recorded == actual, f"{leg}: report bytes or producer identity changed") + require( + all(recorded.get(key) == value for key, value in identity.items()), + f"{leg}: stale producer", + ) + legs[leg] = { + **actual["inventory"], + "artifact": f"native-leg-{leg}-{identity['attempt']}", + } + # Do not query/wait for this running aggregate's own completed job. + return {**identity, "legs": legs} + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=("network", "native-leg", "native-aggregate")) + parser.add_argument("--directory", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--leg", choices=NATIVE_LEGS) + args = parser.parse_args() + root = Path.cwd() + if args.command == "network": + value = network_proof(root, args.directory) + elif args.command == "native-leg": + require(args.leg is not None, "native leg identifier is required") + value = native_leg(root, args.leg, args.directory) + else: + value = native_aggregate( + root, args.directory, object_json(os.environ["NEEDS_JSON"].encode()) + ) + args.output.write_bytes(json_bytes(value)) + + +if __name__ == "__main__": + main() diff --git a/scripts/release_chain/publishing.py b/scripts/release_chain/publishing.py index b605b54..dea3400 100644 --- a/scripts/release_chain/publishing.py +++ b/scripts/release_chain/publishing.py @@ -14,7 +14,7 @@ from typing import Any from urllib.error import HTTPError -from .automation import api, finish, git, inventory, marker, pr_identity, receipt, snapshot +from .automation import api, finish, git, inventory, marker, pr_identity, snapshot from .contract import ( LOCK, MANIFEST, @@ -31,9 +31,16 @@ verify_artifacts, verify_local, ) -from .evidence import distribution_hashes, gate_for_run +from .evidence import ( + ProofPending, + distribution_hashes, + gate_for_run, + latest_main_run, + matching_main_runs, +) +from .main_proofs import NATIVE_WORKFLOW from .receipts import candidate as record_candidate -from .receipts import canonical, served, timestamp +from .receipts import canonical, receipt, served, timestamp # TEMPORARY first-adopter exemption (TX-DATA-BASELINE2, L6 10-19): # remove the data exemption and the hub form together, only after 0.3.1 is verified AND @@ -125,6 +132,7 @@ def candidate_gate(repo: str, pr: dict[str, Any]) -> dict[str, Any]: meta = marker(pr) head = (pr.get("head") or {}).get("sha") require(head, "pre-merge CI evidence missing") + assert head is not None runs = inventory( repo, "actions/workflows/ci.yml/runs?event=pull_request&head_sha=" + head, @@ -133,6 +141,8 @@ def candidate_gate(repo: str, pr: dict[str, Any]) -> dict[str, Any]: runs = [run for run in runs if run_belongs_to_pr(run, pr)] require(runs, "pre-merge CI evidence missing") run = max(runs, key=lambda row: row["id"]) + require(meta is not None, "pre-merge release marker missing") + assert meta is not None evidence = gate_for_run( repo, run["id"], head=pr["head"]["sha"], base=meta["base"], pr=pr["number"] ) @@ -161,6 +171,7 @@ def verified_package( and sum(name.endswith(".tar.gz") for name in files) == 1, "previous publication needs a wheel and sdist", ) + assert files is not None metadata = object_json(fetch(f"https://pypi.org/pypi/pineforge-data/{version}/json")) require( {entry["filename"]: entry["digests"]["sha256"] for entry in metadata["urls"]} == files, @@ -235,6 +246,7 @@ def verified_package( "previous distribution lacks manifest/facts provenance; " "establish a baseline before enablement", ) + assert lock is not None wheel = next(data for name, data in downloads.items() if name.endswith(".whl")) with zipfile.ZipFile(io.BytesIO(wheel)) as zipped: for name in (LOCK, MANIFEST, RUNTIME): @@ -281,32 +293,52 @@ def recover_applied(repo: str, source: Path, report: dict[str, Any]) -> None: saved["state"] == "verified" and saved["event_id"] == report["event_id"], "stored publication identity differs", ) + image_digest = verified_package(repo, version, saved)["image_digest"] + lock = verify_local(source) + require(lock is not None, "stored publication source lock missing") + assert lock is not None require( - verified_package(repo, version, saved)["image_digest"] - == verify_local(source)["event"]["image"]["digest"], + image_digest == lock["event"]["image"]["digest"], "stored publication does not match installed defaults", ) report.update(saved) return prs = inventory(repo, "pulls?state=closed&base=main&sort=updated&direction=desc") candidates = [ - pr for pr in prs if pr.get("merged_at") and marker(pr) and marker(pr)["version"] == version + pr + for pr in prs + if pr.get("merged_at") and (meta := marker(pr)) and meta["version"] == version ] require(len(candidates) == 1, "applied release has no unique merged update PR") pr = candidates[0] + meta = marker(pr) + assert meta is not None report["pr"] = pr_identity({**pr, "merged": True}) - report["pr"]["base_sha"] = marker(pr)["base"] - runs = api( - repo, - "actions/workflows/ci-post.yml/runs?event=push&per_page=100&head_sha=" - + pr["merge_commit_sha"], - )["workflow_runs"] - runs = [run for run in runs if run["head_sha"] == pr["merge_commit_sha"]] - if not runs or max(runs, key=lambda run: run["id"])["status"] != "completed": + report["pr"]["base_sha"] = meta["base"] + runs = [] + page = 1 + while True: + suffix = "" if page == 1 else f"&page={page}" + response = api( + repo, + "actions/workflows/ci-post.yml/runs?event=push&per_page=100&head_sha=" + + pr["merge_commit_sha"] + suffix, + ) + require(isinstance(response, dict), "missing workflow run observations") + observed = response.get("workflow_runs") + runs.extend(matching_main_runs(observed, "ci-post.yml", head=pr["merge_commit_sha"])) + if len(observed) < 100: + break + page += 1 + if not runs: report["state"] = "testing" return run = max(runs, key=lambda run: run["id"]) - gate_for_run(repo, run["id"], head=pr["merge_commit_sha"]) + try: + gate_for_run(repo, run["id"], head=pr["merge_commit_sha"]) + except ProofPending: + report["state"] = "testing" + return lock = verify_local(source) if lock: record_candidate(report, lock["event"], candidate_gate(repo, pr), version) @@ -399,6 +431,14 @@ def prepare( run = api(repo, f"actions/runs/{run_id}") require(run["event"] == "push" and run["head_branch"] == "main", "not landed main CI") source = run["head_sha"] + resolve_pair = ( + os.environ.get("GITHUB_EVENT_NAME") == "workflow_run" or run.get("path") == NATIVE_WORKFLOW + ) + if resolve_pair: + require( + run.get("path") in {".github/workflows/ci-post.yml", NATIVE_WORKFLOW}, + "unexpected main proof callback workflow", + ) # Check the associated PR before executing or checking out any candidate code. prs = api(repo, f"commits/{source}/pulls?per_page=100") matches = [ @@ -415,7 +455,26 @@ def prepare( pr = api(repo, f"pulls/{matches[0]['number']}") meta = marker(pr) require(pr["user"]["login"] == os.environ["RELEASE_BOT_LOGIN"], "unexpected release PR owner") + require(meta is not None, "release PR marker missing") + assert meta is not None require(pr["head"]["sha"] == meta["generated_head"], "release PR head changed") + try: + if resolve_pair: + # Either completion re-resolves the newest pair, including current attempts. + selected = latest_main_run(repo, "ci-post.yml", head=source) + run_id = selected["id"] + run = api(repo, f"actions/runs/{run_id}") + require( + run.get("id") == run_id + and run.get("event") == "push" + and run.get("head_branch") == "main" + and run.get("head_sha") == source + and run.get("path") == ".github/workflows/ci-post.yml", + "counterpart is not exact-head main CI", + ) + evidence = gate_for_run(repo, run_id, head=source) + except ProofPending: + return {"eligible": "false", "waiting": "main-proofs-pending"} git("fetch", "origin", source) # Keep this process and scripts at the trusted default branch. Read source as data. candidate = output / "source" @@ -426,7 +485,6 @@ def prepare( report.update(receipt(event, "prepare-publication")) report["pr"] = pr_identity(pr) report["pr"]["base_sha"] = meta["base"] - evidence = gate_for_run(repo, run_id, head=source) require( git("rev-parse", f"{source}^{{tree}}") == evidence["tree"] and evidence["tested_sha"] == source, @@ -435,6 +493,7 @@ def prepare( # Retain the independent pre-merge candidate gates, as required by the hub. # Main CI still authorizes the exact distributions we publish. if event: + assert lock is not None record_candidate(report, event, candidate_gate(repo, pr), meta["version"]) previous_version = lock["previous"]["package_version"] report["previous"] = verified_package(repo, previous_version, adopter_lock=lock) diff --git a/scripts/release_chain/reporting.py b/scripts/release_chain/reporting.py index d41c7bb..77b3f8e 100644 --- a/scripts/release_chain/reporting.py +++ b/scripts/release_chain/reporting.py @@ -11,7 +11,7 @@ from pathlib import Path from typing import Any -from .automation import api, finish, inventory, marker, pr_identity, receipt +from .automation import api, finish, inventory, marker, pr_identity from .contract import ( HUB, gh_bytes, @@ -22,7 +22,7 @@ sha256, verify_local, ) -from .evidence import artifact_json, gate_for_run +from .evidence import artifact_json, gate_for_run, semantic_evidence_identity from .publishing import candidate_gate, stored_receipt, verified_package from .receipts import ( DEPLOY_JOBS, @@ -30,6 +30,7 @@ FIELDS, WORKFLOW, candidate, + receipt, signal, timestamp, ) @@ -192,9 +193,11 @@ def publication(repo: str, run_id: int, report: dict[str, Any]) -> None: pr["merged"] and pr["head"]["repo"]["full_name"] == repo and pr["base"]["ref"] == "main" + and meta is not None and pr["head"]["sha"] == meta["generated_head"], "publication PR identity moved", ) + assert meta is not None source = pr["merge_commit_sha"] require(report["pr"] == pr_identity(pr, base=meta["base"]), "publication PR evidence differs") commit = api(repo, f"commits/{source}") @@ -221,12 +224,15 @@ def publication(repo: str, run_id: int, report: dict[str, Any]) -> None: check=True, ) main_gate = object_json((bundle / "evidence.json").read_bytes()) + fresh_gate = gate_for_run(repo, main_gate["run_id"], head=source) require( - gate_for_run(repo, main_gate["run_id"], head=source) == main_gate, + json_bytes(semantic_evidence_identity(fresh_gate)) + == json_bytes(semantic_evidence_identity(main_gate)), "landed CI distributions/evidence moved", ) lock = verify_local(bundle / "source") require(lock is not None and lock["event"] == event, "served bundle target differs") + assert lock is not None version = lock["package_version"] require(version == meta["version"], "served package differs from PR") candidate(report, event, candidate_gate(repo, pr), version) @@ -299,7 +305,10 @@ def retain(repo: str, report: dict[str, Any]) -> None: if report["state"] != "verified": return pr = api(repo, f"pulls/{report['pr']['number']}") - version = marker(pr)["version"] + meta = marker(pr) + require(meta is not None, "retained release marker missing") + assert meta is not None + version = meta["version"] saved = stored_receipt(repo, version) if saved: require( diff --git a/tests/test_consumer_receipt.py b/tests/test_consumer_receipt.py index c4ff25f..9321eba 100644 --- a/tests/test_consumer_receipt.py +++ b/tests/test_consumer_receipt.py @@ -7,17 +7,1700 @@ import os import zipfile from pathlib import Path +from urllib.parse import parse_qs, urlsplit +from xml.etree import ElementTree import hub_receipt_validator as hub import pytest import yaml -from release_chain import automation, contract, publishing, receipts, reporting +from release_chain import automation, contract, evidence, publishing, receipts, reporting FIXTURES = Path(__file__).parent / "fixtures/release_chain" REPO = "owner/data" HEAD, BASE, MERGE = "a" * 40, "b" * 40, "c" * 40 +@pytest.fixture(params=[("transport-one", 480, 2), ("transport-two", 580, 5)]) +def main_proof_transport(monkeypatch, request): + """Offline GitHub observations; only the bytes transport is replaced.""" + label, python_id, attempt = request.param + head = contract.sha256(label.encode())[:40] + tree = contract.sha256(f"{label}:tree".encode())[:40] + node = "tests/test_consumer_receipt.py::test_bootstrap_recorded_hashes_match_live_pypi" + native_names = { + "release-amd64": "Release (ubuntu-24.04)", + "release-arm64": "Release (ubuntu-24.04-arm)", + "release-macos": "Release (macos-15)", + "asan-ubsan": "ASan + UBSan", + "tsan": "TSan", + } + python_run = { + "id": python_id, "run_attempt": attempt, "head_sha": head, "head_branch": "main", + "status": "completed", "conclusion": "success", "event": "push", + "path": ".github/workflows/ci-post.yml", + } + native_run = { + **python_run, "id": python_id + 20, "run_attempt": attempt + 1, + "path": ".github/workflows/native-feed-post.yml", + } + legs = {} + for leg in native_names: + names = ["feed-unit", "feed-export-unit", "feed-mock-export", "feed-mock"] + if leg == "tsan": + names.append("feed-tsan-seeded-race") + junit = ElementTree.Element("testsuite", name=f"{label}:{leg}") + for name in names: + ElementTree.SubElement(junit, "testcase", name=name) + legs[leg] = { + "registered": names, "executed": list(names), "skipped": [], + "failures": [], "errors": [], + "junit_sha256": contract.sha256(ElementTree.tostring(junit)), + } + network_junit = ElementTree.Element("testsuite", name=f"{label}:network") + ElementTree.SubElement(network_junit, "testcase", name=node) + network = { + "run_id": python_id, "attempt": attempt, "head": head, "tested_sha": head, + "tree": tree, "node_id": node, "registered": [node], "executed": [node], + "skipped": [], "failures": [], "errors": [], + "junit_sha256": contract.sha256(ElementTree.tostring(network_junit)), + } + python_evidence = { + "run_id": python_id, "attempt": attempt, "head": head, "tested_sha": head, + "tree": tree, "base": None, "pr": None, "event": "push", + "jobs": dict.fromkeys(evidence.JOBS, "success"), + "e2e": {key: sorted(value) for key, value in evidence.TESTS.items()}, + # Native proof must be discovered independently, never supplied by this artifact. + "main_proofs": {"network": network}, + } + workflow = yaml.load( + (Path(__file__).parents[1] / ".github/workflows/ci-post.yml").read_text(), + Loader=yaml.BaseLoader, + ) + network_names = [ + job.get("name", job_id) + for job_id, job in workflow["jobs"].items() + if any(node in step.get("run", "") for step in job.get("steps", [])) + ] + network_name = network_names[0] if len(network_names) == 1 else "Live PyPI hashes" + python_jobs = [ + {"name": name, "status": "completed", "conclusion": "success"} + for name in (*evidence.JOBS, "release/ready") + ] + if not any(job["name"] == network_name for job in python_jobs): + python_jobs.append( + {"name": network_name, "status": "completed", "conclusion": "success"} + ) + native_jobs = [ + {"name": name, "status": "completed", "conclusion": "success"} + for name in (*native_names.values(), "Native proof") + ] + attestation = { + "run_id": native_run["id"], "attempt": native_run["run_attempt"], + "head": head, "tested_sha": head, "tree": tree, "legs": legs, + } + state = { + "python_run": python_run, "python_evidence": python_evidence, + "python_jobs": python_jobs, "network_name": network_name, + "native_run": native_run, "native_jobs": native_jobs, + "native_runs": [native_run], "attestation": attestation, + "other_native_jobs": copy.deepcopy(native_jobs), + "other_attestation": copy.deepcopy(attestation), + "native_artifact_missing": False, "calls": [], + "native_artifact_attempt_delta": 0, + "python_runs": [python_run], + "other_python_jobs": copy.deepcopy(python_jobs), + "other_python_evidence": copy.deepcopy(python_evidence), + "python_artifact_missing": False, "python_artifact_attempt_delta": 0, + "artifact_observations": [], + "artifact_raw": {}, + } + + def gh_bytes(route, **kwargs): + state["calls"].append(route) + assert set(kwargs) <= {"accept"}, "the gate transport must remain read-only" + parsed = urlsplit(route) + path = parsed.path.removeprefix(f"repos/{REPO}/") + if path == "actions/workflows/native-feed-post.yml/runs": + query = parse_qs(parsed.query) + assert query.get("event") == ["push"] + assert query.get("head_sha") == [head] + return contract.json_bytes({"workflow_runs": state["native_runs"]}) + if path == "actions/workflows/ci-post.yml/runs": + query = parse_qs(parsed.query) + assert query.get("event") == ["push"] + assert query.get("head_sha") == [head] + return contract.json_bytes({"workflow_runs": state["python_runs"]}) + observed_python = {row["id"]: row for row in state["python_runs"]} + # A list observation may predate a rerun. Detail reads expose the current attempt. + observed_python[python_id] = python_run + selected_python = None + for observed in observed_python.values(): + observed_id, observed_attempt = observed["id"], observed["run_attempt"] + if path == f"actions/runs/{observed_id}": + return contract.json_bytes(observed) + if path == f"actions/runs/{observed_id}/attempts/{observed_attempt}/jobs": + observed_jobs = ( + python_jobs if observed_id == python_id else state["other_python_jobs"] + ) + return contract.json_bytes({"jobs": observed_jobs}) + if path == f"actions/runs/{observed_id}/artifacts": + artifact_attempt = observed_attempt + state["python_artifact_attempt_delta"] + name = f"release-evidence-{artifact_attempt}" + state["artifact_observations"].append((observed_id, observed_attempt, name)) + missing = observed_id == python_id and state["python_artifact_missing"] + return contract.json_bytes({"artifacts": [] if missing else [{ + "id": observed_id * 100, "name": name, "expired": False, + }]}) + if path == f"actions/artifacts/{observed_id * 100}/zip": + selected_python = observed + break + if selected_python is not None: + filename = "evidence.json" + payload = copy.deepcopy( + python_evidence if selected_python["id"] == python_id + else state["other_python_evidence"] + ) + if selected_python["id"] != python_id: + payload.update( + run_id=selected_python["id"], attempt=selected_python["run_attempt"], + head=selected_python["head_sha"], tested_sha=selected_python["head_sha"], + ) + network = payload.get("main_proofs", {}).get("network") + if network is not None: + network.update( + run_id=selected_python["id"], attempt=selected_python["run_attempt"], + head=selected_python["head_sha"], tested_sha=selected_python["head_sha"], + ) + else: + # Serve a fully usable older green proof too: an erroneous fallback + # must be caught by the verdict/call assertions, not by absent mock setup. + observed_runs = {row["id"]: row for row in state["native_runs"]} + observed_runs[native_run["id"]] = native_run + selected = None + for observed in observed_runs.values(): + native_id = observed["id"] + native_attempt = observed["run_attempt"] + if path == f"actions/runs/{native_id}": + return contract.json_bytes(observed) + if path == f"actions/runs/{native_id}/attempts/{native_attempt}/jobs": + observed_jobs = ( + native_jobs if native_id == native_run["id"] + else state["other_native_jobs"] + ) + return contract.json_bytes({"jobs": observed_jobs}) + if path == f"actions/runs/{native_id}/artifacts": + missing = ( + native_id == native_run["id"] and state["native_artifact_missing"] + ) + artifacts = [] if missing else [{ + "id": native_id * 100, + "name": ( + "native-evidence-" + f"{native_attempt + state['native_artifact_attempt_delta']}" + ), + "expired": False, + }] + state["artifact_observations"].append(( + native_id, native_attempt, + "native-evidence-" + f"{native_attempt + state['native_artifact_attempt_delta']}", + )) + return contract.json_bytes({"artifacts": artifacts}) + if path == f"actions/artifacts/{native_id * 100}/zip": + selected = observed + break + if selected is None: + raise AssertionError(f"unexpected gate transport read: {route}") + payload = copy.deepcopy( + attestation if selected["id"] == native_run["id"] + else state["other_attestation"] + ) + if selected["id"] != native_run["id"]: + payload.update( + run_id=selected["id"], attempt=selected["run_attempt"], + head=selected["head_sha"], tested_sha=selected["head_sha"], + ) + filename = "native-evidence.json" + archive = io.BytesIO() + with zipfile.ZipFile(archive, "w") as zipped: + raw = state["artifact_raw"].get(filename, contract.json_bytes(payload)) + zipped.writestr(filename, raw) + return archive.getvalue() + + monkeypatch.setattr(evidence, "gh_bytes", gh_bytes) + return state + + +def assert_native_discovery(state): + assert any( + "/actions/workflows/native-feed-post.yml/runs?" in route + for route in state["calls"] + ), "the real gate never requested native workflow observations" + + +@pytest.fixture +def observed_main_parser(main_proof_transport, monkeypatch): + """Record inputs while executing the original JSON and proof validators.""" + state = main_proof_transport + state.update( + parsed_bytes=[], parsed_objects=[], gate_inputs=[], main_inputs=[], main_validations=[], + waiting_inputs=[], waiting_validations=[], + ) + real_object_json = evidence.object_json + real_validate_gate = evidence.validate_gate + real_validate_main = evidence.validate_main + real_validate_waiting_python = evidence._validate_waiting_python + + def parse(raw): + state["parsed_bytes"].append(raw) + result = real_object_json(raw) + state["parsed_objects"].append(copy.deepcopy(result)) + return result + + def validate_gate(proof, run, jobs, **kwargs): + state["gate_inputs"].append(copy.deepcopy((proof, run, jobs, kwargs))) + return real_validate_gate(proof, run, jobs, **kwargs) + + def validate_main(proof, run, jobs): + state["main_inputs"].append(copy.deepcopy((proof, run, jobs))) + result = real_validate_main(proof, run, jobs) + state["main_validations"].append(copy.deepcopy(proof)) + return result + + def validate_waiting_python(proof, run, jobs, **kwargs): + state["waiting_inputs"].append(copy.deepcopy((proof, run, jobs, kwargs))) + result = real_validate_waiting_python(proof, run, jobs, **kwargs) + state["waiting_validations"].append(copy.deepcopy(proof)) + return result + + monkeypatch.setattr(evidence, "object_json", parse) + monkeypatch.setattr(evidence, "validate_gate", validate_gate) + monkeypatch.setattr(evidence, "validate_main", validate_main) + monkeypatch.setattr(evidence, "_validate_waiting_python", validate_waiting_python) + return state + + +@pytest.fixture +def incident_transport(monkeypatch): + """Real incident/inventory logic, with mutable issue observations at the API.""" + state = {"issues": [], "reads": [], "writes": [], "comments": []} + monkeypatch.setenv("GITHUB_RUN_ID", "930") + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "2") + + def api(repo, endpoint, method="GET", data=None): + assert repo == REPO + if method == "GET": + state["reads"].append(endpoint) + assert data is None + assert endpoint == ( + "issues?state=all&sort=updated&direction=desc&per_page=100&page=1" + ) + return copy.deepcopy(state["issues"]) + state["writes"].append((endpoint, method, copy.deepcopy(data))) + if endpoint == "issues" and method == "POST": + issue = {"number": 41, "state": "open", **copy.deepcopy(data)} + state["issues"].append(issue) + return copy.deepcopy(issue) + assert len(state["issues"]) == 1 + issue = state["issues"][0] + if endpoint == f"issues/{issue['number']}/comments" and method == "POST": + state["comments"].append(copy.deepcopy(data)) + return {"id": len(state["comments"]), **data} + assert endpoint == f"issues/{issue['number']}" and method == "PATCH" + issue.update(copy.deepcopy(data)) + return copy.deepcopy(issue) + + # inventory's original implementation resolves automation.api at call time. + monkeypatch.setattr(automation, "api", api) + monkeypatch.setattr(reporting, "api", api) + return state + + +def collect_callback_without_incident(state, incidents): + directory = state["callback_root"] / "downloaded-stages" + assert list(directory.glob("release-stage-*/release-receipt.json")) == [] + report = reporting.collect(directory, failed=False, stage="prepare-publication") + assert report["state"] == "pending" and report["event_id"] is None + assert report["pr"] is report["ci"] is report["deployed"] is None + reporting.incident(REPO, report, "release-owner", "prepare-publication") + assert incidents["reads"] == incidents["writes"] == [] + return report + + +def test_main_gate_consumes_matching_native_transport_positive_twin(main_proof_transport): + state = main_proof_transport + run = state["python_run"] + native = state["native_run"] + # An older green match and higher-id unrelated runs cannot replace the selected pair. + older = {**native, "id": native["id"] - 1, "run_attempt": 1} + unrelated = [ + {**native, "id": native["id"] + 1, "head_sha": "e" * 40}, + {**native, "id": native["id"] + 2, "head_branch": "topic"}, + {**native, "id": native["id"] + 3, "event": "pull_request"}, + {**native, "id": native["id"] + 4, "path": ".github/workflows/native-feed.yml"}, + ] + state["native_runs"] = [older, *unrelated, native] + if run["run_attempt"] % 2 == 0: + state["native_runs"].reverse() + result = publishing.gate_for_run(REPO, run["id"], head=run["head_sha"]) + assert_native_discovery(state) + prefix = f"repos/{REPO}/actions" + assert f"{prefix}/runs/{run['id']}" in state["calls"] + assert ( + f"{prefix}/runs/{run['id']}/attempts/{run['run_attempt']}/jobs?per_page=100" + in state["calls"] + ) + assert f"{prefix}/runs/{run['id']}/artifacts?per_page=100" in state["calls"] + assert f"{prefix}/artifacts/{run['id'] * 100}/zip" in state["calls"] + assert ( + f"{prefix}/runs/{native['id']}/attempts/{native['run_attempt']}/jobs?per_page=100" + in state["calls"] + ) + assert f"{prefix}/runs/{native['id']}/artifacts?per_page=100" in state["calls"] + assert f"{prefix}/artifacts/{native['id'] * 100}/zip" in state["calls"] + assert not any(f"/runs/{older['id']}/" in route for route in state["calls"]) + assert set(result.get("main_proofs", {})) == {"native", "network"} + assert result["main_proofs"]["native"] == { + "workflow_path": native["path"], "run": native, + "jobs": state["native_jobs"], "attestation": state["attestation"], + } + assert result["main_proofs"]["network"] == state["python_evidence"]["main_proofs"]["network"] + + +@pytest.mark.parametrize( + "fault", + [ + "missing-run", "wrong-head", "failed-run", "missing-artifact", "wrong-tree", + "wrong-attempt", "old-attempt-artifact", "native-job-red", "aggregate-red", + "network-disabled", "network-skipped", "network-job-red", "network-job-missing", + "network-job-duplicate", + ], +) +def test_main_gate_refuses_incomplete_observed_proofs(main_proof_transport, fault): + state = main_proof_transport + run = state["python_run"] + if fault == "missing-run": + state["native_runs"] = [] + elif fault == "wrong-head": + state["native_run"]["head_sha"] = "e" * 40 + elif fault == "failed-run": + state["native_run"]["conclusion"] = "failure" + elif fault == "missing-artifact": + state["native_artifact_missing"] = True + elif fault == "wrong-tree": + state["attestation"]["tree"] = "e" * 40 + elif fault == "wrong-attempt": + state["attestation"]["attempt"] -= 1 + elif fault == "old-attempt-artifact": + state["native_artifact_attempt_delta"] = -1 + elif fault in {"native-job-red", "aggregate-red"}: + name = "TSan" if fault == "native-job-red" else "Native proof" + next(job for job in state["native_jobs"] if job["name"] == name)["conclusion"] = "failure" + elif fault in {"network-job-red", "network-job-missing", "network-job-duplicate"}: + job = next( + job for job in state["python_jobs"] if job["name"] == state["network_name"] + ) + if fault == "network-job-missing": + state["python_jobs"].remove(job) + elif fault == "network-job-duplicate": + state["python_jobs"].append(copy.deepcopy(job)) + else: + job["conclusion"] = "skipped" + else: + network = state["python_evidence"]["main_proofs"]["network"] + network["executed"] = [] + if fault == "network-skipped": + network["skipped"] = [network["node_id"]] + with pytest.raises(evidence.GateRejected): + publishing.gate_for_run(REPO, run["id"], head=run["head_sha"]) + + +@pytest.mark.parametrize( + "fault", + ["in_progress", "failure", "cancelled", "missing-artifact", "job-failure", "wrong-tree"], +) +def test_main_gate_never_falls_back_from_newer_bad_native_proof(main_proof_transport, fault): + state = main_proof_transport + run, native = state["python_run"], state["native_run"] + older = {**native, "id": native["id"] - 1, "run_attempt": 1} + state["native_runs"] = [older, native] + if fault == "in_progress": + native.update(status="in_progress", conclusion=None) + elif fault == "missing-artifact": + state["native_artifact_missing"] = True + elif fault == "job-failure": + next(job for job in state["native_jobs"] if job["name"] == "TSan")[ + "conclusion" + ] = "failure" + elif fault == "wrong-tree": + state["attestation"]["tree"] = "e" * 40 + else: + native["conclusion"] = fault + with pytest.raises(evidence.GateRejected): + publishing.gate_for_run(REPO, run["id"], head=run["head_sha"]) + assert_native_discovery(state) + assert not any( + f"/runs/{older['id']}/" in route or f"/artifacts/{older['id'] * 100}/" in route + for route in state["calls"] + ), "an older successful native run must never substitute for the newest matching run" + + +def test_pr_gate_keeps_its_existing_contract_without_main_discovery(main_proof_transport): + state = main_proof_transport + gate, run = state["python_evidence"], state["python_run"] + gate.update(event="pull_request", pr=7, base=BASE) + gate.pop("main_proofs") + gate["jobs"].pop(state["network_name"], None) + state["python_jobs"][:] = [ + job for job in state["python_jobs"] if job["name"] != state["network_name"] + ] + run.update(event="pull_request", path=".github/workflows/ci.yml") + result = publishing.gate_for_run( + REPO, run["id"], head=run["head_sha"], base=BASE, pr=7 + ) + assert result == gate + assert not any("native-feed-post.yml" in route for route in state["calls"]) + + +@pytest.fixture +def candidate_marker_transport(main_proof_transport, monkeypatch): + state = main_proof_transport + gate, run = state["python_evidence"], state["python_run"] + gate.update(event="pull_request", pr=7, base=BASE) + gate.pop("main_proofs") + state["python_jobs"][:] = [ + job for job in state["python_jobs"] if job["name"] != state["network_name"] + ] + for job in state["python_jobs"]: + job["completed_at"] = "2026-11-07T11:59:00Z" + run.update( + event="pull_request", path=".github/workflows/ci.yml", + pull_requests=[{"number": 7}], + ) + pr = { + "number": 7, "merged_at": "2026-11-07T12:00:00Z", + "head": {"sha": run["head_sha"], "repo": {"full_name": REPO}}, + "body": automation.body({ + "kind": "release", "version": "0.3.1", "base": BASE, + "generated_head": run["head_sha"], + }), + } + state.update(candidate_pr=pr, candidate_reads=[]) + + def api(repo, endpoint, method="GET", data=None): + assert repo == REPO and method == "GET" and data is None + state["candidate_reads"].append(endpoint) + if endpoint == ( + "actions/workflows/ci.yml/runs?event=pull_request&head_sha=" + + run["head_sha"] + "&per_page=100&page=1" + ): + return {"workflow_runs": copy.deepcopy(state["python_runs"])} + assert endpoint == ( + f"actions/runs/{run['id']}/attempts/{run['run_attempt']}/jobs" + "?per_page=100&page=1" + ) + return contract.object_json(evidence.gh_bytes(f"repos/{repo}/{endpoint}")) + + monkeypatch.setattr(automation, "api", api) + return state + + +@pytest.mark.parametrize("body", [None, "", "Release notes without machine metadata"]) +def test_candidate_gate_refuses_missing_release_marker(candidate_marker_transport, body): + state = candidate_marker_transport + pr = state["candidate_pr"] + pr["body"] = body + assert automation.marker(pr) is None + with pytest.raises(ValueError, match=r"^pre-merge release marker missing$"): + publishing.candidate_gate(REPO, pr) + assert len(state["candidate_reads"]) == 1 + assert state["calls"] == state["artifact_observations"] == [] + + +def test_candidate_gate_accepts_release_marker_with_real_gate(candidate_marker_transport): + state = candidate_marker_transport + result = publishing.candidate_gate(REPO, state["candidate_pr"]) + assert result == state["python_evidence"] + assert len(state["candidate_reads"]) == 2 + run = state["python_run"] + assert f"repos/{REPO}/actions/artifacts/{run['id'] * 100}/zip" in state["calls"] + assert not any("native-feed-post.yml" in route for route in state["calls"]) + + +@pytest.mark.parametrize("missing", ["head", "run"]) +def test_candidate_gate_keeps_ci_priority_before_missing_marker( + candidate_marker_transport, missing +): + state = candidate_marker_transport + pr = state["candidate_pr"] + pr["body"] = None + if missing == "head": + pr["head"]["sha"] = None + else: + state["python_runs"] = [] + with pytest.raises(ValueError, match=r"^pre-merge CI evidence missing$"): + publishing.candidate_gate(REPO, pr) + assert len(state["candidate_reads"]) == int(missing == "run") + assert state["calls"] == [] + + +class PublicationFetchBoundary(RuntimeError): + """Stop immediately before prepare's first external git fetch.""" + + +@pytest.fixture +def main_callback_transport(main_proof_transport, tmp_path, monkeypatch): + """Real prepare/gate/validators; only observed API bytes and I/O boundaries vary.""" + state = main_proof_transport + source = state["python_run"]["head_sha"] + generated = contract.sha256(f"{source}:release-pr".encode())[:40] + event = contract.object_json((FIXTURES / "ready-event.json").read_bytes()) + pr = { + "number": 7, "merged": True, "merged_at": "2026-11-07T12:00:00Z", + "merge_commit_sha": source, "base": {"ref": "main", "sha": BASE}, + "head": {"sha": generated, "repo": {"full_name": REPO}}, + "user": {"login": "release[bot]"}, + "body": automation.body({ + "kind": "release", "version": "0.3.1", + "generated_head": generated, "base": BASE, "event_id": event["event_id"], + }), + } + state.update( + callback_pr=pr, api_calls=[], gate_attempts=[], validated_pairs=[], effects=[], + callback_root=tmp_path, output=tmp_path / "publication", + callback_report=receipts.receipt(None), + ) + state["report_before"] = copy.deepcopy(state["callback_report"]) + + def api(repo, endpoint, method="GET", data=None): + assert repo == REPO and method == "GET" and data is None + state["api_calls"].append(endpoint) + if endpoint.startswith("actions/runs/"): + return contract.object_json(evidence.gh_bytes(f"repos/{repo}/{endpoint}")) + if endpoint == f"commits/{source}/pulls?per_page=100": + return [copy.deepcopy(pr)] + if endpoint == "pulls/7": + return copy.deepcopy(pr) + raise AssertionError(f"unexpected callback API read: {endpoint}") + + real_gate = publishing.gate_for_run + + def observe_real_gate(repo, run_id, **kwargs): + state["gate_attempts"].append((repo, run_id, kwargs)) + result = real_gate(repo, run_id, **kwargs) + state["validated_pairs"].append(copy.deepcopy(result)) + return result + + def before_git_fetch(*args, **kwargs): + state["effects"].append(("git", args)) + assert not kwargs + assert args == ("fetch", "origin", source) + # This sentinel is after the actual gate, before the fetch is executed. + raise PublicationFetchBoundary("before git fetch origin exact main head") + + def reject_snapshot(*args, **kwargs): + state["effects"].append(("snapshot", args)) + raise AssertionError("prepare must not snapshot before complete proof") + + def reject_process(command, **kwargs): + state["effects"].append(("process", command)) + raise AssertionError("prepare must not download or publish before complete proof") + + def reject_publication_transport(*args, **kwargs): + state["effects"].append(("publication-transport", args)) + raise AssertionError("publication byte transport reached before the fetch boundary") + + monkeypatch.setenv("GITHUB_EVENT_NAME", "workflow_run") + monkeypatch.setenv("RELEASE_BOT_LOGIN", "release[bot]") + monkeypatch.setattr(publishing, "api", api) + monkeypatch.setattr(publishing, "gate_for_run", observe_real_gate) + monkeypatch.setattr(publishing, "git", before_git_fetch) + monkeypatch.setattr(publishing, "snapshot", reject_snapshot) + monkeypatch.setattr(publishing.subprocess, "run", reject_process) + monkeypatch.setattr(publishing, "fetch", reject_publication_transport) + monkeypatch.setattr(publishing, "gh_bytes", reject_publication_transport) + return state + + +def prepare_main_callback(state, source, *, run_id=None): + if run_id is None: + run_id = state["python_run" if source == "CI" else "native_run"]["id"] + return publishing.prepare( + state["callback_root"], REPO, run_id, state["output"], state["callback_report"] + ) + + +def assert_callback_created_nothing(state): + assert state["effects"] == [] + assert not state["output"].exists() + assert state["callback_report"] == state["report_before"] + assert state["validated_pairs"] == [] + + +def assert_callback_validated_current_pair(state, *, since=0): + python, native = state["python_run"], state["native_run"] + calls = state["calls"][since:] + result = state["validated_pairs"][-1] + assert result["run_id"] == python["id"] and result["attempt"] == python["run_attempt"] + assert result["main_proofs"]["network"] == state["python_evidence"]["main_proofs"]["network"] + assert result["main_proofs"]["native"] == { + "workflow_path": native["path"], "run": native, "jobs": state["native_jobs"], + "attestation": state["attestation"], + } + assert state["gate_attempts"][-1] == (REPO, python["id"], {"head": python["head_sha"]}) + assert "pulls/7" in state["api_calls"] + assert any("/actions/workflows/native-feed-post.yml/runs?" in route for route in calls) + prefix = f"repos/{REPO}/actions" + for run, artifact in ((python, "release-evidence"), (native, "native-evidence")): + assert ( + f"{prefix}/runs/{run['id']}/attempts/{run['run_attempt']}/jobs?per_page=100" + in calls + ) + assert f"{prefix}/artifacts/{run['id'] * 100}/zip" in calls + assert ( + run["id"], run["run_attempt"], f"{artifact}-{run['run_attempt']}" + ) in state["artifact_observations"] + assert not state["output"].exists() + assert state["callback_report"] == state["report_before"] + + +@pytest.fixture +def refreshed_release_pr(main_callback_transport, monkeypatch): + state = main_callback_transport + refreshed = copy.deepcopy(state["callback_pr"]) + real_api = publishing.api + + def api(repo, endpoint, method="GET", data=None): + observed = real_api(repo, endpoint, method, data) + return copy.deepcopy(refreshed) if endpoint == "pulls/7" else observed + + monkeypatch.setattr(publishing, "api", api) + return state, refreshed + + +def test_prepare_refuses_missing_refreshed_release_marker(refreshed_release_pr): + state, refreshed = refreshed_release_pr + refreshed["body"] = None + assert automation.marker(state["callback_pr"]) is not None + assert automation.marker(refreshed) is None + with pytest.raises(ValueError, match=r"^release PR marker missing$"): + prepare_main_callback(state, "CI") + source = state["python_run"]["head_sha"] + assert state["api_calls"] == [ + f"actions/runs/{state['python_run']['id']}", + f"commits/{source}/pulls?per_page=100", "pulls/7", + ] + assert state["calls"] == [f"repos/{REPO}/actions/runs/{state['python_run']['id']}"] + assert state["gate_attempts"] == [] + assert_callback_created_nothing(state) + + +def test_prepare_keeps_owner_priority_before_missing_marker(refreshed_release_pr): + state, refreshed = refreshed_release_pr + refreshed.update(body=None, user={"login": "other-owner"}) + with pytest.raises(ValueError, match=r"^unexpected release PR owner$"): + prepare_main_callback(state, "CI") + assert state["gate_attempts"] == [] + assert_callback_created_nothing(state) + + +def test_prepare_accepts_refreshed_release_marker(refreshed_release_pr): + state, _refreshed = refreshed_release_pr + with pytest.raises( + PublicationFetchBoundary, match=r"^before git fetch origin exact main head$" + ): + prepare_main_callback(state, "CI") + assert_callback_validated_current_pair(state) + assert state["effects"] == [("git", ("fetch", "origin", state["python_run"]["head_sha"]))] + + +@pytest.mark.parametrize("first_source", ["CI", "Native feed"]) +def test_main_callbacks_validate_the_same_pair_before_fetch(main_callback_transport, first_source): + state = main_callback_transport + second_source = "Native feed" if first_source == "CI" else "CI" + for source in (first_source, second_source): + since = len(state["calls"]) + with pytest.raises(PublicationFetchBoundary, match="before git fetch"): + prepare_main_callback(state, source) + assert_callback_validated_current_pair(state, since=since) + assert len(state["gate_attempts"]) == len(state["validated_pairs"]) == 2 + assert state["validated_pairs"][0] == state["validated_pairs"][1] + assert state["effects"] == [ + ("git", ("fetch", "origin", state["python_run"]["head_sha"])) + ] * 2 + # Two unit callbacks reach one validated pair. Publication/idempotency is not run. + + +@pytest.mark.parametrize("first_source", ["CI", "Native feed"]) +@pytest.mark.parametrize("counterpart_state", ["missing", "pending", "failure"]) +def test_main_callback_completion_order_waits_for_counterpart( + main_callback_transport, incident_transport, first_source, counterpart_state +): + state = main_callback_transport + counterpart = "native" if first_source == "CI" else "python" + run = state[f"{counterpart}_run"] + if counterpart_state == "missing": + state[f"{counterpart}_runs"] = [] + workflow = "native-feed-post.yml" if counterpart == "native" else "ci-post.yml" + reason = f"missing exact-head main {workflow} run" + else: + if counterpart_state == "pending": + run.update(status="in_progress", conclusion=None) + else: + run["conclusion"] = "failure" + reason = ( + "newest native proof has not completed successfully" if counterpart == "native" + else "CI has not completed successfully" + ) + if counterpart_state == "failure": + with pytest.raises(ValueError) as rejected: + prepare_main_callback(state, first_source) + assert str(rejected.value) == reason + else: + assert prepare_main_callback(state, first_source) == { + "eligible": "false", "waiting": "main-proofs-pending", + } + collect_callback_without_incident(state, incident_transport) + assert_callback_created_nothing(state) + if counterpart == "python" and counterpart_state == "missing": + assert state["gate_attempts"] == [] + else: + assert state["gate_attempts"] == [ + (REPO, state["python_run"]["id"], {"head": state["python_run"]["head_sha"]}) + ] + # The later counterpart completion changes the input; both real gates remain. + run.update(status="completed", conclusion="success") + state[f"{counterpart}_runs"] = [run] + second_source = "Native feed" if first_source == "CI" else "CI" + since = len(state["calls"]) + with pytest.raises(PublicationFetchBoundary, match="before git fetch"): + prepare_main_callback(state, second_source) + assert_callback_validated_current_pair(state, since=since) + assert state["effects"] == [("git", ("fetch", "origin", run["head_sha"]))] + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize("slot", ["python", "native"]) +@pytest.mark.parametrize("readiness", ["missing", "queued", "in_progress"]) +def test_main_callbacks_wait_without_incident_for_observed_pending_pair( + main_callback_transport, observed_main_parser, incident_transport, source, slot, readiness +): + state = main_callback_transport + run = state[f"{slot}_run"] + if readiness == "missing": + state[f"{slot}_runs"] = [] + else: + run.update(status=readiness, conclusion=None) + state[f"{slot}_artifact_missing"] = True + for job in state[f"{slot}_jobs"]: + job.update(status=readiness, conclusion=None) + assert prepare_main_callback(state, source) == { + "eligible": "false", "waiting": "main-proofs-pending", + } + assert_callback_created_nothing(state) + collect_callback_without_incident(state, incident_transport) + workflow = "ci-post.yml" if slot == "python" else "native-feed-post.yml" + assert any(f"/actions/workflows/{workflow}/runs?" in row for row in state["calls"]) + observed = {"workflow_runs": state[f"{slot}_runs"]} + assert observed in state["parsed_objects"], "the real parser did not consume readiness" + assert state["main_validations"] == [] + if slot == "python" and readiness == "missing": + assert state["gate_attempts"] == [] # There is no Python run ID to gate yet. + else: + python = state["python_run"] + assert state["gate_attempts"] == [(REPO, python["id"], {"head": python["head_sha"]})] + + # The same lifecycle consumes new completion input and reaches the real gate. + run.update(status="completed", conclusion="success") + state[f"{slot}_runs"] = [run] + state[f"{slot}_artifact_missing"] = False + for job in state[f"{slot}_jobs"]: + job.update(status="completed", conclusion="success") + with pytest.raises(PublicationFetchBoundary, match="before git fetch"): + prepare_main_callback(state, source) + assert_callback_validated_current_pair(state) + assert state["main_validations"][-1] == state["validated_pairs"][-1] + assert incident_transport["reads"] == incident_transport["writes"] == [] + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize("readiness", ["missing", "queued", "in_progress"]) +@pytest.mark.parametrize( + "fault,reason", + [ + ("stale-attempt", "stale CI attempt"), + ("network-non-object", "network: expected an object"), + ( + "network-job-red", + "required job 'Live PyPI hashes' missing, duplicated, failed, cancelled or skipped", + ), + ("wrong-tree", "network: wrong tree"), + ("e2e-incomplete", "incomplete E2E evidence"), + ], +) +@pytest.mark.parametrize("existing_issue", [False, True], ids=["new-issue", "existing-issue"]) +def test_main_callbacks_refuse_invalid_python_while_native_pending( + main_callback_transport, observed_main_parser, incident_transport, + source, readiness, fault, reason, existing_issue, +): + state = main_callback_transport + assert observed_main_parser is state + python, native = state["python_run"], state["native_run"] + current_python = copy.deepcopy(python) + proof = state["python_evidence"] + if readiness == "missing": + state["native_runs"] = [] + else: + native.update(status=readiness, conclusion=None) + state["native_artifact_missing"] = True + for job in state["native_jobs"]: + job.update(status=readiness, conclusion=None) + if fault == "stale-attempt": + proof["attempt"] -= 1 + elif fault == "network-non-object": + proof["main_proofs"]["network"] = [] + elif fault == "network-job-red": + next(job for job in state["python_jobs"] if job["name"] == state["network_name"])[ + "conclusion" + ] = "failure" + elif fault == "wrong-tree": + original_tree = proof["tree"] + proof["tree"] = contract.sha256(b"another completed Python evidence tree")[:40] + assert proof["tree"] != original_tree + assert proof["main_proofs"]["network"]["tree"] == original_tree + else: + assert fault == "e2e-incomplete" + for suite in ("docker", "database"): + proof["e2e"][suite].pop() + assert python == current_python + assert python["status"] == "completed" and python["conclusion"] == "success" + with pytest.raises(evidence.GateRejected) as rejected: + prepare_main_callback(state, source) + assert type(rejected.value) is evidence.GateRejected + assert str(rejected.value) == reason + assert_callback_created_nothing(state) + assert state["gate_attempts"] == [(REPO, python["id"], {"head": python["head_sha"]})] + assert_native_discovery(state) + assert {"workflow_runs": state["native_runs"]} in state["parsed_objects"] + if readiness != "missing": + assert native in state["parsed_objects"] + assert proof in state["parsed_objects"] + assert {"jobs": state["python_jobs"]} in state["parsed_objects"] + assert state["waiting_inputs"] == [ + (proof, python, state["python_jobs"], {"head": python["head_sha"], "base": None}), + ] + assert state["waiting_validations"] == state["gate_inputs"] == state["main_inputs"] == [] + assert state["artifact_observations"] == [ + (python["id"], python["run_attempt"], f"release-evidence-{python['run_attempt']}"), + ] + incidents = incident_transport + if existing_issue: + incidents["issues"].append({ + "number": 41, "state": "closed", + "body": "\nolder observation", + }) + report = reporting.collect( + state["callback_root"] / "downloaded-stages", failed=True, stage="prepare-publication" + ) + assert report["state"] == "failed" and report["event_id"] is None + assert report["attempt"] == 930 and report["sequence"] == 2 + reporting.incident(REPO, report, "release-owner", "prepare-publication") + assert len(incidents["reads"]) == 1 + assert [(route, method) for route, method, _ in incidents["writes"]] == ( + [("issues/41/comments", "POST"), ("issues/41", "PATCH")] if existing_issue else + [("issues", "POST"), ("issues/41", "PATCH")] + ) + assert len(incidents["issues"]) == 1 + issue = incidents["issues"][0] + assert issue["state"] == "open" and issue["title"] == "Data release failed: run-930" + assert "**failed**" in issue["body"] and "`prepare-publication`" in issue["body"] + assert_callback_created_nothing(state) + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize("readiness", ["missing", "queued", "in_progress"]) +def test_main_callbacks_wait_with_valid_python_while_native_pending_positive_twin( + main_callback_transport, observed_main_parser, incident_transport, source, readiness +): + state = main_callback_transport + assert observed_main_parser is state + python, native = state["python_run"], state["native_run"] + if readiness == "missing": + state["native_runs"] = [] + else: + native.update(status=readiness, conclusion=None) + state["native_artifact_missing"] = True + for job in state["native_jobs"]: + job.update(status=readiness, conclusion=None) + assert python["status"] == "completed" and python["conclusion"] == "success" + assert prepare_main_callback(state, source) == { + "eligible": "false", "waiting": "main-proofs-pending", + } + assert_callback_created_nothing(state) + collect_callback_without_incident(state, incident_transport) + assert incident_transport["issues"] == incident_transport["comments"] == [] + assert state["gate_attempts"] == [(REPO, python["id"], {"head": python["head_sha"]})] + assert_native_discovery(state) + assert {"workflow_runs": state["native_runs"]} in state["parsed_objects"] + if readiness != "missing": + assert native in state["parsed_objects"] + proof = state["python_evidence"] + assert proof in state["parsed_objects"] + assert {"jobs": state["python_jobs"]} in state["parsed_objects"] + assert state["waiting_inputs"] == [ + (proof, python, state["python_jobs"], {"head": python["head_sha"], "base": None}), + ] + assert state["waiting_validations"] == [proof] + assert state["gate_inputs"] == state["main_inputs"] == [] + assert state["artifact_observations"] == [ + (python["id"], python["run_attempt"], f"release-evidence-{python['run_attempt']}"), + ] + + +@pytest.mark.parametrize("existing_issue", [False, True], ids=["new-issue", "existing-issue"]) +def test_callback_reporting_failure_still_opens_or_updates_incident( + tmp_path, incident_transport, existing_issue +): + incidents = incident_transport + if existing_issue: + incidents["issues"].append({ + "number": 41, "state": "closed", + "body": "\nolder observation", + }) + report = reporting.collect(tmp_path, failed=True, stage="prepare-publication") + assert report["state"] == "failed" and report["event_id"] is None + assert report["attempt"] == 930 and report["sequence"] == 2 + reporting.incident(REPO, report, "release-owner", "prepare-publication") + assert len(incidents["reads"]) == 1 + assert [(route, method) for route, method, _ in incidents["writes"]] == ( + [("issues/41/comments", "POST"), ("issues/41", "PATCH")] if existing_issue else + [("issues", "POST"), ("issues/41", "PATCH")] + ) + issue = incidents["issues"][0] + assert issue["state"] == "open" + assert issue["title"] == "Data release failed: run-930" + assert "**failed**" in issue["body"] and "`prepare-publication`" in issue["body"] + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize("slot", ["python", "native"]) +@pytest.mark.parametrize( + "fault", ["failure", "cancelled", "wrong-head", "wrong-attempt", + "malformed-proof", "malformed-json", "missing-artifact"], +) +def test_main_callbacks_keep_completed_and_malformed_proofs_decisive( + main_callback_transport, observed_main_parser, incident_transport, source, slot, fault +): + state = main_callback_transport + run = state[f"{slot}_run"] + callback_id = None + if fault in {"failure", "cancelled"}: + run["conclusion"] = fault + reason = ( + "CI has not completed successfully" if slot == "python" + else "newest native proof has not completed successfully" + ) + elif fault == "wrong-head": + # The list still has the requested head; the current detail read contradicts it. + state[f"{slot}_runs"] = [copy.deepcopy(run)] + if source == ("CI" if slot == "python" else "Native feed"): + older_callback = {**run, "id": run["id"] - 1, "run_attempt": 1} + state[f"{slot}_runs"].append(older_callback) + callback_id = older_callback["id"] + run["head_sha"] = contract.sha256(b"contradictory current head")[:40] + reason = ( + "counterpart is not exact-head main CI" if slot == "python" + else "newest native proof has not completed successfully" + ) + elif fault == "wrong-attempt": + if slot == "python": + state["python_evidence"]["attempt"] -= 1 + reason = "stale CI attempt" + else: + state["attestation"]["attempt"] -= 1 + reason = "native: wrong attempt" + elif fault == "malformed-proof": + if slot == "python": + state["python_evidence"]["main_proofs"]["network"] = [] + reason = "network: expected an object" + else: + state["attestation"]["legs"] = [] + reason = "native legs: expected an object" + elif fault == "malformed-json": + filename = "evidence.json" if slot == "python" else "native-evidence.json" + state["artifact_raw"][filename] = b'{"run_id":1,"run_id":2}' + reason = "duplicate JSON key: run_id" + else: + state[f"{slot}_artifact_missing"] = True + artifact = "release-evidence" if slot == "python" else "native-evidence" + reason = f"missing unique {artifact}-{run['run_attempt']} artifact" + with pytest.raises(ValueError) as rejected: + prepare_main_callback(state, source, run_id=callback_id) + assert str(rejected.value) == reason + assert_callback_created_nothing(state) + assert state["parsed_objects"], "the real parser must observe the supplied refusal input" + if fault in {"wrong-attempt", "malformed-proof"}: + assert state["gate_inputs"], "the real validator must consume the artifact mutation" + if fault == "malformed-json": + assert b'{"run_id":1,"run_id":2}' in state["parsed_bytes"] + # A real refusal must still take the actionable failure path. + report = reporting.collect( + state["callback_root"], failed=True, stage="prepare-publication" + ) + reporting.incident(REPO, report, "release-owner", "prepare-publication") + assert [(route, method) for route, method, _ in incident_transport["writes"]] == [ + ("issues", "POST"), ("issues/41", "PATCH"), + ] + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize( + "fault", + ["native-tree", "native-attempt", "native-leg", "native-job-red", + "network-skipped", "network-job-red", "missing-native-artifact"], +) +def test_main_callbacks_refuse_invalid_proof_before_any_preparation( + main_callback_transport, source, fault +): + state = main_callback_transport + if fault == "native-tree": + state["attestation"]["tree"] = contract.sha256(b"other tree")[:40] + reason = "native: wrong tree" + elif fault == "native-attempt": + state["attestation"]["attempt"] -= 1 + reason = "native: wrong attempt" + elif fault == "native-leg": + state["attestation"]["legs"].pop("tsan") + reason = "native: missing or unexpected leg" + elif fault == "native-job-red": + next(job for job in state["native_jobs"] if job["name"] == "TSan")[ + "conclusion" + ] = "failure" + reason = "required job 'TSan' missing, duplicated, failed, cancelled or skipped" + elif fault == "network-skipped": + network = state["python_evidence"]["main_proofs"]["network"] + network.update(executed=[], skipped=[network["node_id"]]) + reason = "network executed: missing nonempty inventory" + elif fault == "network-job-red": + next(job for job in state["python_jobs"] if job["name"] == state["network_name"])[ + "conclusion" + ] = "failure" + reason = "required job 'Live PyPI hashes' missing, duplicated, failed, cancelled or skipped" + else: + state["native_artifact_missing"] = True + reason = f"missing unique native-evidence-{state['native_run']['run_attempt']} artifact" + with pytest.raises(evidence.GateRejected) as rejected: + prepare_main_callback(state, source) + assert str(rejected.value) == reason + assert state["gate_attempts"] == [ + (REPO, state["python_run"]["id"], {"head": state["python_run"]["head_sha"]}) + ] + assert_callback_created_nothing(state) + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize("fault", ["owner", "changed-pr-head", "ambiguous-marker"]) +def test_main_callbacks_keep_release_authorization_before_gate( + main_callback_transport, source, fault +): + state = main_callback_transport + pr = state["callback_pr"] + if fault == "owner": + pr["user"]["login"] = "other[bot]" + reason = "unexpected release PR owner" + elif fault == "changed-pr-head": + pr["head"]["sha"] = contract.sha256(b"changed PR head")[:40] + reason = "release PR head changed" + else: + pr["body"] += pr["body"] + reason = "ambiguous release PR marker" + with pytest.raises(ValueError) as rejected: + prepare_main_callback(state, source) + assert str(rejected.value) == reason + assert state["gate_attempts"] == [] + assert_callback_created_nothing(state) + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize( + "fault", ["pending", "failure", "cancelled", "stale-attempt", "network-red"] +) +def test_main_callbacks_never_replace_newer_bad_python_with_old_green( + main_callback_transport, source, fault +): + state = main_callback_transport + python = state["python_run"] + older = {**python, "id": python["id"] - 1, "run_attempt": 1} + state["python_runs"] = [python, older] + if fault == "pending": + python.update(status="in_progress", conclusion=None) + reason = "CI has not completed successfully" + elif fault in {"failure", "cancelled"}: + python["conclusion"] = fault + reason = "CI has not completed successfully" + elif fault == "stale-attempt": + state["python_evidence"]["attempt"] -= 1 + reason = "stale CI attempt" + else: + next(job for job in state["python_jobs"] if job["name"] == state["network_name"])[ + "conclusion" + ] = "failure" + reason = "required job 'Live PyPI hashes' missing, duplicated, failed, cancelled or skipped" + callback_id = older["id"] if source == "CI" else state["native_run"]["id"] + if fault == "pending": + assert prepare_main_callback(state, source, run_id=callback_id) == { + "eligible": "false", "waiting": "main-proofs-pending", + } + else: + with pytest.raises(evidence.GateRejected) as rejected: + prepare_main_callback(state, source, run_id=callback_id) + assert str(rejected.value) == reason + assert state["gate_attempts"] == [(REPO, python["id"], {"head": python["head_sha"]})] + assert not any( + f"/runs/{older['id']}/" in route or f"/artifacts/{older['id'] * 100}/" in route + for route in state["calls"] + ), "the old callback may be read; its jobs/artifacts cannot replace the newer Python run" + assert_callback_created_nothing(state) + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize( + "fault", ["pending", "failure", "cancelled", "missing-artifact", "wrong-tree"] +) +def test_main_callbacks_never_replace_newer_bad_native_with_old_green( + main_callback_transport, source, fault +): + state = main_callback_transport + native = state["native_run"] + older = {**native, "id": native["id"] - 1, "run_attempt": 1} + state["native_runs"] = [native, older] + if fault == "pending": + native.update(status="in_progress", conclusion=None) + reason = "newest native proof has not completed successfully" + elif fault in {"failure", "cancelled"}: + native["conclusion"] = fault + reason = "newest native proof has not completed successfully" + elif fault == "missing-artifact": + state["native_artifact_missing"] = True + reason = f"missing unique native-evidence-{native['run_attempt']} artifact" + else: + state["attestation"]["tree"] = contract.sha256(b"other native tree")[:40] + reason = "native: wrong tree" + callback_id = older["id"] if source == "Native feed" else state["python_run"]["id"] + if fault == "pending": + assert prepare_main_callback(state, source, run_id=callback_id) == { + "eligible": "false", "waiting": "main-proofs-pending", + } + else: + with pytest.raises(evidence.GateRejected) as rejected: + prepare_main_callback(state, source, run_id=callback_id) + assert str(rejected.value) == reason + assert state["gate_attempts"] == [ + (REPO, state["python_run"]["id"], {"head": state["python_run"]["head_sha"]}) + ] + assert_native_discovery(state) + assert not any( + f"/runs/{older['id']}/" in route or f"/artifacts/{older['id'] * 100}/" in route + for route in state["calls"] + ), "an old native callback cannot select old native jobs/artifacts" + assert_callback_created_nothing(state) + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +def test_main_callbacks_use_reobserved_current_attempt(main_callback_transport, source): + state = main_callback_transport + python, native = state["python_run"], state["native_run"] + state["python_runs"] = [copy.deepcopy(python)] + state["native_runs"] = [copy.deepcopy(native)] + for run in (python, native): + run["run_attempt"] += 1 + state["python_evidence"]["attempt"] = python["run_attempt"] + state["python_evidence"]["main_proofs"]["network"]["attempt"] = python["run_attempt"] + state["attestation"]["attempt"] = native["run_attempt"] + with pytest.raises(PublicationFetchBoundary, match="before git fetch"): + prepare_main_callback(state, source) + assert_callback_validated_current_pair(state) + for run in (python, native): + assert not any( + f"/runs/{run['id']}/attempts/{run['run_attempt'] - 1}/" in route + for route in state["calls"] + ) + + +@pytest.mark.parametrize("source", ["CI", "Native feed"]) +@pytest.mark.parametrize("slot", ["python", "native"]) +def test_main_callbacks_refuse_artifact_from_previous_attempt( + main_callback_transport, source, slot +): + state = main_callback_transport + state[f"{slot}_artifact_attempt_delta"] = -1 + run = state[f"{slot}_run"] + artifact = "release-evidence" if slot == "python" else "native-evidence" + with pytest.raises(evidence.GateRejected) as rejected: + prepare_main_callback(state, source) + assert str(rejected.value) == f"missing unique {artifact}-{run['run_attempt']} artifact" + assert state["gate_attempts"] == [ + (REPO, state["python_run"]["id"], {"head": state["python_run"]["head_sha"]}) + ] + assert_callback_created_nothing(state) + + +def test_main_callback_old_green_transport_is_a_usable_positive_twin(main_callback_transport): + state = main_callback_transport + python, native = state["python_run"], state["native_run"] + older_python = {**python, "id": python["id"] - 1, "run_attempt": 1} + older_native = {**native, "id": native["id"] - 1, "run_attempt": 1} + state["python_runs"], state["native_runs"] = [older_python], [older_native] + with pytest.raises(PublicationFetchBoundary, match="before git fetch"): + prepare_main_callback(state, "CI", run_id=older_python["id"]) + result = state["validated_pairs"][-1] + assert result["run_id"] == older_python["id"] and result["attempt"] == 1 + assert result["main_proofs"]["network"]["run_id"] == older_python["id"] + assert result["main_proofs"]["native"]["run"]["id"] == older_native["id"] + assert result["main_proofs"]["native"]["attestation"]["run_id"] == older_native["id"] + assert state["effects"] == [("git", ("fetch", "origin", python["head_sha"]))] + assert not state["output"].exists() + assert state["callback_report"] == state["report_before"] + + +class PublicationCandidateBoundary(RuntimeError): + """Stop before the later, independent pre-merge candidate API inventory.""" + + +@pytest.fixture +def publication_main_transport( + main_callback_transport, observed_main_parser, bootstrap_lock, tmp_path, monkeypatch +): + """Save a real full gate, then re-observe it through publication's lower I/O.""" + state = main_callback_transport + python, native = state["python_run"], state["native_run"] + source = python["head_sha"] + event = contract.object_json((FIXTURES / "ready-event.json").read_bytes()) + manifest = (FIXTURES / "release-manifest.json").read_bytes() + pr = state["callback_pr"] + # Reuse the authored package-byte inputs of test_prepare_passes_adopter_lock_for_previous. + # These controls validate proof/hash handling; they do not install or serve a package. + package_directory = tmp_path / "observed-package-bytes" + package_directory.mkdir() + for name, raw in { + "pineforge_data-0.3.1-py3-none-any.whl": b"tested wheel", + "pineforge_data-0.3.1.tar.gz": b"tested sdist", + }.items(): + (package_directory / name).write_bytes(raw) + state["python_evidence"]["distributions"] = evidence.distribution_hashes(package_directory) + native.update( + updated_at="2026-11-07T12:00:00Z", display_title="Original completed run title", + ) + for run, jobs in ((python, state["python_jobs"]), (native, state["native_jobs"])): + for index, job in enumerate(jobs, 1): + job.update( + id=run["id"] * 1000 + index, run_id=run["id"], + run_attempt=run["run_attempt"], head_sha=source, + updated_at="2026-11-07T12:00:00Z", + ) + for leg, inventory in state["attestation"]["legs"].items(): + inventory["artifact"] = f"native-leg-{leg}-{native['run_attempt']}" + inventory["registration_sha256"] = contract.sha256(contract.json_bytes({ + "tests": [{"name": name} for name in inventory["registered"]], + })) + saved = reporting.gate_for_run(REPO, python["id"], head=source) + assert set(saved["main_proofs"]) == {"network", "native"} + assert state["main_validations"][-1] == saved + staged = contract.object_json((FIXTURES / "sample-release-receipt.json").read_bytes()) + staged.update( + pr=automation.pr_identity(pr), state="deploying", + deployed=receipts.served(event, source, "0.3.1", saved["distributions"]), + ) + lock = {**copy.deepcopy(bootstrap_lock), "event": event} + source_files = { + contract.RUNTIME: ( + f'DEFAULT_RELEASE_IMAGE = (\n "{contract.image_ref(event)}"\n)\n' + ).encode(), + contract.DOCKERFILE: ( + f"ARG PINEFORGE_RELEASE_IMAGE={contract.image_ref(event)}\n" + f"ARG PINEFORGE_HUB_COMMIT={event['hub_commit']}\n" + ).encode(), + "pyproject.toml": b'[project]\nname = "pineforge-data"\nversion = "0.3.1"\n', + contract.LOCK: contract.json_bytes(lock), contract.MANIFEST: manifest, + } + state.update( + saved_gate=copy.deepcopy(saved), staged=staged, package_directory=package_directory, + publication_reads=[], publication_writes=[], fresh_gates=[], local_validations=[], + candidate_boundaries=[], downloads=[], download_files=[], + ) + real_transport = evidence.gh_bytes + real_gate = reporting.gate_for_run + real_verify_local = reporting.verify_local + real_temporary_directory = reporting.tempfile.TemporaryDirectory + + def api(repo, endpoint, method="GET", data=None): + assert repo == REPO + if method != "GET": + state["publication_writes"].append((endpoint, method, copy.deepcopy(data))) + raise AssertionError("publication must not mutate before the candidate boundary") + assert data is None + state["publication_reads"].append(endpoint) + if endpoint == "actions/runs/200": + return { + "id": 200, "run_attempt": 1, "path": ".github/workflows/release.yml", + "event": "workflow_run", "head_sha": source, + "status": "completed", "conclusion": "success", + } + if endpoint == "actions/runs/200/artifacts?per_page=100&page=1": + return {"artifacts": [{ + "id": 200001, "name": "release-stage-1-50-verify", "expired": False, + }]} + if endpoint == "pulls/7": + return copy.deepcopy(pr) + if endpoint == f"commits/{source}": + return {"parents": [{"sha": BASE}]} + if endpoint == ( + "actions/workflows/ci.yml/runs?event=pull_request&head_sha=" + + pr["head"]["sha"] + "&per_page=100&page=1" + ): + state["candidate_boundaries"].append(endpoint) + assert len(state["fresh_gates"]) == len(state["local_validations"]) == 1 + assert state["local_validations"][0] == lock + raise PublicationCandidateBoundary("before pre-merge candidate inventory read") + raise AssertionError(f"unexpected publication API read: {endpoint}") + + def gh_bytes(route, **kwargs): + if route == f"repos/{REPO}/actions/runs/200/artifacts?per_page=100": + return contract.json_bytes({"artifacts": [{ + "id": 200001, "name": "release-stage-1-50-verify", "expired": False, + }]}) + if route == f"repos/{REPO}/actions/artifacts/200001/zip": + archive = io.BytesIO() + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr("release-receipt.json", contract.json_bytes(staged)) + return archive.getvalue() + return real_transport(route, **kwargs) + + def hub_bytes(route, *, accept="application/vnd.github+json"): + state["publication_reads"].append(route) + if route == f"repos/{contract.HUB}/releases/tags/v{event['release_version']}": + assert accept == "application/vnd.github+json" + return contract.json_bytes({ + "draft": False, "prerelease": False, + "assets": [{"id": 81, "name": "release-manifest.json"}], + }) + assert route == f"repos/{contract.HUB}/releases/assets/81" + assert accept == "application/octet-stream" + return manifest + + def download(command, *, check): + assert check is True + bundle = Path(command[-1]) + assert command == [ + "gh", "run", "download", "200", "--repo", REPO, + "--name", "validated-release-bundle-1", "--dir", str(bundle), + ] + paths = [bundle / "evidence.json", *(bundle / "source" / name for name in source_files)] + for path in paths: + if ( + path.name.endswith(".env") + or not path.resolve().is_relative_to(tmp_path.resolve()) + ): + raise AssertionError(f"refused publication fixture file: {path}") + state["download_files"].append([str(path) for path in paths]) + state["downloads"].append(copy.deepcopy(command)) + (bundle / "evidence.json").write_bytes(contract.json_bytes(state["saved_gate"])) + for name, raw in source_files.items(): + path = bundle / "source" / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(raw) + + def observe_gate(repo, run_id, **kwargs): + result = real_gate(repo, run_id, **kwargs) + state["fresh_gates"].append(copy.deepcopy(result)) + return result + + def verify_local(root): + result = real_verify_local(root) + state["local_validations"].append(copy.deepcopy(result)) + return result + + monkeypatch.setenv("GITHUB_RUN_ID", "300") + monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "1") + monkeypatch.setattr(automation, "api", api) + monkeypatch.setattr(reporting, "api", api) + monkeypatch.setattr(evidence, "gh_bytes", gh_bytes) + monkeypatch.setattr(contract, "gh_bytes", hub_bytes) + monkeypatch.setattr(reporting.subprocess, "run", download) + monkeypatch.setattr(reporting, "gate_for_run", observe_gate) + monkeypatch.setattr(reporting, "verify_local", verify_local) + monkeypatch.setattr( + reporting.tempfile, "TemporaryDirectory", + lambda: real_temporary_directory(dir=tmp_path), + ) + return state + + +@pytest.mark.parametrize( + "metadata", + ["unchanged", "native-updated-at", "native-display-title", + "native-job-updated-at", "native-job-order"], +) +def test_publication_revalidates_full_main_proofs_despite_benign_metadata( + publication_main_transport, metadata +): + state = publication_main_transport + native = state["native_run"] + if metadata == "native-updated-at": + native["updated_at"] = "2026-11-08T00:00:00Z" + elif metadata == "native-display-title": + native["display_title"] = "Updated display title for the same completed run" + elif metadata == "native-job-updated-at": + state["native_jobs"][0]["updated_at"] = "2026-11-08T00:01:00Z" + elif metadata == "native-job-order": + state["native_jobs"].reverse() + report = receipts.receipt(None) + with pytest.raises(PublicationCandidateBoundary, match="before pre-merge candidate inventory"): + reporting.publication(REPO, 200, report) + fresh = state["fresh_gates"][0] + assert fresh["main_proofs"]["native"]["run"] == native + assert fresh["main_proofs"]["native"]["jobs"] == state["native_jobs"] + assert fresh == state["main_validations"][-1] + assert native in state["parsed_objects"] + assert {"jobs": state["native_jobs"]} in state["parsed_objects"] + assert state["python_evidence"] in state["parsed_objects"] + assert fresh["distributions"] == evidence.distribution_hashes(state["package_directory"]) + assert (fresh == state["saved_gate"]) is (metadata == "unchanged") + assert len(state["candidate_boundaries"]) == len(state["downloads"]) == 1 + assert len(state["download_files"][0]) == 6 + assert report["state"] == "deploying" and state["publication_writes"] == [] + # Partial lifecycle: real main proof and local metadata validation precede the sentinel. + # Pre-merge candidate validation, OCI/PyPI/previous identity and finalization do not run. + + +def test_publication_refuses_missing_release_marker(publication_main_transport): + state = publication_main_transport + state["callback_pr"]["body"] = None + assert automation.marker(state["callback_pr"]) is None + report = receipts.receipt(None) + with pytest.raises(ValueError, match=r"^publication PR identity moved$"): + reporting.publication(REPO, 200, report) + assert state["publication_reads"][-1] == "pulls/7" + assert state["fresh_gates"] == state["local_validations"] == [] + assert state["downloads"] == state["download_files"] == [] + assert state["candidate_boundaries"] == state["publication_writes"] == [] + assert report["state"] == "deploying" + + +def test_publication_accepts_release_marker_before_candidate_boundary( + publication_main_transport, +): + state = publication_main_transport + report = receipts.receipt(None) + with pytest.raises(PublicationCandidateBoundary, match="before pre-merge candidate inventory"): + reporting.publication(REPO, 200, report) + assert len(state["fresh_gates"]) == len(state["local_validations"]) == 1 + assert len(state["downloads"]) == len(state["candidate_boundaries"]) == 1 + assert state["publication_writes"] == [] + assert report["state"] == "deploying" + + +@pytest.mark.parametrize( + "fault,reason,gate_completes", + [ + ("native-id", "landed CI distributions/evidence moved", True), + ("native-attempt", "landed CI distributions/evidence moved", True), + ("tree", "landed CI distributions/evidence moved", True), + ("native-job-id", "landed CI distributions/evidence moved", True), + ("native-inventory", "landed CI distributions/evidence moved", True), + ("native-junit-digest", "landed CI distributions/evidence moved", True), + ("native-registration-digest", "landed CI distributions/evidence moved", True), + ("network-junit-digest", "landed CI distributions/evidence moved", True), + ("distribution", "landed CI distributions/evidence moved", True), + ("native-head", "newest native proof has not completed successfully", False), + ("native-workflow", "newest native proof has not completed successfully", False), + ("native-event", "newest native proof has not completed successfully", False), + ("native-conclusion", "newest native proof has not completed successfully", False), + ("native-tested-sha", "native: wrong tested_sha", False), + ("native-stale-attestation", "native: wrong attempt", False), + ("native-job-conclusion", + "required job 'TSan' missing, duplicated, failed, cancelled or skipped", False), + ("network-inventory", "network: missing unique required live-PyPI row", False), + ], +) +def test_publication_refuses_real_main_proof_identity_or_content_change( + publication_main_transport, fault, reason, gate_completes +): + state = publication_main_transport + native, attestation = state["native_run"], state["attestation"] + network = state["python_evidence"]["main_proofs"]["network"] + leg = attestation["legs"]["tsan"] + if fault in {"native-id", "native-attempt"}: + field = "id" if fault == "native-id" else "run_attempt" + native[field] += 1 + attestation.update(run_id=native["id"], attempt=native["run_attempt"]) + for job in state["native_jobs"]: + job.update(run_id=native["id"], run_attempt=native["run_attempt"]) + for name, inventory in attestation["legs"].items(): + inventory["artifact"] = f"native-leg-{name}-{native['run_attempt']}" + elif fault == "tree": + tree = contract.sha256(b"different landed tree")[:40] + state["python_evidence"]["tree"] = network["tree"] = attestation["tree"] = tree + elif fault == "native-job-id": + state["native_jobs"][0]["id"] += 1 + elif fault == "native-inventory": + leg["registered"].append("additional-native-row") + leg["executed"].append("additional-native-row") + junit = ElementTree.Element("testsuite") + for name in leg["executed"]: + ElementTree.SubElement(junit, "testcase", name=name) + leg["junit_sha256"] = contract.sha256(ElementTree.tostring(junit)) + leg["registration_sha256"] = contract.sha256(contract.json_bytes({ + "tests": [{"name": name} for name in leg["registered"]], + })) + elif fault in {"native-junit-digest", "native-registration-digest", "network-junit-digest"}: + target = network if fault == "network-junit-digest" else leg + field = "registration_sha256" if fault == "native-registration-digest" else "junit_sha256" + target[field] = contract.sha256(f"different report bytes: {fault}".encode()) + elif fault == "distribution": + wheel = state["package_directory"] / "pineforge_data-0.3.1-py3-none-any.whl" + wheel.write_bytes(wheel.read_bytes() + b"changed") + state["python_evidence"]["distributions"] = evidence.distribution_hashes( + state["package_directory"] + ) + elif fault in {"native-head", "native-workflow", "native-event"}: + state["native_runs"] = [copy.deepcopy(native)] + field, value = { + "native-head": ("head_sha", contract.sha256(b"different head")[:40]), + "native-workflow": ("path", ".github/workflows/native-feed.yml"), + "native-event": ("event", "workflow_dispatch"), + }[fault] + native[field] = value + elif fault == "native-conclusion": + native["conclusion"] = "failure" + elif fault == "native-tested-sha": + attestation["tested_sha"] = contract.sha256(b"different tested source")[:40] + elif fault == "native-stale-attestation": + attestation["attempt"] -= 1 + elif fault == "native-job-conclusion": + next(job for job in state["native_jobs"] if job["name"] == "TSan")[ + "conclusion" + ] = "failure" + else: + network["registered"].append("unrelated-network-row") + network["executed"].append("unrelated-network-row") + report = receipts.receipt(None) + with pytest.raises(ValueError) as rejected: + reporting.publication(REPO, 200, report) + assert str(rejected.value) == reason + assert len(state["fresh_gates"]) == int(gate_completes) + if gate_completes: + assert state["fresh_gates"][0] == state["main_validations"][-1] + assert state["fresh_gates"][0] != state["saved_gate"] + assert state["local_validations"] == state["candidate_boundaries"] == [] + assert len(state["downloads"]) == 1 and state["publication_writes"] == [] + assert report["state"] == "deploying" + + +@pytest.fixture +def retained_receipt_transport(tmp_path, monkeypatch): + report = contract.object_json((FIXTURES / "sample-release-receipt.json").read_bytes()) + pr = { + "number": report["pr"]["number"], "merged": True, "merge_commit_sha": MERGE, + "base": {"ref": "main", "sha": BASE}, + "head": {"sha": HEAD, "repo": {"full_name": REPO}}, + "body": automation.body({ + "kind": "release", "version": "0.3.1", "base": BASE, + "generated_head": HEAD, "event_id": report["event_id"], + }), + } + state = { + "report": report, "pr": pr, "saved": True, + "reads": [], "asset_reads": [], "uploads": [], "upload_files": [], + "temporary_directories": [], + } + + def api(repo, endpoint, method="GET", data=None): + state["reads"].append(endpoint) + assert repo == REPO and method == "GET" and data is None + if endpoint == "pulls/7": + return copy.deepcopy(pr) + assert endpoint == "releases?per_page=100&page=1" + return [{ + "tag_name": "v0.3.1", + "assets": [{"id": 98, "name": "release-receipt.json"}] if state["saved"] else [], + }] + + def gh_bytes(endpoint, *, accept): + state["asset_reads"].append(endpoint) + assert endpoint == f"repos/{REPO}/releases/assets/98" + assert accept == "application/octet-stream" + return receipts.canonical(report) + + def upload(command, *, check): + path = Path(command[4]) + if ( + path.name.endswith(".env") + or not path.resolve().is_relative_to(tmp_path.resolve()) + ): + raise AssertionError(f"refused retained receipt fixture file: {path}") + assert check is True + assert command == [ + "gh", "release", "upload", "v0.3.1", str(path), "--repo", REPO, + ] + state["upload_files"].append([path.name]) + state["uploads"].append(path.read_bytes()) + + real_temporary_directory = reporting.tempfile.TemporaryDirectory + + def temporary_directory(): + state["temporary_directories"].append(tmp_path) + return real_temporary_directory(dir=tmp_path) + + monkeypatch.setattr(automation, "api", api) + monkeypatch.setattr(reporting, "api", api) + monkeypatch.setattr(publishing, "gh_bytes", gh_bytes) + monkeypatch.setattr(reporting.subprocess, "run", upload) + monkeypatch.setattr(reporting.tempfile, "TemporaryDirectory", temporary_directory) + return state + + +def test_retain_refuses_missing_release_marker(retained_receipt_transport): + state = retained_receipt_transport + state["pr"]["body"] = None + report = state["report"] + before = copy.deepcopy(report) + assert automation.marker(state["pr"]) is None + with pytest.raises(ValueError, match=r"^retained release marker missing$"): + reporting.retain(REPO, report) + assert state["reads"] == ["pulls/7"] + assert state["asset_reads"] == state["uploads"] == state["upload_files"] == [] + assert state["temporary_directories"] == [] + assert report == before + + +@pytest.mark.parametrize("saved", [True, False]) +def test_retain_accepts_release_marker_and_preserves_receipt(retained_receipt_transport, saved): + state = retained_receipt_transport + state["saved"] = saved + report = state["report"] + before = copy.deepcopy(report) + reporting.retain(REPO, report) + assert state["reads"] == ["pulls/7", "releases?per_page=100&page=1"] + assert state["asset_reads"] == ([f"repos/{REPO}/releases/assets/98"] if saved else []) + assert state["upload_files"] == ([] if saved else [["release-receipt.json"]]) + assert state["uploads"] == ([] if saved else [receipts.canonical(report)]) + assert len(state["temporary_directories"]) == int(not saved) + assert report == before + + +@pytest.mark.parametrize("status", ["pending", "testing", "merged", "deploying", "failed"]) +def test_retain_other_states_do_not_require_a_marker(retained_receipt_transport, status): + state = retained_receipt_transport + state["pr"]["body"] = None + state["report"]["state"] = status + before = copy.deepcopy(state["report"]) + reporting.retain(REPO, state["report"]) + assert state["reads"] == state["asset_reads"] == state["uploads"] == state["upload_files"] == [] + assert state["temporary_directories"] == [] + assert state["report"] == before + + class ObservedGitHub: def __init__(self): self.writes = [] diff --git a/tests/test_main_proofs.py b/tests/test_main_proofs.py new file mode 100644 index 0000000..a6ef0df --- /dev/null +++ b/tests/test_main_proofs.py @@ -0,0 +1,892 @@ +"""Authored report fixtures; these controls do not execute CTest or live PyPI.""" + +from __future__ import annotations + +import copy +import hashlib +import json +import subprocess +from pathlib import Path +from xml.etree import ElementTree + +import pytest +from release_chain import contract, evidence, main_proofs + +ROOT = Path(__file__).resolve().parents[1] +NETWORK_NODE = ( + "tests/test_consumer_receipt.py::test_bootstrap_recorded_hashes_match_live_pypi" +) +NATIVE_LEGS = ("release-amd64", "release-arm64", "release-macos", "asan-ubsan", "tsan") +NATIVE_WORKFLOW = ".github/workflows/native-feed-post.yml" +# Required subsets declared independently from the validators, from native/feed/CMakeLists.txt. +COMMON_NATIVE_ROWS = ("feed-unit", "feed-export-unit", "feed-mock-export", "feed-mock") +REQUIRED_NATIVE_ROWS = { + leg: COMMON_NATIVE_ROWS + (("feed-tsan-seeded-race",) if leg == "tsan" else ()) + for leg in NATIVE_LEGS +} +REQUIRED_NATIVE_OMISSIONS = [ + pytest.param(leg, row, id=f"{leg}-without-{row}") + for leg, rows in REQUIRED_NATIVE_ROWS.items() + for row in rows +] +NATIVE_JOB_NAMES = ( + "Release (ubuntu-24.04)", "Release (ubuntu-24.04-arm)", "Release (macos-15)", + "ASan + UBSan", "TSan", "Native proof", +) + + +def native_report_names(leg: str) -> list[str]: + # Distinct extra rows keep cross-leg report substitution observable. + return [*REQUIRED_NATIVE_ROWS[leg], f"{leg}-unit", f"{leg}-integration"] + + +def write_junit(path: Path, names: list[str], *, network: bool = False) -> bytes: + """Minimal pytest/CTest-shaped XML, with counters tied to actual fixture rows.""" + root = ElementTree.Element("testsuites", tests=str(len(names))) + suite = ElementTree.SubElement( + root, "testsuite", tests=str(len(names)), failures="0", errors="0", skipped="0" + ) + for name in names: + attributes = {"name": name} + if network: + attributes["classname"] = "tests.test_consumer_receipt" + ElementTree.SubElement(suite, "testcase", **attributes) + raw = ElementTree.tostring(root) + path.write_bytes(raw) + return raw + + +def write_registration(path: Path, names: list[str]) -> bytes: + raw = json.dumps( + {"kind": "ctestInfo", "version": {"major": 1, "minor": 0}, + "tests": [{"name": name} for name in names]}, + sort_keys=True, + ).encode() + path.write_bytes(raw) + return raw + + +def expected_inventory(names: list[str], raw: bytes) -> dict: + return { + "registered": names, "executed": list(names), + "skipped": [], "failures": [], "errors": [], + "junit_sha256": hashlib.sha256(raw).hexdigest(), + } + + +def expected_native_inventory(directory: Path, names: list[str]) -> dict: + return { + **expected_inventory(names, (directory / "native-junit.xml").read_bytes()), + "registration_sha256": hashlib.sha256( + (directory / "native-registration.json").read_bytes() + ).hexdigest(), + } + + +def write_native_report_pair(directory: Path, names: list[str]) -> dict: + registered_raw = write_registration(directory / "native-registration.json", names) + junit_raw = write_junit(directory / "native-junit.xml", names) + return { + **expected_inventory(names, junit_raw), + "registration_sha256": hashlib.sha256(registered_raw).hexdigest(), + } + + +def record_native_leg( + directory: Path, identity: dict, leg: str, inventory: dict +) -> None: + """Record authored report content, without asking a producer to accept it.""" + (directory / "native-leg.json").write_bytes(contract.json_bytes({ + **identity, "leg": leg, "inventory": inventory, + })) + + +@pytest.mark.parametrize("network", [False, True], ids=["ctest", "pytest-network"]) +@pytest.mark.parametrize("status", [None, "run", "passed"]) +def test_junit_inventory_uses_actual_rows_and_raw_digest(tmp_path, network, status): + names = ( + ["test_bootstrap_recorded_hashes_match_live_pypi"] if network + else ["native-unit-one", "native-unit-two"] + ) + path = tmp_path / "report.xml" + write_junit(path, names, network=network) + root = ElementTree.fromstring(path.read_bytes()) + if status is not None: + for case in root.iter("testcase"): + case.set("status", status) + raw = ElementTree.tostring(root) + path.write_bytes(raw) + actual_names = [NETWORK_NODE] if network else names + assert main_proofs.junit_inventory(path, network=network) == expected_inventory( + actual_names, raw + ) + # An input-dependent second report prevents a constant inventory/digest twin. + changed_names = names[:1] if not network else [names[0] + "_different"] + changed_raw = write_junit(path, changed_names, network=network) + changed = main_proofs.junit_inventory(path, network=network) + expected_names = [NETWORK_NODE + "_different"] if network else changed_names + assert changed == expected_inventory(expected_names, changed_raw) + assert changed["junit_sha256"] != hashlib.sha256(raw).hexdigest() + + +@pytest.mark.parametrize( + "fault,reason", + [ + ("empty", "missing nonempty inventory"), + ("empty-name", "missing nonempty inventory"), + ("duplicate", "duplicate inventory"), + ("skipped", "registration/execution disagree|required rows skipped"), + ("failure", "required rows failures"), + ("error", "required rows errors"), + ("notrun", "registration/execution disagree|required rows skipped"), + ("disabled", "registration/execution disagree|required rows skipped"), + ("unknown-status", "unsuccessful or unknown testcase status"), + ("omitted-row", "suite count disagrees"), + ("suite-overcount", "suite count disagrees"), + ("suite-undercount", "suite count disagrees"), + ("parent-overcount", "suite count disagrees"), + ("suite-skipped", "suite skipped"), + ("suite-failures", "suite failures"), + ("suite-errors", "suite errors"), + ("suite-disabled", "suite disabled"), + ("collection-error", "suite errors"), + ], +) +def test_junit_inventory_refuses_non_full_reports(tmp_path, fault, reason): + path = tmp_path / "report.xml" + names = [] if fault == "empty" else ["native-one", "native-two"] + write_junit(path, names) + root = ElementTree.fromstring(path.read_bytes()) + suite = root.find("testsuite") + cases = list(root.iter("testcase")) + if fault == "empty-name": + cases[0].set("name", "") + elif fault == "duplicate": + cases[1].set("name", cases[0].get("name")) + elif fault in {"skipped", "failure", "error"}: + ElementTree.SubElement(cases[0], fault, message="authored unsuccessful row") + elif fault in {"notrun", "disabled"}: + cases[0].set("status", fault) + elif fault == "unknown-status": + cases[0].set("status", "failed") + elif fault == "omitted-row": + suite.remove(cases[-1]) + elif fault in {"suite-overcount", "suite-undercount"}: + suite.set("tests", "3" if fault == "suite-overcount" else "1") + elif fault == "parent-overcount": + root.set("tests", "3") + elif fault.startswith("suite-"): + suite.set(fault.removeprefix("suite-"), "1") + elif fault == "collection-error": + suite.set("errors", "1") + ElementTree.SubElement(suite, "error", message="collection failed") + path.write_bytes(ElementTree.tostring(root)) + with pytest.raises(ValueError, match=reason): + main_proofs.junit_inventory(path) + + +@pytest.mark.parametrize("parent", ["testsuite", "testsuites"]) +@pytest.mark.parametrize("tag", ["error", "failure", "skipped"]) +def test_junit_inventory_refuses_suite_error_even_when_counter_claims_zero( + tmp_path, parent, tag +): + path = tmp_path / "collection.xml" + write_junit(path, ["collected-before-error"]) + root = ElementTree.fromstring(path.read_bytes()) + container = root if parent == "testsuites" else root.find("testsuite") + ElementTree.SubElement(container, tag, message="authored suite problem") + path.write_bytes(ElementTree.tostring(root)) + # A collection error is not made successful by an inconsistent zero counter. + with pytest.raises(ValueError) as rejected: + main_proofs.junit_inventory(path) + assert str(rejected.value) == f"{path}: suite contains {tag} element" + + +@pytest.mark.parametrize("parent", ["testsuite", "testsuites"]) +@pytest.mark.parametrize("content", ["system-out", "system-err", "text"]) +@pytest.mark.parametrize( + "names", [["unit-alpha"], ["unit-alpha", "unit-beta"]], ids=["one-row", "two-rows"] +) +def test_junit_inventory_accepts_harmless_suite_logs_and_text( + tmp_path, parent, content, names +): + path = tmp_path / "logs.xml" + original = write_junit(path, names) + root = ElementTree.fromstring(original) + container = root if parent == "testsuites" else root.find("testsuite") + message = "error/failure/skipped are harmless words in captured output" + if content == "text": + container.text = message + else: + ElementTree.SubElement(container, content).text = message + raw = ElementTree.tostring(root) + path.write_bytes(raw) + assert main_proofs.junit_inventory(path) == expected_inventory(names, raw) + assert raw != original + + +@pytest.mark.parametrize("raw", [b"", b"", b"not XML"]) +def test_junit_inventory_refuses_malformed_xml(tmp_path, raw): + path = tmp_path / "malformed.xml" + path.write_bytes(raw) + with pytest.raises(ElementTree.ParseError): + main_proofs.junit_inventory(path) + + +def test_junit_inventory_requires_a_report_and_junit_root(tmp_path): + # File absence and malformed syntax are I/O/parser refusals, not semantic results. + with pytest.raises(FileNotFoundError): + main_proofs.junit_inventory(tmp_path / "absent.xml") + path = tmp_path / "wrong-root.xml" + path.write_bytes(b'') + with pytest.raises(ValueError, match="not a JUnit report"): + main_proofs.junit_inventory(path) + + +@pytest.mark.parametrize("names", [["unit-alpha"], ["unit-alpha", "unit-beta"]]) +def test_native_inventory_consumes_registration_and_junit(tmp_path, names): + registration, junit = tmp_path / "registration.json", tmp_path / "junit.xml" + registered_raw = write_registration(registration, list(reversed(names))) + junit_raw = write_junit(junit, names) + expected = expected_inventory(names, junit_raw) + expected["registered"] = list(reversed(names)) + expected["registration_sha256"] = hashlib.sha256(registered_raw).hexdigest() + assert main_proofs.native_inventory(registration, junit) == expected + + +@pytest.mark.parametrize( + "fault,reason", + [ + ("empty-registration", "missing nonempty inventory"), + ("duplicate-registration", "duplicate inventory"), + ("omitted-registration", "registration/execution disagree"), + ("omitted-execution", "registration/execution disagree"), + ("extra-execution", "registration/execution disagree"), + ("empty-execution", "missing nonempty inventory"), + ], +) +def test_native_inventory_refuses_registration_execution_drift(tmp_path, fault, reason): + registered, executed = ["unit-alpha", "unit-beta"], ["unit-alpha", "unit-beta"] + if fault == "empty-registration": + registered = [] + elif fault == "duplicate-registration": + registered = ["unit-alpha", "unit-alpha"] + elif fault == "omitted-registration": + registered = registered[:1] + elif fault == "omitted-execution": + executed = executed[:1] + elif fault == "extra-execution": + executed.append("unregistered") + else: + executed = [] + registration, junit = tmp_path / "registration.json", tmp_path / "junit.xml" + write_registration(registration, registered) + write_junit(junit, executed) + with pytest.raises(ValueError, match=reason): + main_proofs.native_inventory(registration, junit) + + +@pytest.mark.parametrize( + "collected,reason", + [ + ({}, "CTest registration has no tests list"), + ({"tests": None}, "CTest registration has no tests list"), + ({"tests": {}}, "CTest registration has no tests list"), + ({"tests": [None]}, "CTest test: expected an object"), + ({"tests": [{}]}, "missing nonempty inventory"), + ], +) +def test_native_inventory_refuses_malformed_registration(tmp_path, collected, reason): + registration, junit = tmp_path / "registration.json", tmp_path / "junit.xml" + registration.write_text(json.dumps(collected)) + write_junit(junit, ["unit-alpha"]) + with pytest.raises(ValueError, match=reason): + main_proofs.native_inventory(registration, junit) + + +@pytest.mark.parametrize("missing", ["registration", "junit"]) +def test_native_inventory_requires_both_report_files(tmp_path, missing): + registration, junit = tmp_path / "registration.json", tmp_path / "junit.xml" + if missing != "registration": + write_registration(registration, ["unit-alpha"]) + if missing != "junit": + write_junit(junit, ["unit-alpha"]) + absent = registration if missing == "registration" else junit + with pytest.raises(FileNotFoundError) as rejected: + main_proofs.native_inventory(registration, junit) + assert rejected.value.filename == str(absent) + + +@pytest.fixture +def producer_repository(tmp_path, monkeypatch): + """A normal local test repository; no hook/config/signing overrides.""" + root = tmp_path / "repository" + workflows = root / ".github/workflows" + workflows.mkdir(parents=True) + for name in ("ci-post.yml", "native-feed-post.yml"): + (workflows / name).write_bytes((ROOT / ".github/workflows" / name).read_bytes()) + commands = [ + ["git", "init", "-b", "main"], + ["git", "add", ".github/workflows/ci-post.yml", NATIVE_WORKFLOW], + ["git", "-c", "user.name=Producer control", + "-c", "user.email=producer-control@example.invalid", + "commit", "-m", "Unit producer identity"], + ] + for command in commands: + subprocess.run(command, cwd=root, check=True, capture_output=True) + head = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=root, text=True + ).strip() + tree = subprocess.check_output( + ["git", "rev-parse", "HEAD^{tree}"], cwd=root, text=True + ).strip() + for name, value in { + "GITHUB_EVENT_NAME": "push", "GITHUB_REF": "refs/heads/main", + "GITHUB_REPOSITORY": "owner/data", + "GITHUB_WORKFLOW_REF": "owner/data/.github/workflows/ci-post.yml@refs/heads/main", + "GITHUB_SHA": head, "GITHUB_RUN_ID": "731", "GITHUB_RUN_ATTEMPT": "3", + }.items(): + monkeypatch.setenv(name, value) + return root, { + "run_id": 731, "attempt": 3, "head": head, "tested_sha": head, "tree": tree, + } + + +@pytest.fixture +def network_reports(producer_repository, tmp_path): + root, identity = producer_repository + directory = tmp_path / "network-reports" + directory.mkdir() + raw = write_junit( + directory / "network.xml", + ["test_bootstrap_recorded_hashes_match_live_pypi"], + network=True, + ) + proof = main_proofs.network_proof(root, directory) + (directory / "network-proof.json").write_bytes(contract.json_bytes(proof)) + return root, directory, identity, raw + + +def test_network_producer_and_consumer_bind_actual_report(network_reports): + root, directory, identity, raw = network_reports + expected = { + **identity, "node_id": NETWORK_NODE, + **expected_inventory([NETWORK_NODE], raw), + } + assert main_proofs.network_proof(root, directory) == expected + assert main_proofs.consume_network(root, directory) == expected + + +@pytest.mark.parametrize( + "fault,reason", + [ + ("missing-classname", "network JUnit has no source classname"), + ("wrong-classname", "wrong live-PyPI inventory"), + ("wrong-node", "wrong live-PyPI inventory"), + ("duplicate-node", "duplicate inventory"), + ("omitted-node", "missing nonempty inventory"), + ("skipped-node", "executed: missing nonempty inventory"), + ("failed-node", "required rows failures"), + ("errored-node", "required rows errors"), + ], +) +def test_network_proof_refuses_wrong_source_or_non_full_row(network_reports, fault, reason): + root, directory, _identity, _raw = network_reports + path = directory / "network.xml" + xml = ElementTree.fromstring(path.read_bytes()) + suite, case = xml.find("testsuite"), xml.find(".//testcase") + if fault == "missing-classname": + case.attrib.pop("classname") + elif fault == "wrong-classname": + case.set("classname", "tests.test_release_adapter") + elif fault == "wrong-node": + case.set("name", "test_an_unrelated_network_request") + elif fault == "duplicate-node": + suite.append(copy.deepcopy(case)) + suite.set("tests", "2") + xml.set("tests", "2") + elif fault == "omitted-node": + suite.remove(case) + suite.set("tests", "0") + xml.set("tests", "0") + else: + tag = {"skipped-node": "skipped", "failed-node": "failure", "errored-node": "error"}[fault] + ElementTree.SubElement(case, tag, message="authored network refusal") + path.write_bytes(ElementTree.tostring(xml)) + with pytest.raises(ValueError, match=reason): + main_proofs.network_proof(root, directory) + + +@pytest.mark.parametrize( + "name,value,reason", + [ + ("GITHUB_EVENT_NAME", "pull_request", "required push/main workflow"), + ("GITHUB_REF", "refs/heads/topic", "required push/main workflow"), + ("GITHUB_REPOSITORY", "other/data", "required push/main workflow"), + ("GITHUB_WORKFLOW_REF", "owner/data/.github/workflows/ci.yml@refs/heads/main", + "required push/main workflow"), + ("GITHUB_SHA", "e" * 40, "checkout differs from event head"), + ], +) +def test_network_proof_refuses_wrong_producer_source( + network_reports, monkeypatch, name, value, reason +): + root, directory, _identity, _raw = network_reports + monkeypatch.setenv(name, value) + with pytest.raises(ValueError, match=reason): + main_proofs.network_proof(root, directory) + + +@pytest.mark.parametrize( + "field", ["run_id", "attempt", "head", "tested_sha", "tree", "junit_sha256", "node_id"] +) +def test_consume_network_refuses_recorded_identity_or_digest_tampering(network_reports, field): + root, directory, _identity, _raw = network_reports + path = directory / "network-proof.json" + proof = contract.object_json(path.read_bytes()) + original = proof[field] + proof[field] = original + 1 if type(original) is int else ( + hashlib.sha256(f"tampered:{field}".encode()).hexdigest()[:len(original)] + ) + path.write_bytes(contract.json_bytes(proof)) + with pytest.raises(ValueError, match="network report bytes or producer identity changed"): + main_proofs.consume_network(root, directory) + + +def test_consume_network_refuses_changed_raw_bytes_with_unchanged_rows(network_reports): + root, directory, _identity, _raw = network_reports + path = directory / "network.xml" + path.write_bytes(path.read_bytes() + b"\n") + with pytest.raises(ValueError, match="network report bytes or producer identity changed"): + main_proofs.consume_network(root, directory) + + +@pytest.mark.parametrize("field", ["registered", "executed", "skipped", "failures", "errors"]) +def test_consume_network_refuses_recorded_inventory_tampering(network_reports, field): + root, directory, _identity, _raw = network_reports + path = directory / "network-proof.json" + proof = contract.object_json(path.read_bytes()) + proof[field] = [] if field in {"registered", "executed"} else [NETWORK_NODE] + path.write_bytes(contract.json_bytes(proof)) + with pytest.raises(ValueError, match="network report bytes or producer identity changed"): + main_proofs.consume_network(root, directory) + + +@pytest.mark.parametrize("missing", ["network.xml", "network-proof.json"]) +def test_consume_network_requires_both_report_files(network_reports, tmp_path, missing): + root, directory, _identity, _raw = network_reports + incomplete = tmp_path / "incomplete-network" + incomplete.mkdir() + for name in ("network.xml", "network-proof.json"): + if name != missing: + (incomplete / name).write_bytes((directory / name).read_bytes()) + with pytest.raises(FileNotFoundError) as rejected: + main_proofs.consume_network(root, incomplete) + assert rejected.value.filename == str(incomplete / missing) + + +@pytest.mark.parametrize("name", ["GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT"]) +def test_consume_network_refuses_previous_run_or_attempt(network_reports, monkeypatch, name): + root, directory, identity, _raw = network_reports + previous = identity["run_id" if name == "GITHUB_RUN_ID" else "attempt"] + monkeypatch.setenv(name, str(previous + 1)) + with pytest.raises(ValueError, match="network report bytes or producer identity changed"): + main_proofs.consume_network(root, directory) + + +@pytest.fixture +def native_reports(producer_repository, tmp_path, monkeypatch): + root, identity = producer_repository + monkeypatch.setenv( + "GITHUB_WORKFLOW_REF", f"owner/data/{NATIVE_WORKFLOW}@refs/heads/main" + ) + directory = tmp_path / "native-reports" + directory.mkdir() + for leg in NATIVE_LEGS: + reports = directory / leg + reports.mkdir() + # Different reports in each leg make accidental report substitution observable. + names = native_report_names(leg) + write_registration(reports / "native-registration.json", names) + write_junit(reports / "native-junit.xml", names) + (reports / "native-leg.json").write_bytes( + contract.json_bytes(main_proofs.native_leg(root, leg, reports)) + ) + needs = {name: {"result": "success"} for name in ("authz", "main-ref", *NATIVE_LEGS)} + return root, directory, identity, needs + + +def test_native_leg_and_aggregate_consume_real_files_and_authoritative_needs( + native_reports, monkeypatch +): + root, directory, identity, needs = native_reports + api_calls = [] + + def reject_job_api(*args, **kwargs): + api_calls.append((args, kwargs)) + raise AssertionError("a running native aggregate must not query its own job") + + monkeypatch.setattr(contract, "gh_bytes", reject_job_api) + monkeypatch.setattr(evidence, "gh_bytes", reject_job_api) + git_reads = [] + real_check_output = subprocess.check_output + + def observe_identity_read(command, **kwargs): + git_reads.append(command) + assert command in ( + ["git", "rev-parse", "HEAD"], ["git", "rev-parse", "HEAD^{tree}"] + ), "native aggregation may read local identity, never query a completed job" + assert kwargs == {"cwd": root, "text": True} + return real_check_output(command, **kwargs) + + monkeypatch.setattr(main_proofs.subprocess, "check_output", observe_identity_read) + result = main_proofs.native_aggregate(root, directory, needs) + expected_legs = {} + for leg in NATIVE_LEGS: + reports = directory / leg + inventory = expected_native_inventory(reports, native_report_names(leg)) + assert main_proofs.native_leg(root, leg, reports) == { + **identity, "leg": leg, "inventory": inventory, + } + expected_legs[leg] = { + **inventory, "artifact": f"native-leg-{leg}-{identity['attempt']}", + } + assert result == {**identity, "legs": expected_legs} + assert api_calls == [] + assert git_reads + assert "Native proof" not in needs + + +@pytest.mark.parametrize("leg", NATIVE_LEGS) +def test_native_required_subset_accepts_extra_rows_and_binds_their_bytes(native_reports, leg): + root, directory, identity, needs = native_reports + reports = directory / leg + required = list(REQUIRED_NATIVE_ROWS[leg]) + minimum = write_native_report_pair(reports, required) + record_native_leg(reports, identity, leg, minimum) + assert main_proofs.native_leg(root, leg, reports) == { + **identity, "leg": leg, "inventory": minimum, + } + before = main_proofs.native_aggregate(root, directory, needs) + artifact = f"native-leg-{leg}-{identity['attempt']}" + assert before["legs"][leg] == {**minimum, "artifact": artifact} + + # Represent CTest's WILL_FAIL success with a normal, unskipped fixture row. + if leg == "tsan": + assert "feed-tsan-seeded-race" in minimum["executed"] + else: + assert "feed-tsan-seeded-race" not in minimum["registered"] + extended = write_native_report_pair(reports, native_report_names(leg)) + record_native_leg(reports, identity, leg, extended) + assert main_proofs.native_leg(root, leg, reports) == { + **identity, "leg": leg, "inventory": extended, + } + after = main_proofs.native_aggregate(root, directory, needs) + assert after == { + **before, "legs": {**before["legs"], leg: {**extended, "artifact": artifact}}, + } + assert extended["executed"] != minimum["executed"] + for field in ("registration_sha256", "junit_sha256"): + assert extended[field] != minimum[field] + + +@pytest.mark.parametrize("leg,missing", REQUIRED_NATIVE_OMISSIONS) +@pytest.mark.parametrize("producer", ["native-leg", "native-aggregate"]) +def test_native_producers_refuse_mutually_reduced_required_coverage( + native_reports, leg, missing, producer +): + root, directory, identity, needs = native_reports + reports = directory / leg + complete = expected_native_inventory(reports, native_report_names(leg)) + assert main_proofs.native_leg(root, leg, reports) == { + **identity, "leg": leg, "inventory": complete, + } + positive = main_proofs.native_aggregate(root, directory, needs) + assert positive["legs"][leg] == { + **complete, "artifact": f"native-leg-{leg}-{identity['attempt']}", + } + + names = [name for name in native_report_names(leg) if name != missing] + reduced = write_native_report_pair(reports, names) + record_native_leg(reports, identity, leg, reduced) + # Both inventories agree, counters are honest, and the recorded hashes are current. + assert names and missing not in names + assert main_proofs.native_inventory( + reports / "native-registration.json", reports / "native-junit.xml" + ) == reduced + for field in ("registration_sha256", "junit_sha256"): + assert reduced[field] != complete[field] + with pytest.raises(ValueError) as rejected: + if producer == "native-leg": + main_proofs.native_leg(root, leg, reports) + else: + main_proofs.native_aggregate(root, directory, needs) + assert str(rejected.value) == f"native {leg}: missing required native rows: {missing}" + + +@pytest.fixture +def complete_main_reports(native_reports, tmp_path, monkeypatch): + """Real local producers supply both proofs; job/run observations are offline inputs.""" + root, directory, identity, needs = native_reports + network_directory = tmp_path / "main-network" + network_directory.mkdir() + network_raw = write_junit( + network_directory / "network.xml", + ["test_bootstrap_recorded_hashes_match_live_pypi"], + network=True, + ) + python_identity = { + **identity, "run_id": identity["run_id"] + 10, "attempt": identity["attempt"] + 1, + } + with monkeypatch.context() as network_environment: + network_environment.setenv( + "GITHUB_WORKFLOW_REF", "owner/data/.github/workflows/ci-post.yml@refs/heads/main" + ) + network_environment.setenv("GITHUB_RUN_ID", str(python_identity["run_id"])) + network_environment.setenv("GITHUB_RUN_ATTEMPT", str(python_identity["attempt"])) + network = main_proofs.network_proof(root, network_directory) + expected_network = { + **python_identity, "node_id": NETWORK_NODE, + **expected_inventory([NETWORK_NODE], network_raw), + } + assert network == expected_network + (network_directory / "network-proof.json").write_bytes(contract.json_bytes(network)) + assert main_proofs.consume_network(root, network_directory) == expected_network + + native = main_proofs.native_aggregate(root, directory, needs) + expected_legs = { + leg: { + **expected_native_inventory(directory / leg, native_report_names(leg)), + "artifact": f"native-leg-{leg}-{identity['attempt']}", + } + for leg in NATIVE_LEGS + } + assert native == {**identity, "legs": expected_legs} + run = { + "id": python_identity["run_id"], "run_attempt": python_identity["attempt"], + "head_sha": identity["head"], "head_branch": "main", "event": "push", + "path": ".github/workflows/ci-post.yml", + "status": "completed", "conclusion": "success", + } + native_run = { + **run, "id": identity["run_id"], "run_attempt": identity["attempt"], + "path": NATIVE_WORKFLOW, + } + + def observed_jobs(names, observation): + return [ + { + "name": name, "status": "completed", "conclusion": "success", + "run_id": observation["id"], "run_attempt": observation["run_attempt"], + "head_sha": observation["head_sha"], + } + for name in names + ] + + jobs = observed_jobs((*evidence.JOBS, "release/ready", "Live PyPI hashes"), run) + proof = { + **python_identity, "event": "push", "base": None, "pr": None, + "jobs": dict.fromkeys(evidence.JOBS, "success"), + "e2e": {suite: sorted(rows) for suite, rows in evidence.TESTS.items()}, + "main_proofs": { + "network": network, + "native": { + "workflow_path": NATIVE_WORKFLOW, "run": native_run, + "jobs": observed_jobs(NATIVE_JOB_NAMES, native_run), "attestation": native, + }, + }, + } + return root, directory, identity, proof, run, jobs + + +def test_validate_main_accepts_complete_local_proofs_and_observed_jobs(complete_main_reports): + _root, _directory, _identity, proof, run, jobs = complete_main_reports + main_proofs.validate_main(proof, run, jobs) + evidence.validate_gate(proof, run, jobs, head=proof["head"], base=None, pr=None) + + +@pytest.mark.parametrize("leg,missing", REQUIRED_NATIVE_OMISSIONS) +def test_validate_main_refuses_mutually_reduced_required_coverage( + complete_main_reports, leg, missing +): + _root, directory, identity, proof, run, jobs = complete_main_reports + # Validate the complete twin before changing any report or proof content. + main_proofs.validate_main(proof, run, jobs) + evidence.validate_gate(proof, run, jobs, head=proof["head"], base=None, pr=None) + native = proof["main_proofs"]["native"]["attestation"] + before = copy.deepcopy(native["legs"][leg]) + reports = directory / leg + names = [name for name in native_report_names(leg) if name != missing] + reduced = write_native_report_pair(reports, names) + record_native_leg(reports, identity, leg, reduced) + assert names and missing not in names + assert main_proofs.native_inventory( + reports / "native-registration.json", reports / "native-junit.xml" + ) == reduced + native["legs"][leg] = {**reduced, "artifact": before["artifact"]} + for field in ("registration_sha256", "junit_sha256"): + assert native["legs"][leg][field] != before[field] + # The consumer sees an internally consistent reduced proof, not stale evidence. + with pytest.raises(ValueError) as rejected: + main_proofs.validate_main(proof, run, jobs) + assert str(rejected.value) == f"native {leg}: missing required native rows: {missing}" + + +@pytest.mark.parametrize("dependency", ["authz", "main-ref", *NATIVE_LEGS]) +@pytest.mark.parametrize("result", ["missing", "failure", "cancelled", "skipped", "in_progress"]) +def test_native_aggregate_refuses_missing_or_unsuccessful_needs(native_reports, dependency, result): + root, directory, _identity, needs = native_reports + if result == "missing": + needs.pop(dependency) + reason = "native needs inventory changed" + else: + needs[dependency]["result"] = result + reason = "native dependency failed, cancelled or skipped" + with pytest.raises(ValueError, match=reason): + main_proofs.native_aggregate(root, directory, needs) + + +def test_native_aggregate_refuses_unexpected_dependency(native_reports): + root, directory, _identity, needs = native_reports + needs["unreviewed-leg"] = {"result": "success"} + with pytest.raises(ValueError, match="native needs inventory changed"): + main_proofs.native_aggregate(root, directory, needs) + + +@pytest.mark.parametrize("leg", NATIVE_LEGS) +@pytest.mark.parametrize( + "field", ["run_id", "attempt", "head", "tested_sha", "tree", "leg", + "junit_sha256", "registration_sha256"] +) +def test_native_aggregate_refuses_one_stale_leg_identity_or_digest(native_reports, leg, field): + root, directory, _identity, needs = native_reports + path = directory / leg / "native-leg.json" + row = contract.object_json(path.read_bytes()) + target = row["inventory"] if field.endswith("sha256") else row + old = target[field] + target[field] = old + 1 if type(old) is int else ( + "other-leg" if field == "leg" + else hashlib.sha256(f"tampered:{field}".encode()).hexdigest()[:len(old)] + ) + path.write_bytes(contract.json_bytes(row)) + with pytest.raises(ValueError, match=f"{leg}: report bytes or producer identity changed"): + main_proofs.native_aggregate(root, directory, needs) + + +@pytest.mark.parametrize("leg", NATIVE_LEGS) +@pytest.mark.parametrize("field", ["registered", "executed", "skipped", "failures", "errors"]) +def test_native_aggregate_refuses_recorded_inventory_tampering(native_reports, leg, field): + root, directory, _identity, needs = native_reports + path = directory / leg / "native-leg.json" + row = contract.object_json(path.read_bytes()) + row["inventory"][field] = ( + [] if field in {"registered", "executed"} else [f"{leg}-unit"] + ) + path.write_bytes(contract.json_bytes(row)) + with pytest.raises(ValueError, match=f"{leg}: report bytes or producer identity changed"): + main_proofs.native_aggregate(root, directory, needs) + + +@pytest.mark.parametrize("leg", NATIVE_LEGS) +@pytest.mark.parametrize( + "missing", ["leg-directory", "native-leg.json", "native-registration.json", "native-junit.xml"] +) +def test_native_aggregate_requires_every_current_leg_report(native_reports, tmp_path, leg, missing): + root, directory, _identity, needs = native_reports + incomplete = tmp_path / "incomplete-native" + incomplete.mkdir() + for selected in NATIVE_LEGS: + if selected == leg and missing == "leg-directory": + continue + reports = incomplete / selected + reports.mkdir() + for name in ("native-leg.json", "native-registration.json", "native-junit.xml"): + if selected != leg or name != missing: + (reports / name).write_bytes((directory / selected / name).read_bytes()) + absent = incomplete / leg / ("native-leg.json" if missing == "leg-directory" else missing) + with pytest.raises(FileNotFoundError) as rejected: + main_proofs.native_aggregate(root, incomplete, needs) + assert rejected.value.filename == str(absent) + + +@pytest.mark.parametrize("leg", NATIVE_LEGS) +@pytest.mark.parametrize("filename", ["native-registration.json", "native-junit.xml"]) +def test_native_aggregate_reparses_raw_report_bytes(native_reports, leg, filename): + root, directory, _identity, needs = native_reports + path = directory / leg / filename + path.write_bytes(path.read_bytes() + b"\n") + with pytest.raises(ValueError, match=f"{leg}: report bytes or producer identity changed"): + main_proofs.native_aggregate(root, directory, needs) + + +@pytest.mark.parametrize("leg", NATIVE_LEGS) +@pytest.mark.parametrize( + "fault,reason", + [ + ("empty", "missing nonempty inventory"), + ("omitted", "registration/execution disagree"), + ("duplicate", "duplicate inventory"), + ("skipped", "registration/execution disagree|required rows skipped"), + ("failure", "required rows failures"), + ("error", "required rows errors"), + ], +) +def test_native_aggregate_refuses_non_full_leg_reports(native_reports, leg, fault, reason): + root, directory, _identity, needs = native_reports + reports = directory / leg + names = native_report_names(leg) + if fault == "empty": + names = [] + elif fault == "omitted": + names = names[:1] + elif fault == "duplicate": + names = [names[0], names[0]] + path = reports / "native-junit.xml" + write_junit(path, names) + if fault in {"skipped", "failure", "error"}: + xml = ElementTree.fromstring(path.read_bytes()) + ElementTree.SubElement(xml.find(".//testcase"), fault) + path.write_bytes(ElementTree.tostring(xml)) + # Even a digest refreshed to match the bad bytes cannot bypass report parsing. + saved = reports / "native-leg.json" + recorded = contract.object_json(saved.read_bytes()) + recorded["inventory"]["junit_sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + saved.write_bytes(contract.json_bytes(recorded)) + with pytest.raises(ValueError, match=reason): + main_proofs.native_aggregate(root, directory, needs) + + +@pytest.mark.parametrize("name", ["GITHUB_RUN_ID", "GITHUB_RUN_ATTEMPT"]) +def test_native_aggregate_refuses_previous_run_or_attempt(native_reports, monkeypatch, name): + root, directory, identity, needs = native_reports + previous = identity["run_id" if name == "GITHUB_RUN_ID" else "attempt"] + monkeypatch.setenv(name, str(previous + 1)) + with pytest.raises(ValueError, match="report bytes or producer identity changed"): + main_proofs.native_aggregate(root, directory, needs) + + +@pytest.mark.parametrize( + "name,value,reason", + [ + ("GITHUB_EVENT_NAME", "workflow_dispatch", "required push/main workflow"), + ("GITHUB_REF", "refs/heads/topic", "required push/main workflow"), + ("GITHUB_WORKFLOW_REF", "owner/data/.github/workflows/ci-post.yml@refs/heads/main", + "required push/main workflow"), + ("GITHUB_SHA", "e" * 40, "checkout differs from event head"), + ], +) +def test_native_leg_refuses_wrong_producer_source(native_reports, monkeypatch, name, value, reason): + root, directory, _identity, _needs = native_reports + monkeypatch.setenv(name, value) + with pytest.raises(ValueError, match=reason): + main_proofs.native_leg(root, "tsan", directory / "tsan") + + +def test_native_producers_refuse_changed_workflow_bytes(native_reports): + root, directory, _identity, needs = native_reports + path = root / NATIVE_WORKFLOW + path.write_bytes(path.read_bytes() + b"\n") + with pytest.raises(ValueError, match="authoritative needs require independent workflow review"): + main_proofs.native_leg(root, "tsan", directory / "tsan") + with pytest.raises(ValueError, match="authoritative needs require independent workflow review"): + main_proofs.native_aggregate(root, directory, needs) diff --git a/tests/test_post_merge_workflows.py b/tests/test_post_merge_workflows.py index 2e02201..35042a7 100644 --- a/tests/test_post_merge_workflows.py +++ b/tests/test_post_merge_workflows.py @@ -5,7 +5,10 @@ import copy import json import os +import re +import shlex import subprocess +from fnmatch import fnmatchcase from pathlib import Path from typing import Any @@ -16,6 +19,9 @@ ROOT = Path(__file__).resolve().parents[1] WORKFLOWS = ROOT / ".github/workflows" PAIRS = ("ci", "native-feed") +NETWORK_NODE = ( + "tests/test_consumer_receipt.py::test_bootstrap_recorded_hashes_match_live_pypi" +) ROUTES = { "workflow-lint": "ubuntu-24.04", "test": "ubuntu-24.04", @@ -62,6 +68,196 @@ def assert_post_events(workflow: dict[str, Any]) -> None: assert workflow["concurrency"]["cancel-in-progress"] == "false" +def workflow_uses(value: Any, location: str = "$") -> list[tuple[str, Any]]: + """Observe step and reusable-job references through every mapping/list.""" + references = [] + if isinstance(value, dict): + for key, child in value.items(): + path = f"{location}.{key}" + if key == "uses": + references.append((path, child)) + references.extend(workflow_uses(child, path)) + elif isinstance(value, list): + for index, child in enumerate(value): + references.extend(workflow_uses(child, f"{location}[{index}]")) + return references + + +def assert_action_pins(workflow: dict[str, Any]) -> None: + references = workflow_uses(workflow) + assert references, "post workflow has no uses references" + offending = [ + (path, reference) for path, reference in references + if not isinstance(reference, str) + or re.fullmatch(r"[^@\s]+@[0-9a-fA-F]{40}", reference) is None + ] + assert not offending, ( + "uses references require an @ suffix of exactly 40 hexadecimal characters: " + + "; ".join(f"{path}={reference!r}" for path, reference in offending) + ) + + +@pytest.mark.parametrize("stem", PAIRS) +def test_post_proof_workflows_pin_every_actual_uses_reference(stem: str) -> None: + path = WORKFLOWS / f"{stem}-post.yml" + workflow = yaml.load(path.read_text(), Loader=yaml.BaseLoader) + assert_action_pins(workflow) + + +@pytest.fixture(params=PAIRS) +def pinned_uses_control(request) -> dict[str, Any]: + path = WORKFLOWS / f"{request.param}-post.yml" + workflow = yaml.load(path.read_text(), Loader=yaml.BaseLoader) + observed = workflow_uses(workflow) + assert observed, f"{path.name}: missing actual uses inventory for the control" + pinned = [ + reference for _path, reference in observed + if isinstance(reference, str) + and re.fullmatch(r"[^@\s]+@[0-9a-fA-F]{40}", reference) + ] + assert pinned, f"{path.name}: no existing pinned reference for the positive control" + # Fixture-only mappings exercise traversal; they are not runnable workflows. + return { + "jobs": { + "reusable-control": {"uses": pinned[0]}, + "step-control": {"steps": [{"uses": reference} for reference in pinned]}, + }, + "nested-control": [{"mapping": {"uses": pinned[-1]}}], + } + + +def test_action_pin_control_accepts_references_from_the_actual_files(pinned_uses_control) -> None: + assert_action_pins(pinned_uses_control) + + +@pytest.mark.parametrize("location", ["step", "reusable-job", "nested"]) +@pytest.mark.parametrize("fault", ["floating-tag", "short-sha", "long-sha", "nonhex-sha"]) +def test_action_pin_control_refuses_a_changed_reference( + pinned_uses_control, location: str, fault: str +) -> None: + assert_action_pins(pinned_uses_control) + changed = copy.deepcopy(pinned_uses_control) + owners = { + "step": ("$.jobs.step-control.steps[0].uses", + changed["jobs"]["step-control"]["steps"][0]), + "reusable-job": ("$.jobs.reusable-control.uses", + changed["jobs"]["reusable-control"]), + "nested": ("$.nested-control[0].mapping.uses", + changed["nested-control"][0]["mapping"]), + } + path, owner = owners[location] + action, revision = owner["uses"].rsplit("@", 1) + suffix = { + "floating-tag": "v6", "short-sha": revision[:7], + "long-sha": revision + "0", "nonhex-sha": "g" + revision[1:], + }[fault] + owner["uses"] = offending = f"{action}@{suffix}" + with pytest.raises(AssertionError) as rejected: + assert_action_pins(changed) + message = str(rejected.value) + assert "uses references require an @ suffix of exactly 40 hexadecimal characters" in message + assert f"{path}={offending!r}" in message + + +def test_action_pin_control_refuses_an_empty_reference_inventory() -> None: + with pytest.raises(AssertionError) as rejected: + assert_action_pins({"jobs": {"run-only": {"steps": [{"run": "true"}]}}}) + assert str(rejected.value).splitlines()[0] == "post workflow has no uses references" + + +def native_ctest_step(workflow: dict[str, Any], job_id: str) -> dict[str, Any]: + steps = [ + step for step in workflow["jobs"][job_id]["steps"] + if re.search(r"(?m)^[ \t]*ctest(?:[ \t]|$)", step.get("run", "")) + ] + assert len(steps) == 1, f"{job_id}: expected one actual CTest-producing step" + return steps[0] + + +def assert_native_ctest_serial(workflow: dict[str, Any], job_id: str) -> None: + step = native_ctest_step(workflow, job_id) + commands = [ + shlex.split(line, comments=True) + for line in step["run"].replace("\\\n", " ").splitlines() + ] + ctest = [command for command in commands if command and command[0] == "ctest"] + assert len(ctest) == 2, f"{job_id}: require registration and execution commands" + for command in ctest: + parallel = [ + token for token in command[1:] + if token.startswith("-j") or token == "--parallel" or token.startswith("--parallel=") + ] + assert not parallel, ( + f"{job_id}: CTest must remain serial; parallel request {parallel!r} " + f"in {shlex.join(command)!r}" + ) + assert ctest[0] == [ + "ctest", "--test-dir", "build-feed", "--show-only=json-v1", + ">", "native-registration.json", + ], f"{job_id}: registration command changed" + assert ctest[1] == [ + "ctest", "--test-dir", "build-feed", "--output-on-failure", + "--output-junit", "$GITHUB_WORKSPACE/native-junit.xml", + ], f"{job_id}: JUnit execution command changed" + assert ["cmake", "--build", "build-feed", "-j4"] in commands, ( + f"{job_id}: preserve existing build parallelism" + ) + + +@pytest.mark.parametrize("job_id", NATIVE_ROUTES) +def test_native_post_ctest_execution_remains_serial(job_id: str) -> None: + path = WORKFLOWS / "native-feed-post.yml" + workflow = yaml.load(path.read_text(), Loader=yaml.BaseLoader) + assert_native_ctest_serial(workflow, job_id) + + +@pytest.fixture(params=NATIVE_ROUTES) +def serial_ctest_control(request) -> tuple[dict[str, Any], str]: + path = WORKFLOWS / "native-feed-post.yml" + workflow = yaml.load(path.read_text(), Loader=yaml.BaseLoader) + job_id = request.param + step = native_ctest_step(workflow, job_id) + # Fixture-only serial copy of the actual step, never a workflow-file repair. + step["run"] = re.sub( + r"(?m)^([ \t]*ctest[^\n]*?) -j4[ \t]*$", r"\1", step["run"] + ) + return workflow, job_id + + +def test_native_ctest_control_accepts_serial_tests_with_parallel_builds( + serial_ctest_control, +) -> None: + workflow, job_id = serial_ctest_control + assert_native_ctest_serial(workflow, job_id) + + +@pytest.mark.parametrize("command", ["registration", "execution"]) +@pytest.mark.parametrize("parallel", ["-j4", "-j 4", "--parallel 4", "--parallel=4"]) +def test_native_ctest_control_refuses_inserted_parallelism( + serial_ctest_control, command: str, parallel: str +) -> None: + workflow, job_id = serial_ctest_control + assert_native_ctest_serial(workflow, job_id) + changed = copy.deepcopy(workflow) + step = native_ctest_step(changed, job_id) + lines = step["run"].splitlines(keepends=True) + targets = [ + index for index, line in enumerate(lines) + if line.lstrip().startswith("ctest ") + and ("--show-only=json-v1" in line) == (command == "registration") + ] + assert len(targets) == 1 + index = targets[0] + # Put the option on the CTest invocation, before any registration redirection. + lines[index] = lines[index].replace("ctest ", f"ctest {parallel} ", 1) + step["run"] = "".join(lines) + with pytest.raises(AssertionError) as rejected: + assert_native_ctest_serial(changed, job_id) + message = str(rejected.value) + assert f"{job_id}: CTest must remain serial; parallel request" in message + assert parallel in message + + @pytest.mark.parametrize("stem", PAIRS) def test_pr_workflows_have_only_the_pull_request_trigger(stem: str) -> None: assert set(load(stem)["on"]) == {"pull_request"} @@ -83,27 +279,117 @@ def test_post_event_control_rejects_every_pr_event(stem: str, event: str) -> Non assert_post_events(changed) -def test_native_paths_cover_both_workflow_definitions() -> None: +def test_native_pr_paths_cover_both_workflow_definitions() -> None: paths = [ "native/feed/**", ".github/workflows/native-feed.yml", ".github/workflows/native-feed-post.yml", ] assert load("native-feed")["on"]["pull_request"]["paths"] == paths - assert load("native-feed", post=True)["on"]["push"]["paths"] == paths + + +@pytest.mark.parametrize( + "changed_path", + [ + "scripts/release_chain/evidence.py", + "tests/test_consumer_receipt.py", + ".github/workflows/ci-post.yml", + ], +) +def test_main_native_proof_runs_for_changes_outside_native_paths(changed_path: str) -> None: + workflow = load("native-feed", post=True) + assert_post_events(workflow) + assert (ROOT / changed_path).is_file() + assert not changed_path.startswith("native/feed/") + push = workflow["on"]["push"] + paths = push.get("paths") + assert paths is None or any(fnmatchcase(changed_path, pattern) for pattern in paths), ( + f"the actual main push path list omits a merge changing only {changed_path}" + ) + assert not any(fnmatchcase(changed_path, pattern) for pattern in push.get("paths-ignore", [])) + assert not {"paths", "paths-ignore"} & set(push), ( + f"a main merge changing only {changed_path} must still schedule all native legs" + ) + assert_native_post(workflow) + + +def assert_main_network_job(workflow: dict[str, Any]) -> str: + jobs = workflow["jobs"] + selected = [ + (job_id, job, step) + for job_id, job in jobs.items() + for step in job.get("steps", []) + if NETWORK_NODE in step.get("run", "") + ] + assert len(selected) == 1, "main CI is missing its single dedicated live-PyPI row" + job_id, job, step = selected[0] + assert job_id != "test" and "matrix" not in job.get("strategy", {}) + assert "if" not in job and "continue-on-error" not in job + assert needs(job) == {"authz", "main-ref"} + assert step.get("if") in { + None, "needs.authz.result == 'success'", "${{ needs.authz.result == 'success' }}" + } + assert "continue-on-error" not in step + tokens = shlex.split(step["run"]) + assert NETWORK_NODE in tokens + assert "pytest" in tokens + assert [token for token in tokens if token.startswith("tests/")] == [NETWORK_NODE] + assert not {"--collect-only", "--co"} & set(tokens) + enabled = { + **workflow.get("env", {}), **job.get("env", {}), **step.get("env", {}) + }.get("PINEFORGE_NETWORK_TEST") + assert enabled == "1" or "PINEFORGE_NETWORK_TEST=1" in tokens, ( + "the required row is skipped unless its real opt-in is enabled" + ) + assert any(token.startswith("--junitxml=") for token in tokens) or "--junitxml" in tokens + assert job_id in needs(jobs["ready"]) + assert job_id in needs(jobs["check"]) + return job_id + + +def test_main_ci_explicitly_executes_the_required_live_pypi_row() -> None: + assert_main_network_job(load("ci", post=True)) + + +def test_pr_ci_keeps_the_live_pypi_proof_in_the_main_tier() -> None: + workflow = load("ci") + assert workflow.get("env", {}).get("PINEFORGE_NETWORK_TEST") != "1" + for job in workflow["jobs"].values(): + assert job.get("env", {}).get("PINEFORGE_NETWORK_TEST") != "1" + for step in job.get("steps", []): + assert step.get("env", {}).get("PINEFORGE_NETWORK_TEST") != "1" + command = step.get("run", "") + assert NETWORK_NODE not in command + assert "PINEFORGE_NETWORK_TEST=1" not in command + + +def assert_native_aggregate(jobs: dict[str, Any]) -> str: + aggregates = [ + (job_id, job) for job_id, job in jobs.items() if job.get("name") == "Native proof" + ] + assert len(aggregates) == 1, "native legs alone do not produce immutable native evidence" + job_id, aggregate = aggregates[0] + assert needs(aggregate) == {"authz", "main-ref", *NATIVE_ROUTES} + assert "continue-on-error" not in aggregate + return job_id + + +def test_main_native_workflow_requires_the_producing_aggregate() -> None: + assert_native_aggregate(load("native-feed", post=True)["jobs"]) def assert_post_ci(workflow: dict[str, Any]) -> None: jobs = workflow["jobs"] + network_id = assert_main_network_job(workflow) assert "changes" not in jobs - assert set(jobs) == {"authz", "main-ref", "check", *ROUTES} + assert set(jobs) == {"authz", "main-ref", "check", network_id, *ROUTES} for job_id, route in ROUTES.items(): job = jobs[job_id] assert job["runs-on"] == route, job_id assert "if" not in job, job_id expected = {"authz", "main-ref"} if job_id == "ready": - expected |= evidence.NEEDS + expected |= evidence.NEEDS | {network_id} assert needs(job) == expected, job_id check = jobs["check"] assert check["if"] == "${{ always() }}" @@ -157,7 +443,45 @@ def assert_post_ci(workflow: dict[str, Any]) -> None: '"python-package-distributions-$ATTEMPT" --dir dist' ), } - assert jobs[job_id]["steps"] == expected, job_id + expected.insert(3, { + "name": "Download this attempt's actual live PyPI report", + "if": "needs.authz.result == 'success'", + "env": { + "GH_TOKEN": "${{ github.token }}", + "GH_REPO": "${{ github.repository }}", + "RUN_ID": "${{ github.run_id }}", + "ATTEMPT": "${{ github.run_attempt }}", + }, + "run": ( + 'gh run download "$RUN_ID" --name ' + '"network-proof-$ATTEMPT" --dir reports/network' + ), + }) + observed = copy.deepcopy(jobs[job_id]["steps"]) + aggregate_steps = [ + step for step in observed if "NEEDS_JSON" in step.get("env", {}) + ] + assert len(aggregate_steps) == 1 + encoded_needs = aggregate_steps[0]["env"].pop("NEEDS_JSON") + # Check every dependency-to-result binding, including the new main-only job. + # Keep all other existing step bytes under the original comparison. + expanded = re.sub( + r"\$\{\{\s*toJSON\(needs\.([A-Za-z0-9_-]+)\)\s*\}\}", + r'"\1"', + encoded_needs, + ) + pairs = json.loads(expanded, object_pairs_hook=list) + required = evidence.NEEDS | {network_id} + assert len(pairs) == len(required) + assert dict(pairs) == {name: name for name in required} + expected_aggregate = next( + step for step in expected if "NEEDS_JSON" in step.get("env", {}) + ) + expected_aggregate["env"].pop("NEEDS_JSON") + expected_aggregate["run"] += " --network-proof reports/network" + else: + observed = jobs[job_id]["steps"] + assert observed == expected, job_id def test_post_ci_has_every_proof_without_pr_classification() -> None: @@ -185,9 +509,47 @@ def test_post_ci_control_rejects_reverting_a_proof(fault: str) -> None: assert_post_ci(changed) +def with_native_reports(steps: list[dict[str, Any]], leg: str) -> list[dict[str, Any]]: + """Only augment the original commands with registration/JUnit production.""" + test_command = "ctest --test-dir build-feed --output-on-failure\n" + producing = [step for step in steps if test_command in step.get("run", "")] + assert len(producing) == 1 + producing[0]["run"] = "set -euo pipefail\n" + producing[0]["run"].replace( + test_command, + "ctest --test-dir build-feed --show-only=json-v1 > native-registration.json\n" + 'ctest --test-dir build-feed --output-on-failure --output-junit ' + '"$GITHUB_WORKSPACE/native-junit.xml"\n', + ) + return [ + *steps, + { + "uses": PINS["actions/setup-python@v6"], + "with": {"python-version": "3.13"}, + }, + { + "name": "Bind the actual registration and JUnit to this leg", + "env": {"PYTHONPATH": "scripts"}, + "run": ( + f"python -m release_chain.main_proofs native-leg --leg {leg} " + "--directory . --output native-leg.json" + ), + }, + { + "uses": PINS["actions/upload-artifact@v5"], + "if": "always()", + "with": { + "name": f"native-leg-{leg}-${{{{ github.run_attempt }}}}", + "path": "native-registration.json\nnative-junit.xml\nnative-leg.json\n", + "if-no-files-found": "error", + }, + }, + ] + + def assert_native_post(workflow: dict[str, Any]) -> None: jobs = workflow["jobs"] - assert set(jobs) == {"authz", "main-ref", *NATIVE_ROUTES} + aggregate_id = assert_native_aggregate(jobs) + assert set(jobs) == {"authz", "main-ref", aggregate_id, *NATIVE_ROUTES} for job_id, (name, route) in NATIVE_ROUTES.items(): job = jobs[job_id] assert job["name"] == name @@ -203,11 +565,11 @@ def assert_native_post(workflow: dict[str, Any]) -> None: expected = copy.deepcopy(original["steps"]) expected[0]["uses"] = PINS[expected[0]["uses"]] expected[1]["run"] += " libdigest-sha-perl" - assert jobs[job_id]["steps"] == expected + assert jobs[job_id]["steps"] == with_native_reports(expected, job_id) for job_id in ("asan-ubsan", "tsan"): expected = copy.deepcopy(load("native-feed")["jobs"][job_id]["steps"][1:]) expected[0]["run"] += " libdigest-sha-perl" - assert jobs[job_id]["steps"][1:] == expected + assert jobs[job_id]["steps"][1:] == with_native_reports(expected, job_id) def test_native_post_keeps_linux_proofs_and_the_hosted_macos_leg() -> None: @@ -216,9 +578,10 @@ def test_native_post_keeps_linux_proofs_and_the_hosted_macos_leg() -> None: def test_native_pods_install_the_curl_scripts_checksum_tool() -> None: jobs = load("native-feed", post=True)["jobs"] - for job_id, job in jobs.items(): - if job_id in {"authz", "main-ref", "release-macos"}: + for job_id in NATIVE_ROUTES: + if job_id == "release-macos": continue + job = jobs[job_id] commands = "\n".join(step.get("run", "") for step in job["steps"]) assert "libdigest-sha-perl" in commands assert commands.index("libdigest-sha-perl") < commands.index("build_ci_curl.sh") diff --git a/tests/test_release_adapter.py b/tests/test_release_adapter.py index 3d00849..7d43bb3 100644 --- a/tests/test_release_adapter.py +++ b/tests/test_release_adapter.py @@ -1,9 +1,11 @@ from __future__ import annotations import copy +import io import shutil import subprocess import time +import zipfile from pathlib import Path from typing import Any from xml.etree import ElementTree @@ -376,19 +378,231 @@ def green_gate() -> tuple[dict[str, Any], dict[str, Any], list[dict[str, Any]]]: return gate, run, jobs +MAIN_NETWORK_NODE = ( + "tests/test_consumer_receipt.py::test_bootstrap_recorded_hashes_match_live_pypi" +) +MAIN_NATIVE_JOBS = { + "release-amd64": "Release (ubuntu-24.04)", + "release-arm64": "Release (ubuntu-24.04-arm)", + "release-macos": "Release (macos-15)", + "asan-ubsan": "ASan + UBSan", + "tsan": "TSan", +} + + +@pytest.fixture(params=[("landed-one", 80, 2), ("landed-two", 180, 3)]) +def main_proof_inputs(request): + """Decided schema as unit input; these are not observed hosted results.""" + label, run_id, attempt = request.param + gate, run, jobs = green_gate() + head = contract.sha256(label.encode())[:40] + tree = contract.sha256(f"{label}:tree".encode())[:40] + gate.update( + run_id=run_id, attempt=attempt, head=head, tested_sha=head, tree=tree, + event="push", pr=None, base=None, + ) + run.update( + id=run_id, run_attempt=attempt, head_sha=head, head_branch="main", + event="push", path=".github/workflows/ci-post.yml", + ) + # The job label is fixture data, not a new public API or product constant. + # Once present, use the actual workflow's display name rather than pinning a choice here. + workflow = yaml.load( + (ROOT / ".github/workflows/ci-post.yml").read_text(), Loader=yaml.BaseLoader + ) + network_names = [ + job.get("name", job_id) + for job_id, job in workflow["jobs"].items() + if any(MAIN_NETWORK_NODE in step.get("run", "") for step in job.get("steps", [])) + ] + network_name = network_names[0] if len(network_names) == 1 else "Live PyPI hashes" + if not any(job["name"] == network_name for job in jobs): + jobs.append({"name": network_name, "status": "completed", "conclusion": "success"}) + native_run = { + **run, "id": run_id + 10, "run_attempt": attempt + 1, + "path": ".github/workflows/native-feed-post.yml", + } + legs = {} + for leg in MAIN_NATIVE_JOBS: + names = ["feed-unit", "feed-export-unit", "feed-mock-export", "feed-mock"] + if leg == "tsan": + names.append("feed-tsan-seeded-race") + junit = ElementTree.Element("testsuite", name=f"{label}:{leg}") + for name in names: + ElementTree.SubElement(junit, "testcase", name=name) + legs[leg] = { + "registered": names, "executed": list(names), "skipped": [], + "failures": [], "errors": [], + "junit_sha256": contract.sha256(ElementTree.tostring(junit)), + } + network_junit = ElementTree.Element("testsuite", name=f"{label}:network") + ElementTree.SubElement(network_junit, "testcase", name=MAIN_NETWORK_NODE) + gate["main_proofs"] = { + "native": { + "workflow_path": native_run["path"], + "run": native_run, + "jobs": [ + {"name": name, "status": "completed", "conclusion": "success"} + for name in (*MAIN_NATIVE_JOBS.values(), "Native proof") + ], + "attestation": { + "run_id": native_run["id"], "attempt": native_run["run_attempt"], + "head": head, "tested_sha": head, "tree": tree, "legs": legs, + }, + }, + "network": { + "run_id": run_id, "attempt": attempt, "head": head, "tested_sha": head, + "tree": tree, "node_id": MAIN_NETWORK_NODE, + "registered": [MAIN_NETWORK_NODE], "executed": [MAIN_NETWORK_NODE], + "skipped": [], "failures": [], "errors": [], + "junit_sha256": contract.sha256(ElementTree.tostring(network_junit)), + }, + } + return gate, run, jobs + + +def test_main_complete_matching_proofs_positive_twin(main_proof_inputs) -> None: + gate, run, jobs = main_proof_inputs + # Acceptance alone is uncredited until the negative twins and I/O controls work. + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) + + +@pytest.mark.parametrize("missing", ["main_proofs", "native", "network"]) +def test_main_python_green_requires_both_proof_slots(main_proof_inputs, missing) -> None: + gate, run, jobs = main_proof_inputs + if missing == "main_proofs": + gate.pop("main_proofs") + else: + gate["main_proofs"].pop(missing) + with pytest.raises(ValueError): + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) + + +@pytest.mark.parametrize("shape", ["null", "empty", "extra"]) +def test_main_proof_object_has_exactly_the_decided_slots(main_proof_inputs, shape) -> None: + gate, run, jobs = main_proof_inputs + if shape == "null": + gate["main_proofs"] = None + elif shape == "empty": + gate["main_proofs"] = {} + else: + gate["main_proofs"]["unobserved"] = {"conclusion": "success"} + with pytest.raises(ValueError): + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) + + +@pytest.mark.parametrize( + "path,value", + [ + (("workflow_path",), ".github/workflows/native-feed.yml"), + (("run", "path"), ".github/workflows/ci-post.yml"), + (("run", "event"), "pull_request"), + (("run", "head_branch"), "topic"), + (("run", "head_sha"), "f" * 40), + (("run", "id"), 999), + (("run", "run_attempt"), 1), + (("run", "status"), "in_progress"), + (("run", "conclusion"), "failure"), + (("attestation", "run_id"), 999), + (("attestation", "attempt"), 1), + (("attestation", "head"), "e" * 40), + (("attestation", "tested_sha"), "e" * 40), + (("attestation", "tree"), "e" * 40), + ], +) +def test_main_refuses_native_identity_or_run_drift(main_proof_inputs, path, value) -> None: + gate, run, jobs = main_proof_inputs + target = gate["main_proofs"]["native"] + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + with pytest.raises(ValueError): + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) + + +@pytest.mark.parametrize("name", [*MAIN_NATIVE_JOBS.values(), "Native proof"]) +@pytest.mark.parametrize( + "fault", ["missing", "duplicate", "in_progress", "failure", "cancelled", "skipped"] +) +def test_main_refuses_incomplete_native_jobs(main_proof_inputs, name, fault) -> None: + gate, run, jobs = main_proof_inputs + native_jobs = gate["main_proofs"]["native"]["jobs"] + job = next(row for row in native_jobs if row["name"] == name) + if fault == "missing": + native_jobs.remove(job) + elif fault == "duplicate": + native_jobs.append(copy.deepcopy(job)) + elif fault == "in_progress": + job.update(status="in_progress", conclusion=None) + else: + job["conclusion"] = fault + with pytest.raises(ValueError): + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) + + +@pytest.mark.parametrize("leg", MAIN_NATIVE_JOBS) +@pytest.mark.parametrize( + "fault", + ["missing", "empty", "omitted", "duplicate", "skipped", "failures", "errors", "digest"], +) +def test_main_refuses_vacuous_or_incomplete_native_inventory( + main_proof_inputs, leg, fault +) -> None: + gate, run, jobs = main_proof_inputs + legs = gate["main_proofs"]["native"]["attestation"]["legs"] + row = legs[leg] + if fault == "missing": + legs.pop(leg) + elif fault == "empty": + row.update(registered=[], executed=[]) + elif fault == "omitted": + row["executed"].pop() + elif fault == "duplicate": + row["registered"].append(row["registered"][0]) + row["executed"].append(row["executed"][0]) + elif fault == "digest": + row["junit_sha256"] = "" + else: + row[fault] = [row["registered"][0]] + with pytest.raises(ValueError): + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) + + +@pytest.mark.parametrize( + "field,value", + [ + ("run_id", 999), ("attempt", 1), ("head", "e" * 40), + ("tested_sha", "e" * 40), ("tree", "e" * 40), + ("node_id", "tests/test_consumer_receipt.py::other"), + ("registered", []), ("executed", []), + ("registered", ["tests/test_consumer_receipt.py::other"]), + ("executed", ["tests/test_consumer_receipt.py::other"]), + ("registered", [MAIN_NETWORK_NODE, MAIN_NETWORK_NODE]), + ("executed", [MAIN_NETWORK_NODE, MAIN_NETWORK_NODE]), + ("skipped", [MAIN_NETWORK_NODE]), ("failures", [MAIN_NETWORK_NODE]), + ("errors", [MAIN_NETWORK_NODE]), ("junit_sha256", ""), + ], +) +def test_main_disabled_or_wrong_network_row_is_non_full( + main_proof_inputs, field, value +) -> None: + gate, run, jobs = main_proof_inputs + gate["main_proofs"]["network"][field] = value + with pytest.raises(ValueError): + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) + + def test_gate_accepts_only_the_complete_current_attempt() -> None: gate, run, jobs = green_gate() evidence.validate_gate(gate, run, jobs, head="a" * 40, base="b" * 40, pr=7) -def test_landed_evidence_requires_the_post_workflow() -> None: - gate, run, jobs = green_gate() - gate.update(event="push", pr=None, base=None) - run.update(event="push", path=".github/workflows/ci-post.yml") - evidence.validate_gate(gate, run, jobs, head="a" * 40, base=None, pr=None) +def test_landed_evidence_requires_the_post_workflow(main_proof_inputs) -> None: + gate, run, jobs = main_proof_inputs + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) run["path"] = ".github/workflows/ci.yml" with pytest.raises(ValueError, match="unexpected CI workflow"): - evidence.validate_gate(gate, run, jobs, head="a" * 40, base=None, pr=None) + evidence.validate_gate(gate, run, jobs, head=gate["head"], base=None, pr=None) def test_pr_evidence_cannot_use_the_post_workflow() -> None: @@ -429,6 +643,349 @@ def api(repo, endpoint): assert report["pr"]["merge"] == "a" * 40 +class RecoveryPublishersBoundary(RuntimeError): + """A validated recovery is about to read the later publisher-run inventory.""" + + +@pytest.fixture +def recovery_main_transport(baseline, main_proof_inputs, monkeypatch): + """Keep recovery, markers, local metadata, artifact parsing and gates real.""" + gate, python, jobs = main_proof_inputs + native = gate["main_proofs"]["native"] + python_proof = copy.deepcopy(gate) + python_proof["main_proofs"] = {"network": gate["main_proofs"]["network"]} + pr = { + "number": 7, "merged_at": "2026-11-07T12:00:00Z", + "merge_commit_sha": python["head_sha"], + "head": {"sha": "b" * 40}, "base": {"ref": "main", "sha": "c" * 40}, + "body": automation.body({ + "version": contract.package_version(baseline), "base": "c" * 40, + }), + } + state = { + "root": baseline, "pr": pr, "python": python, "python_runs": [python], + "python_proof": python_proof, "python_jobs": jobs, + "native": native, "native_runs": [native["run"]], + "native_artifact_missing": False, "reads": [], "writes": [], + "parsed": [], "gate_calls": [], "validated": [], "local_checks": [], + "publisher_reads": [], "local_attempts": [], + "waiting_inputs": [], "waiting_validations": [], + } + + def gh_bytes(route, **kwargs): + assert not kwargs + state["reads"].append(route) + prefix = "repos/owner/data/" + assert route.startswith(prefix) + endpoint = route.removeprefix(prefix) + path = endpoint.split("?", 1)[0] + if path == "actions/workflows/native-feed-post.yml/runs": + assert endpoint == ( + "actions/workflows/native-feed-post.yml/runs" + f"?event=push&head_sha={python['head_sha']}&per_page=100" + ) + return contract.json_bytes({"workflow_runs": state["native_runs"]}) + for run, observed_jobs, payload, artifact, filename in ( + (python, jobs, python_proof, "release-evidence", "evidence.json"), + (native["run"], native["jobs"], native["attestation"], + "native-evidence", "native-evidence.json"), + ): + run_id, attempt = run["id"], run["run_attempt"] + if path == f"actions/runs/{run_id}": + return contract.json_bytes(run) + if path == f"actions/runs/{run_id}/attempts/{attempt}/jobs": + return contract.json_bytes({"jobs": observed_jobs}) + if path == f"actions/runs/{run_id}/artifacts": + missing = artifact == "native-evidence" and state["native_artifact_missing"] + return contract.json_bytes({"artifacts": [] if missing else [{ + "id": run_id * 100, "name": f"{artifact}-{attempt}", "expired": False, + }]}) + if path == f"actions/artifacts/{run_id * 100}/zip": + archive = io.BytesIO() + with zipfile.ZipFile(archive, "w") as zipped: + zipped.writestr(filename, contract.json_bytes(payload)) + return archive.getvalue() + raise AssertionError(f"unexpected recovery proof transport: {route}") + + def api(repo, endpoint, method="GET", data=None): + assert repo == "owner/data" + if method != "GET": + state["writes"].append((endpoint, method, data)) + raise AssertionError("recovery dispatched or mutated before its proof boundary") + assert data is None + state["reads"].append(endpoint) + if endpoint == "releases?per_page=100&page=1": + return [] + if endpoint == ( + "pulls?state=closed&base=main&sort=updated&direction=desc&per_page=100&page=1" + ): + return [copy.deepcopy(pr)] + if endpoint == ( + "actions/workflows/ci-post.yml/runs?event=push&per_page=100&head_sha=" + + pr["merge_commit_sha"] + ): + return {"workflow_runs": copy.deepcopy(state["python_runs"])} + if endpoint == "actions/workflows/release.yml/runs?per_page=100": + state["publisher_reads"].append(endpoint) + assert len(state["validated"]) == 1 + assert state["local_checks"] == [(baseline, None)] + raise RecoveryPublishersBoundary("before publisher-run inventory read") + raise AssertionError(f"unexpected recovery API transport: {endpoint}") + + real_parse = evidence.object_json + real_gate = publishing.gate_for_run + real_verify_local = publishing.verify_local + real_validate_waiting_python = evidence._validate_waiting_python + + def parse(raw): + result = real_parse(raw) + state["parsed"].append(copy.deepcopy(result)) + return result + + def observe_gate(repo, run_id, **kwargs): + state["gate_calls"].append((repo, run_id, kwargs)) + result = real_gate(repo, run_id, **kwargs) + state["validated"].append(copy.deepcopy(result)) + return result + + def verify_local(root): + state["local_attempts"].append(root) + result = real_verify_local(root) + state["local_checks"].append((root, copy.deepcopy(result))) + return result + + def validate_waiting_python(proof, run, observed_jobs, **kwargs): + state["waiting_inputs"].append(copy.deepcopy((proof, run, observed_jobs, kwargs))) + result = real_validate_waiting_python(proof, run, observed_jobs, **kwargs) + state["waiting_validations"].append(copy.deepcopy(proof)) + return result + + monkeypatch.setattr(automation, "api", api) + monkeypatch.setattr(publishing, "api", api) + monkeypatch.setattr(reporting, "api", api) + monkeypatch.setattr(evidence, "gh_bytes", gh_bytes) + monkeypatch.setattr(evidence, "object_json", parse) + monkeypatch.setattr(evidence, "_validate_waiting_python", validate_waiting_python) + monkeypatch.setattr(publishing, "gate_for_run", observe_gate) + monkeypatch.setattr(publishing, "verify_local", verify_local) + return state + + +@pytest.mark.parametrize("readiness", ["missing", "queued", "in_progress"]) +def test_recovery_waits_for_missing_or_pending_native_proof(recovery_main_transport, readiness): + state = recovery_main_transport + if readiness == "missing": + state["native_runs"] = [] + else: + state["native"]["run"].update(status=readiness, conclusion=None) + state["native_artifact_missing"] = True + for job in state["native"]["jobs"]: + job.update(status=readiness, conclusion=None) + report = receipts.receipt(None) + publishing.recover_applied("owner/data", state["root"], report) + assert report["state"] == "testing" + assert report["pr"] == automation.pr_identity({**state["pr"], "merged": True}) + assert state["gate_calls"] == [ + ("owner/data", state["python"]["id"], {"head": state["python"]["head_sha"]}), + ] + assert {"workflow_runs": state["native_runs"]} in state["parsed"] + assert state["python_proof"] in state["parsed"], "the Python artifact must be parsed" + assert state["validated"] == state["local_checks"] == state["publisher_reads"] == [] + reporting.incident("owner/data", report, "release-owner", "recover-applied") + assert state["writes"] == [] + + +@pytest.mark.parametrize("readiness", ["missing", "queued", "in_progress"]) +@pytest.mark.parametrize( + "fault,reason", + [ + ("stale-attempt", "stale CI attempt"), + ("network-non-object", "network: expected an object"), + ( + "network-job-red", + "required job 'Live PyPI hashes' missing, duplicated, failed, cancelled or skipped", + ), + ("wrong-tree", "network: wrong tree"), + ("e2e-incomplete", "incomplete E2E evidence"), + ], +) +def test_recovery_refuses_invalid_python_while_native_pending( + recovery_main_transport, readiness, fault, reason +): + state = recovery_main_transport + python, native = state["python"], state["native"] + current_python = copy.deepcopy(python) + proof = state["python_proof"] + if readiness == "missing": + state["native_runs"] = [] + else: + native["run"].update(status=readiness, conclusion=None) + state["native_artifact_missing"] = True + for job in native["jobs"]: + job.update(status=readiness, conclusion=None) + if fault == "stale-attempt": + proof["attempt"] -= 1 + elif fault == "network-non-object": + proof["main_proofs"]["network"] = [] + elif fault == "network-job-red": + next(job for job in state["python_jobs"] if job["name"] == "Live PyPI hashes")[ + "conclusion" + ] = "failure" + elif fault == "wrong-tree": + original_tree = proof["tree"] + proof["tree"] = contract.sha256(b"another completed recovery evidence tree")[:40] + assert proof["tree"] != original_tree + assert proof["main_proofs"]["network"]["tree"] == original_tree + else: + assert fault == "e2e-incomplete" + for suite in ("docker", "database"): + proof["e2e"][suite].pop() + assert python == current_python + assert python["status"] == "completed" and python["conclusion"] == "success" + report = receipts.receipt(None) + before = copy.deepcopy(report) + with pytest.raises(evidence.GateRejected) as rejected: + publishing.recover_applied("owner/data", state["root"], report) + assert type(rejected.value) is evidence.GateRejected + assert str(rejected.value) == reason + assert report["state"] == before["state"] == "pending" + assert report == { + **before, "pr": automation.pr_identity({**state["pr"], "merged": True}), + } + assert state["gate_calls"] == [ + ("owner/data", python["id"], {"head": python["head_sha"]}), + ] + assert {"workflow_runs": state["native_runs"]} in state["parsed"] + if readiness != "missing": + assert native["run"] in state["parsed"] + assert proof in state["parsed"] + assert {"jobs": state["python_jobs"]} in state["parsed"] + assert state["waiting_inputs"] == [ + (proof, python, state["python_jobs"], {"head": python["head_sha"], "base": None}), + ] + assert state["waiting_validations"] == state["validated"] == [] + assert state["local_attempts"] == state["local_checks"] == state["publisher_reads"] == [] + assert state["writes"] == [] + native_id = native["run"]["id"] + assert not any( + f"/runs/{native_id}/attempts/" in route + or f"/runs/{native_id}/artifacts" in route + or f"/artifacts/{native_id * 100}/" in route + for route in state["reads"] + ) + + +@pytest.mark.parametrize("readiness", ["missing", "queued", "in_progress"]) +def test_recovery_waits_with_valid_python_while_native_pending_positive_twin( + recovery_main_transport, readiness +): + state = recovery_main_transport + python, native = state["python"], state["native"] + if readiness == "missing": + state["native_runs"] = [] + else: + native["run"].update(status=readiness, conclusion=None) + state["native_artifact_missing"] = True + for job in native["jobs"]: + job.update(status=readiness, conclusion=None) + assert python["status"] == "completed" and python["conclusion"] == "success" + report = receipts.receipt(None) + publishing.recover_applied("owner/data", state["root"], report) + assert report["state"] == "testing" + assert report["pr"] == automation.pr_identity({**state["pr"], "merged": True}) + assert state["gate_calls"] == [ + ("owner/data", python["id"], {"head": python["head_sha"]}), + ] + assert {"workflow_runs": state["native_runs"]} in state["parsed"] + if readiness != "missing": + assert native["run"] in state["parsed"] + proof = state["python_proof"] + assert proof in state["parsed"] + assert {"jobs": state["python_jobs"]} in state["parsed"] + assert state["waiting_inputs"] == [ + (proof, python, state["python_jobs"], {"head": python["head_sha"], "base": None}), + ] + assert state["waiting_validations"] == [proof] + assert state["validated"] == state["local_attempts"] == state["local_checks"] == [] + assert state["publisher_reads"] == state["writes"] == [] + reporting.incident("owner/data", report, "release-owner", "recover-applied") + assert state["writes"] == [] + + +def test_recovery_without_python_observation_keeps_testing(recovery_main_transport): + state = recovery_main_transport + state["python_runs"] = [] + report = receipts.receipt(None) + publishing.recover_applied("owner/data", state["root"], report) + assert report["state"] == "testing" and report["pr"]["merge"] == state["python"]["head_sha"] + assert state["gate_calls"] == state["parsed"] == state["writes"] == [] + assert state["publisher_reads"] == state["local_checks"] == [] + assert not any("native-feed-post.yml" in route for route in state["reads"]) + + +@pytest.mark.parametrize( + "fault", + ["native-failure", "native-cancelled", "native-attempt", "native-malformed", + "native-missing-artifact", "python-failure", "python-cancelled"], +) +def test_recovery_completed_red_or_malformed_proof_is_not_waiting( + recovery_main_transport, fault +): + state = recovery_main_transport + native = state["native"] + if fault in {"native-failure", "native-cancelled"}: + native["run"]["conclusion"] = fault.removeprefix("native-") + reason = "newest native proof has not completed successfully" + elif fault in {"python-failure", "python-cancelled"}: + state["python"]["conclusion"] = fault.removeprefix("python-") + reason = "CI has not completed successfully" + elif fault == "native-attempt": + native["attestation"]["attempt"] -= 1 + reason = "native: wrong attempt" + elif fault == "native-malformed": + native["attestation"]["legs"] = [] + reason = "native legs: expected an object" + else: + state["native_artifact_missing"] = True + reason = f"missing unique native-evidence-{native['run']['run_attempt']} artifact" + report = receipts.receipt(None) + with pytest.raises(evidence.GateRejected) as rejected: + publishing.recover_applied("owner/data", state["root"], report) + assert str(rejected.value) == reason + assert report["state"] == "pending", "a completed refusal must not become testing" + assert state["gate_calls"] and state["parsed"] + assert state["validated"] == state["local_checks"] == state["publisher_reads"] == [] + assert state["writes"] == [] + + +def test_recovery_complete_pair_reaches_publisher_inventory_boundary(recovery_main_transport): + state = recovery_main_transport + report = receipts.receipt(None) + with pytest.raises(RecoveryPublishersBoundary, match="before publisher-run inventory read"): + publishing.recover_applied("owner/data", state["root"], report) + observed = state["validated"][0] + assert observed["main_proofs"]["native"] == state["native"] + assert observed["main_proofs"]["network"] == state["python_proof"]["main_proofs"]["network"] + assert state["native"]["attestation"] in state["parsed"] + assert state["python_proof"] in state["parsed"] + assert state["local_checks"] == [(state["root"], None)] + assert len(state["publisher_reads"]) == 1 and state["writes"] == [] + # Partial lifecycle only: publisher selection, dispatch, publication and replay are unproved. + + +def test_recovery_complete_pair_still_reads_local_metadata(recovery_main_transport): + state = recovery_main_transport + (state["root"] / contract.DOCKERFILE).write_text( + "ARG PINEFORGE_RELEASE_IMAGE=wrong-runtime-pin\n" + ) + report = receipts.receipt(None) + with pytest.raises(ValueError) as rejected: + publishing.recover_applied("owner/data", state["root"], report) + assert str(rejected.value) == "runtime/server pin mismatch" + assert len(state["validated"]) == 1 + assert state["local_checks"] == state["publisher_reads"] == state["writes"] == [] + + @pytest.mark.parametrize("job_name", [*evidence.JOBS, "release/ready"]) @pytest.mark.parametrize("result", ["failure", "cancelled", "skipped", "missing"]) def test_green_workflow_cannot_hide_a_missing_or_failed_gate(job_name, result) -> None: @@ -443,6 +1000,8 @@ def test_green_workflow_cannot_hide_a_missing_or_failed_gate(job_name, result) - @pytest.fixture def evidence_inputs(tmp_path: Path, monkeypatch): + from release_chain import main_proofs + root = tmp_path / "repository" workflow = root / ".github" / "workflows" / "ci.yml" workflow.parent.mkdir(parents=True) @@ -450,16 +1009,14 @@ def evidence_inputs(tmp_path: Path, monkeypatch): (workflow.parent / "ci-post.yml").write_bytes( (ROOT / ".github/workflows/ci-post.yml").read_bytes() ) - hooks = tmp_path / "empty-hooks" - hooks.mkdir() subprocess.run( - ["git", "-c", "commit.gpgsign=false", "init"], + ["git", "init", "-b", "main"], cwd=root, check=True, capture_output=True, ) subprocess.run( - ["git", "-c", "commit.gpgsign=false", "add", ".github/workflows/ci.yml"], + ["git", "add", ".github/workflows/ci.yml", ".github/workflows/ci-post.yml"], cwd=root, check=True, capture_output=True, @@ -471,10 +1028,6 @@ def evidence_inputs(tmp_path: Path, monkeypatch): "user.name=Release evidence test", "-c", "user.email=release-evidence@example.invalid", - "-c", - "commit.gpgsign=false", - "-c", - f"core.hooksPath={hooks}", "commit", "-m", "Test release evidence", @@ -494,64 +1047,133 @@ def evidence_inputs(tmp_path: Path, monkeypatch): monkeypatch.setenv("GITHUB_RUN_ID", "70") monkeypatch.setenv("GITHUB_RUN_ATTEMPT", "2") monkeypatch.setenv("GITHUB_EVENT_NAME", "push") - needs = {name: {"result": "success"} for name in evidence.NEEDS} - jobs = [{"name": name} for name in evidence.JOBS] + monkeypatch.setenv("GITHUB_REF", "refs/heads/main") + monkeypatch.setenv( + "GITHUB_WORKFLOW_REF", + "pineforge-4pass/pineforge-data/.github/workflows/ci-post.yml@refs/heads/main", + ) + head = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=root, text=True + ).strip() + monkeypatch.setenv("GITHUB_SHA", head) + # Authored unit report bytes, outside the exact two-distribution inventory. + network = tmp_path / "network" + network.mkdir() + suite = ElementTree.Element( + "testsuite", tests="1", failures="0", errors="0", skipped="0" + ) + ElementTree.SubElement( + suite, + "testcase", + classname="tests.test_consumer_receipt", + name="test_bootstrap_recorded_hashes_match_live_pypi", + ) + (network / "network.xml").write_bytes(ElementTree.tostring(suite)) + (network / "network-proof.json").write_bytes( + contract.json_bytes(main_proofs.network_proof(root, network)) + ) + needs = {name: {"result": "success"} for name in evidence.NEEDS | {"network-proof"}} + jobs = [{"name": name} for name in (*evidence.JOBS, "Live PyPI hashes")] def attempt_jobs(repo, run_id, attempt): assert (repo, run_id, attempt) == ("pineforge-4pass/pineforge-data", 70, 2) return jobs monkeypatch.setattr(evidence, "attempt_jobs", attempt_jobs) - return root, directory, needs, jobs + return root, directory, network, needs, jobs @pytest.mark.parametrize("stale_statuses", [False, True], ids=["names-only", "stale-statuses"]) def test_create_evidence_accepts_api_inventory_without_waiting( evidence_inputs, monkeypatch, stale_statuses ) -> None: - root, directory, needs, jobs = evidence_inputs + root, directory, network, needs, jobs = evidence_inputs if stale_statuses: for index, job in enumerate(jobs): job["status"] = "in_progress" if index % 2 == 0 else "queued" + job["conclusion"] = None if index % 2 == 0 else "failure" def reject_sleep(_seconds: float) -> None: raise AssertionError("sleep") monkeypatch.setattr(time, "sleep", reject_sleep) - result = evidence.create_evidence(root, directory, needs) + result = evidence.create_evidence(root, directory, needs, network=network) assert result["jobs"] == dict.fromkeys(evidence.JOBS, "success") + assert result["event"] == "push" and result["pr"] is None and result["base"] is None + assert result["head"] == result["tested_sha"] == subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=root, text=True + ).strip() + assert result["tree"] == subprocess.check_output( + ["git", "rev-parse", "HEAD^{tree}"], cwd=root, text=True + ).strip() + assert result["main_proofs"] == { + "network": contract.object_json((network / "network-proof.json").read_bytes()) + } + assert result["main_proofs"]["network"]["junit_sha256"] == contract.sha256( + (network / "network.xml").read_bytes() + ) + assert result["distributions"] == { + "pineforge_data-0.0.0-py3-none-any.whl": contract.sha256(b"wheel"), + "pineforge_data-0.0.0.tar.gz": contract.sha256(b"sdist"), + } + assert network.parent == directory.parent and network != directory assert not hasattr(evidence, "JOBS_CLOSE_TIMEOUT") assert not hasattr(evidence, "closed_jobs") -@pytest.mark.parametrize("job_id", sorted(evidence.NEEDS)) +@pytest.mark.parametrize("job_id", sorted(evidence.NEEDS | {"network-proof"})) @pytest.mark.parametrize("result", ["failure", "cancelled", "skipped"]) def test_create_evidence_rejects_unsuccessful_needs(evidence_inputs, job_id, result) -> None: - root, directory, needs, _jobs = evidence_inputs + root, directory, network, needs, _jobs = evidence_inputs needs[job_id]["result"] = result with pytest.raises(ValueError, match="a required CI job failed"): - evidence.create_evidence(root, directory, needs) + evidence.create_evidence(root, directory, needs, network=network) -def test_create_evidence_rejects_missing_need(evidence_inputs) -> None: - root, directory, needs, _jobs = evidence_inputs - needs.pop("test") +@pytest.mark.parametrize("job_id", ["test", "network-proof"]) +def test_create_evidence_rejects_missing_need(evidence_inputs, job_id) -> None: + root, directory, network, needs, _jobs = evidence_inputs + needs.pop(job_id) with pytest.raises(ValueError, match="required CI job inventory changed"): - evidence.create_evidence(root, directory, needs) + evidence.create_evidence(root, directory, needs, network=network) -@pytest.mark.parametrize("job_name", evidence.JOBS) +@pytest.mark.parametrize("job_name", [*evidence.JOBS, "Live PyPI hashes"]) @pytest.mark.parametrize("defect", ["missing", "duplicated"]) def test_create_evidence_rejects_missing_or_duplicated_api_names( evidence_inputs, job_name, defect ) -> None: - root, directory, needs, jobs = evidence_inputs + root, directory, network, needs, jobs = evidence_inputs if defect == "missing": jobs[:] = [job for job in jobs if job["name"] != job_name] else: jobs.append({"name": job_name}) with pytest.raises(ValueError, match="missing or duplicated"): - evidence.create_evidence(root, directory, needs) + evidence.create_evidence(root, directory, needs, network=network) + + +def test_create_evidence_main_requires_network_input(evidence_inputs) -> None: + root, directory, _network, needs, _jobs = evidence_inputs + with pytest.raises(ValueError, match="main requires the actual live-PyPI report"): + evidence.create_evidence(root, directory, needs, network=None) + + +@pytest.mark.parametrize("fault", ["raw-report", "recorded-digest", "recorded-attempt"]) +def test_create_evidence_consumes_network_report_bytes(evidence_inputs, fault) -> None: + root, directory, network, needs, _jobs = evidence_inputs + if fault == "raw-report": + report = network / "network.xml" + report.write_bytes(report.read_bytes() + b"\n") + else: + path = network / "network-proof.json" + proof = contract.object_json(path.read_bytes()) + if fault == "recorded-digest": + proof["junit_sha256"] = contract.sha256(b"different raw report") + else: + proof["attempt"] -= 1 + path.write_bytes(contract.json_bytes(proof)) + with pytest.raises(ValueError, match="network report bytes or producer identity changed"): + evidence.create_evidence(root, directory, needs, network=network) @pytest.mark.parametrize( @@ -569,10 +1191,10 @@ def test_create_evidence_rejects_missing_or_duplicated_api_names( ids=["enabled", "comment", "escaped-key", "whitespace-only"], ) def test_create_evidence_rejects_continue_on_error(evidence_inputs, workflow) -> None: - root, directory, needs, _jobs = evidence_inputs + root, directory, network, needs, _jobs = evidence_inputs (root / ".github" / "workflows" / "ci.yml").write_text(workflow) with pytest.raises(ValueError, match=r"ci\.yml changed or uses continue-on-error"): - evidence.create_evidence(root, directory, needs) + evidence.create_evidence(root, directory, needs, network=network) @pytest.mark.parametrize( @@ -593,10 +1215,10 @@ def test_reviewed_ci_workflow_pin_matches_the_file(filename: str, pin: str) -> N def test_create_evidence_rejects_missing_workflow(evidence_inputs) -> None: - root, directory, needs, _jobs = evidence_inputs + root, directory, network, needs, _jobs = evidence_inputs (root / ".github" / "workflows" / "ci.yml").unlink() with pytest.raises(ValueError, match=r"cannot read ci\.yml"): - evidence.create_evidence(root, directory, needs) + evidence.create_evidence(root, directory, needs, network=network) @pytest.mark.parametrize( @@ -1167,6 +1789,223 @@ def test_completed_receipt_replay_does_not_dispatch_or_publish( assert report["deployed"] == saved["deployed"] +@pytest.fixture +def saved_recovery_transport(ready, baseline, monkeypatch): + """Real saved-package/lock/OCI verification over immutable fixture bytes.""" + event, manifest = ready + tagged = baseline / "tagged-source" + baseline_files = { + name: (baseline / name).read_bytes() + for name in (contract.RUNTIME, contract.DOCKERFILE, "pyproject.toml") + } + apply(tagged, baseline_files) + apply(tagged, contract.update_files(tagged, event, manifest)) + source_files = {name: (tagged / name).read_bytes() for name in contract.UPDATE_FILES} + # The installed original pin intentionally has no lock or manifest. + # Its package version identifies the stored receipt without claiming a new pin. + (baseline / "pyproject.toml").write_bytes(source_files["pyproject.toml"]) + archive = io.BytesIO() + with zipfile.ZipFile(archive, "w") as zipped: + for name in (contract.LOCK, contract.MANIFEST, contract.RUNTIME): + zipped.writestr(name.removeprefix("src/"), source_files[name]) + downloads = { + "pineforge_data-0.3.1-py3-none-any.whl": archive.getvalue(), + "pineforge_data-0.3.1.tar.gz": b"retained source distribution fixture", + } + hashes = {name: contract.sha256(raw) for name, raw in downloads.items()} + metadata = {"urls": [ + { + "filename": name, "digests": {"sha256": digest}, + "url": f"https://files.pythonhosted.org/{name}", + } + for name, digest in hashes.items() + ]} + saved = contract.object_json((FIXTURES / "sample-release-receipt.json").read_bytes()) + source = saved["pr"]["merge"] + saved.update( + event_id=event["event_id"], release_version=event["release_version"], + deployed=receipts.served(event, source, "0.3.1", hashes), + ) + assets = [ + {"id": index, "name": name} + for index, name in enumerate([*downloads, "release-receipt.json"], 1) + ] + release = {"tag_name": "v0.3.1", "assets": assets} + state = { + "root": baseline, "baseline_files": baseline_files, + "source_files": source_files, "saved": saved, + "event": event, "downloads": downloads, "reads": [], "git": [], + "steps": [], "package_calls": [], "local_reads": [], "local_results": [], + "processes": [], "snapshot_files": [], + } + registry_fixture(monkeypatch, event) + + def api(repo, endpoint, method="GET", data=None): + state["reads"].append(endpoint) + assert repo == "owner/data" and method == "GET" and data is None + if endpoint == "releases?per_page=100&page=1": + return [copy.deepcopy(release)] + assert endpoint == "releases/tags/v0.3.1" + return copy.deepcopy(release) + + def gh_bytes(endpoint, *, accept): + state["reads"].append(endpoint) + assert accept == "application/octet-stream" + asset = next( + item for item in assets + if endpoint == f"repos/owner/data/releases/assets/{item['id']}" + ) + if asset["name"] == "release-receipt.json": + return receipts.canonical(saved) + return downloads[asset["name"]] + + def fetch(url, headers=None): + state["reads"].append(url) + assert headers is None + if url == "https://pypi.org/pypi/pineforge-data/0.3.1/json": + return contract.json_bytes(metadata) + assert url.startswith("https://files.pythonhosted.org/") + return downloads[url.rsplit("/", 1)[1]] + + def git(*args): + state["git"].append(args) + if args == ("fetch", "origin", "refs/tags/v0.3.1"): + return "" + assert args == ("rev-parse", "v0.3.1^{commit}") + return source + + def snapshot(revision, root): + assert revision == source + paths = [root / name for name in sorted(source_files)] + for path in paths: + if ( + path.name.endswith(".env") + or not path.resolve().is_relative_to(baseline.resolve()) + ): + raise AssertionError(f"refused saved publication fixture file: {path}") + state["snapshot_files"].append([path.relative_to(root).as_posix() for path in paths]) + apply(root, source_files) + + real_verify_local = publishing.verify_local + real_verified_package = publishing.verified_package + real_temporary_directory = publishing.tempfile.TemporaryDirectory + + def observe_local(root): + state["steps"].append(("verify-local", root)) + state["local_reads"].append(root) + result = real_verify_local(root) + state["local_results"].append(copy.deepcopy(result)) + return result + + def observe_package(repo, version, stored): + state["package_calls"].append((repo, version, copy.deepcopy(stored))) + state["steps"].append(("verify-package-start", version)) + result = real_verified_package(repo, version, stored) + state["steps"].append(("verify-package-complete", version)) + return result + + def reject_process(command, **kwargs): + state["processes"].append(command) + raise AssertionError("saved receipt replay must not dispatch or publish") + + monkeypatch.setattr(automation, "api", api) + monkeypatch.setattr(publishing, "api", api) + monkeypatch.setattr(publishing, "gh_bytes", gh_bytes) + monkeypatch.setattr(publishing, "fetch", fetch) + monkeypatch.setattr(publishing, "git", git) + monkeypatch.setattr(publishing, "snapshot", snapshot) + # These observers return only the result of the complete original validators. + monkeypatch.setattr(publishing, "verify_local", observe_local) + monkeypatch.setattr(publishing, "verified_package", observe_package) + monkeypatch.setattr(publishing.subprocess, "run", reject_process) + monkeypatch.setattr( + publishing.tempfile, "TemporaryDirectory", + lambda: real_temporary_directory(dir=baseline), + ) + return state + + +def assert_saved_recovery_verification_order(state): + assert state["package_calls"] == [("owner/data", "0.3.1", state["saved"])] + assert len(state["local_reads"]) == len(state["local_results"]) == 2 + assert state["local_reads"][0] != state["root"] + assert state["local_reads"][1] == state["root"] + assert state["local_results"][0]["event"] == state["event"] + assert state["steps"][-2:] == [ + ("verify-package-complete", "0.3.1"), ("verify-local", state["root"]), + ] + assert state["git"] == [ + ("fetch", "origin", "refs/tags/v0.3.1"), ("rev-parse", "v0.3.1^{commit}"), + ] + assert state["reads"].count("https://pypi.org/pypi/pineforge-data/0.3.1/json") == 2 + assert state["snapshot_files"] == [sorted(contract.UPDATE_FILES)] + assert state["processes"] == [] + + +def test_recovery_saved_receipt_refuses_missing_installed_lock(saved_recovery_transport): + state = saved_recovery_transport + assert contract.verify_local(state["root"]) is None + report = receipts.receipt(state["event"], "reconcile") + before = copy.deepcopy(report) + with pytest.raises(ValueError, match=r"^stored publication source lock missing$"): + publishing.recover_applied("owner/data", state["root"], report) + assert_saved_recovery_verification_order(state) + assert state["local_results"][1] is None + assert report == before + + +def test_recovery_saved_receipt_replays_after_real_verification(saved_recovery_transport): + state = saved_recovery_transport + apply(state["root"], state["source_files"]) + report = receipts.receipt(state["event"], "reconcile") + publishing.recover_applied("owner/data", state["root"], report) + assert_saved_recovery_verification_order(state) + assert state["local_results"][1] == state["local_results"][0] + assert report == state["saved"] + + +def test_recovery_saved_receipt_keeps_installed_image_mismatch_refusal(saved_recovery_transport): + state = saved_recovery_transport + manifest = contract.object_json(state["source_files"][contract.MANIFEST]) + manifest["image"]["digest"] = "sha256:" + "0" * 64 + raw = contract.json_bytes(manifest) + installed_event = contract.event_from_manifest(raw) + apply(state["root"], state["baseline_files"]) + apply(state["root"], contract.update_files(state["root"], installed_event, raw)) + report = receipts.receipt(state["event"], "reconcile") + before = copy.deepcopy(report) + with pytest.raises(ValueError, match=r"^stored publication does not match installed defaults$"): + publishing.recover_applied("owner/data", state["root"], report) + assert_saved_recovery_verification_order(state) + assert state["local_results"][1]["event"] == installed_event + assert report == before + + +def test_recovery_keeps_stored_identity_priority_before_local_lock(saved_recovery_transport): + state = saved_recovery_transport + state["saved"]["event_id"] = "sha256:" + "0" * 64 + report = receipts.receipt(state["event"], "reconcile") + before = copy.deepcopy(report) + with pytest.raises(ValueError, match=r"^stored publication identity differs$"): + publishing.recover_applied("owner/data", state["root"], report) + assert state["package_calls"] == state["local_reads"] == state["git"] == [] + assert state["processes"] == [] + assert report == before + + +def test_recovery_verifies_package_before_requiring_local_lock(saved_recovery_transport): + state = saved_recovery_transport + name = "pineforge_data-0.3.1-py3-none-any.whl" + state["downloads"][name] += b"changed" + report = receipts.receipt(state["event"], "reconcile") + before = copy.deepcopy(report) + with pytest.raises(ValueError, match=r"^PyPI bytes differ$"): + publishing.recover_applied("owner/data", state["root"], report) + assert len(state["package_calls"]) == 1 + assert state["local_reads"] == state["git"] == state["processes"] == [] + assert report == before + + def assert_pull_request_filter_policy(workflow: dict[str, Any], name: str) -> None: assert "pull_request" in workflow["on"] assert "pull_request_target" not in workflow["on"] @@ -1293,7 +2132,7 @@ def test_workflows_keep_every_gate_ahead_of_oidc_publication() -> None: else: assert ci["jobs"][job]["env"]["PINEFORGE_DATABASE_E2E"] == "1" release = yaml.load((workflows / "release.yml").read_text(), Loader=yaml.BaseLoader) - assert release["on"]["workflow_run"]["workflows"] == ["CI"] + assert release["on"]["workflow_run"]["workflows"] == ["CI", "Native feed"] # old: assert set(release["jobs"]["publish"]["needs"]) == {"prepare", "build", "github-release"} assert set(release["jobs"]["publish"]["needs"]) == { "authz",