diff --git a/.github/scripts/check_simulation_entrypoint_environment.sh b/.github/scripts/check_simulation_entrypoint_environment.sh new file mode 100755 index 000000000..7b6555949 --- /dev/null +++ b/.github/scripts/check_simulation_entrypoint_environment.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash + +# Check that a deployment environment calls its own Simulation Entrypoint: +# staging must call the staging service and production the production +# service. The entrypoint names the Cloud Run revision that served a request +# in its x-policyengine-simulation-revision response header, and a Cloud Run +# revision name is its service name followed by a five-digit number and a +# three-character suffix. +# +# The old gateway (SIM_ENTRYPOINT=old_gateway_direct) is one shared Modal app +# that reports no environment, so it is not checked. +# +# Required env vars: DEPLOYMENT_ENVIRONMENT (staging or production), +# SIM_ENTRYPOINT, and the URL variable that SIM_ENTRYPOINT selects. +# Exit codes: 0 when the environments match, 1 when they do not or the +# response names no revision, 2 for an unknown DEPLOYMENT_ENVIRONMENT, and +# curl's own exit code when the request fails. + +set -euo pipefail + +source .github/scripts/simulation_entrypoint_env.sh + +deployment_environment="${DEPLOYMENT_ENVIRONMENT:-}" +case "${deployment_environment}" in + staging) + expected_service="policyengine-simulation-entry-staging" + ;; + production) + expected_service="policyengine-simulation-entry" + ;; + *) + printf 'DEPLOYMENT_ENVIRONMENT must be staging or production, not %q\n' \ + "${deployment_environment}" >&2 + exit 2 + ;; +esac + +if [[ "${SIM_ENTRYPOINT:-}" == "old_gateway_direct" ]]; then + printf 'SIM_ENTRYPOINT=old_gateway_direct: the shared old gateway reports no environment, so there is nothing to check\n' + exit 0 +fi + +simulation_url="$(simulation_entrypoint_selected_url "${SIM_ENTRYPOINT:-}")" +simulation_url="${simulation_url%/}" + +health_response="$( + curl --fail --silent --show-error --include \ + --connect-timeout 10 --max-time 60 \ + "${simulation_url}/health" +)" + +# With --include, curl prints every response's headers; the last revision +# header belongs to the final response. +served_revision="$( + awk ' + tolower($1) == "x-policyengine-simulation-revision:" { + gsub("\r", "", $2) + print $2 + } + ' <<<"${health_response}" | + tail -n 1 +)" + +if [[ -z "${served_revision}" ]]; then + printf '%s/health did not name a Simulation Entrypoint revision; the %s deployment must call the stable URL of %s\n' \ + "${simulation_url}" "${deployment_environment}" "${expected_service}" >&2 + exit 1 +fi + +if ! [[ "${served_revision}" =~ ^(.+)-[0-9]{5}-[a-z0-9]{3}$ ]]; then + printf '%s, served by %s, is not a Cloud Run revision name\n' \ + "${served_revision}" "${simulation_url}" >&2 + exit 1 +fi +served_service="${BASH_REMATCH[1]}" + +if [[ "${served_service}" != "${expected_service}" ]]; then + printf 'The %s deployment calls %s, which revision %s of %s serves; it must call %s\n' \ + "${deployment_environment}" "${simulation_url}" "${served_revision}" \ + "${served_service}" "${expected_service}" >&2 + exit 1 +fi + +printf 'The %s deployment calls %s, served by revision %s of %s\n' \ + "${deployment_environment}" "${simulation_url}" "${served_revision}" \ + "${served_service}" diff --git a/.github/workflows/push.yml b/.github/workflows/push.yml index af02524d3..e8007f472 100644 --- a/.github/workflows/push.yml +++ b/.github/workflows/push.yml @@ -73,11 +73,14 @@ jobs: && (github.event.head_commit.message == 'Update PolicyEngine API') environment: staging env: + DEPLOYMENT_ENVIRONMENT: staging SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }} SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} steps: - name: Checkout repo uses: actions/checkout@v4 + - name: Check the simulation service belongs to staging + run: bash .github/scripts/check_simulation_entrypoint_environment.sh - name: Install jq run: sudo apt-get install -y jq - name: Set up uv for installed bundle validation @@ -438,11 +441,14 @@ jobs: && (github.event.head_commit.message == 'Update PolicyEngine API') environment: production env: + DEPLOYMENT_ENVIRONMENT: production SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }} SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }} steps: - name: Checkout repo uses: actions/checkout@v4 + - name: Check the simulation service belongs to production + run: bash .github/scripts/check_simulation_entrypoint_environment.sh - name: Install jq run: sudo apt-get install -y jq - name: Set up uv for installed bundle validation diff --git a/changelog.d/simulation-entrypoint-environment-check.added.md b/changelog.d/simulation-entrypoint-environment-check.added.md new file mode 100644 index 000000000..2e592c45b --- /dev/null +++ b/changelog.d/simulation-entrypoint-environment-check.added.md @@ -0,0 +1 @@ +Each release now checks, in its staging and production model-version alignment jobs, that the environment's simulation URL is served by that environment's own Simulation Entrypoint service, and fails otherwise; staging must call the staging simulation service, and production the production one. diff --git a/docs/migration/cloud-run-operations.md b/docs/migration/cloud-run-operations.md index a69209d24..4815630e6 100644 --- a/docs/migration/cloud-run-operations.md +++ b/docs/migration/cloud-run-operations.md @@ -259,6 +259,21 @@ not block a direct-Modal deployment. The legacy `SIMULATION_API_URL` secret is intentionally unsupported: its value is opaque, cannot be audited, and must not silently choose a deployment upstream. +Each environment calls its own Simulation Entrypoint. The `staging` secret +holds the stable URL of the `policyengine-simulation-entry-staging` Cloud Run +service, which calls the staging Modal environment; the `production` secret +holds `https://simulation.api.policyengine.org`, served by +`policyengine-simulation-entry`. In each release, the staging and production +model-version alignment jobs first run +`.github/scripts/check_simulation_entrypoint_environment.sh`. It reads the +`x-policyengine-simulation-revision` header from the selected URL's `/health` +response and fails the release unless the revision belongs to that +environment's service, so a release whose `staging` secret points at +production, or the reverse, stops before it deploys. Staging's live simulation +tests therefore run against the staging simulation service and staging Modal +workers. The old gateway is one shared Modal app that reports no environment, +so `old_gateway_direct` is not checked. + ## IAM and bootstrap constraints - The GitHub deploy service account holds `roles/run.developer`: it can deploy to diff --git a/tests/unit/test_simulation_entrypoint_environment_script.py b/tests/unit/test_simulation_entrypoint_environment_script.py new file mode 100644 index 000000000..3f5ca8e04 --- /dev/null +++ b/tests/unit/test_simulation_entrypoint_environment_script.py @@ -0,0 +1,256 @@ +"""Tests for .github/scripts/check_simulation_entrypoint_environment.sh.""" + +from __future__ import annotations + +import os +import subprocess +from dataclasses import dataclass +from pathlib import Path + +import pytest +import yaml + +REPO = Path(__file__).resolve().parents[2] +SCRIPT = REPO / ".github/scripts/check_simulation_entrypoint_environment.sh" +PUSH_WORKFLOW = REPO / ".github/workflows/push.yml" +CHECK_COMMAND = "bash .github/scripts/check_simulation_entrypoint_environment.sh" +SIMULATION_URL = "https://simulation.example.test" +STAGING_REVISION = "policyengine-simulation-entry-staging-00070-abc" +PRODUCTION_REVISION = "policyengine-simulation-entry-00122-low" +HEALTHY_BODY = '{"status":"healthy"}' + + +@dataclass(frozen=True) +class FakeCurlReply: + """What the fake curl prints, and the exit code it returns.""" + + output: str + exit_code: int = 0 + + +# Cloud Run's sample container answers with an HTML page and no revision header. +SAMPLE_CONTAINER_RESPONSE = FakeCurlReply( + "HTTP/2 200\r\ncontent-type: text/html; charset=utf-8\r\n\r\n" +) +# curl --fail exits with 22 when the server answers with an HTTP error. +FAILED_REQUEST = FakeCurlReply("", exit_code=22) + + +def _entrypoint_response(revision: str) -> FakeCurlReply: + return FakeCurlReply( + "HTTP/2 200\r\n" + "content-type: application/json\r\n" + f"x-policyengine-simulation-revision: {revision}\r\n" + "\r\n" + f"{HEALTHY_BODY}" + ) + + +def _install_fake_curl(tmp_path: Path, reply: FakeCurlReply) -> Path: + """Install a fake curl that records its arguments and gives ``reply``.""" + fake_bin_directory = tmp_path / "bin" + fake_bin_directory.mkdir() + response_file = tmp_path / "response.txt" + response_file.write_text(reply.output, encoding="utf-8") + calls_file = tmp_path / "curl-calls.log" + fake_curl = fake_bin_directory / "curl" + fake_curl.write_text( + "#!/usr/bin/env bash\n" + f'echo "$*" >> "{calls_file}"\n' + f'cat "{response_file}"\n' + f"exit {reply.exit_code}\n", + encoding="utf-8", + ) + fake_curl.chmod(0o755) + return fake_bin_directory + + +def _run_check( + tmp_path: Path, + curl_reply: FakeCurlReply, + *, + deployment_environment: str, + sim_entrypoint: str = "cloud_run_simulation_entrypoint", + simulation_url: str = SIMULATION_URL, +) -> tuple[subprocess.CompletedProcess[str], list[str]]: + fake_bin_directory = _install_fake_curl(tmp_path, curl_reply) + result = subprocess.run( + ["bash", str(SCRIPT)], + cwd=REPO, + env={ + "PATH": f"{fake_bin_directory}{os.pathsep}{os.environ['PATH']}", + "DEPLOYMENT_ENVIRONMENT": deployment_environment, + "SIM_ENTRYPOINT": sim_entrypoint, + "SIMULATION_ENTRYPOINT_URL": simulation_url, + "OLD_SIMULATION_GATEWAY_URL": "https://old-gateway.example.test", + }, + text=True, + capture_output=True, + check=False, + ) + calls_file = tmp_path / "curl-calls.log" + curl_calls = calls_file.read_text().splitlines() if calls_file.exists() else [] + return result, curl_calls + + +@pytest.mark.parametrize( + ("deployment_environment", "revision"), + [("staging", STAGING_REVISION), ("production", PRODUCTION_REVISION)], +) +def test_accepts_the_simulation_service_of_the_same_environment( + tmp_path: Path, deployment_environment: str, revision: str +) -> None: + result, curl_calls = _run_check( + tmp_path, + _entrypoint_response(revision), + deployment_environment=deployment_environment, + ) + + assert result.returncode == 0, result.stderr + assert len(curl_calls) == 1 + assert curl_calls[0].endswith(f"{SIMULATION_URL}/health") + assert revision in result.stdout + + +def test_requests_health_once_without_a_doubled_slash(tmp_path: Path) -> None: + result, curl_calls = _run_check( + tmp_path, + _entrypoint_response(STAGING_REVISION), + deployment_environment="staging", + simulation_url=f"{SIMULATION_URL}/", + ) + + assert result.returncode == 0, result.stderr + assert curl_calls[0].endswith(f" {SIMULATION_URL}/health") + + +def test_rejects_staging_calling_the_production_simulation_service( + tmp_path: Path, +) -> None: + result, _ = _run_check( + tmp_path, + _entrypoint_response(PRODUCTION_REVISION), + deployment_environment="staging", + ) + + assert result.returncode == 1 + assert "it must call policyengine-simulation-entry-staging" in result.stderr + assert PRODUCTION_REVISION in result.stderr + + +def test_rejects_production_calling_the_staging_simulation_service( + tmp_path: Path, +) -> None: + result, _ = _run_check( + tmp_path, + _entrypoint_response(STAGING_REVISION), + deployment_environment="production", + ) + + assert result.returncode == 1 + assert "it must call policyengine-simulation-entry\n" in result.stderr + assert STAGING_REVISION in result.stderr + + +def test_rejects_a_url_that_names_no_entrypoint_revision(tmp_path: Path) -> None: + result, _ = _run_check( + tmp_path, SAMPLE_CONTAINER_RESPONSE, deployment_environment="staging" + ) + + assert result.returncode == 1 + assert "did not name a Simulation Entrypoint revision" in result.stderr + + +def test_rejects_a_revision_name_without_a_cloud_run_suffix(tmp_path: Path) -> None: + result, _ = _run_check( + tmp_path, + _entrypoint_response("policyengine-simulation-entry-staging"), + deployment_environment="staging", + ) + + assert result.returncode == 1 + assert "is not a Cloud Run revision name" in result.stderr + + +def test_fails_when_the_health_request_fails(tmp_path: Path) -> None: + result, curl_calls = _run_check( + tmp_path, FAILED_REQUEST, deployment_environment="staging" + ) + + assert result.returncode == 22 + assert len(curl_calls) == 1 + + +def test_rejects_an_unknown_deployment_environment(tmp_path: Path) -> None: + result, curl_calls = _run_check( + tmp_path, + _entrypoint_response(STAGING_REVISION), + deployment_environment="preview", + ) + + assert result.returncode == 2 + assert "DEPLOYMENT_ENVIRONMENT must be staging or production" in result.stderr + assert curl_calls == [] + + +def test_rejects_a_missing_simulation_url(tmp_path: Path) -> None: + result, curl_calls = _run_check( + tmp_path, + _entrypoint_response(STAGING_REVISION), + deployment_environment="staging", + simulation_url="", + ) + + assert result.returncode == 1 + assert "SIMULATION_ENTRYPOINT_URL is required" in result.stderr + assert curl_calls == [] + + +def test_does_not_check_the_shared_old_gateway(tmp_path: Path) -> None: + result, curl_calls = _run_check( + tmp_path, + _entrypoint_response(PRODUCTION_REVISION), + deployment_environment="staging", + sim_entrypoint="old_gateway_direct", + ) + + assert result.returncode == 0, result.stderr + assert curl_calls == [] + assert "old_gateway_direct" in result.stdout + + +@pytest.mark.parametrize( + ("job_name", "deployment_environment"), + [ + ("ensure-staging-model-version-aligns-with-sim-api", "staging"), + ("ensure-production-model-version-aligns-with-sim-api", "production"), + ], +) +def test_each_release_checks_its_own_simulation_service_first( + job_name: str, deployment_environment: str +) -> None: + """Each version-alignment job fails if it reaches the other environment. + + The check runs right after checkout, before the job asks the simulation + service which bundle it supports, so a crossed environment fails the + release instead of validating against the wrong service. + """ + job = yaml.safe_load(PUSH_WORKFLOW.read_text(encoding="utf-8"))["jobs"][job_name] + + assert job["environment"] == deployment_environment + assert job["env"]["DEPLOYMENT_ENVIRONMENT"] == deployment_environment + assert job["env"]["SIM_ENTRYPOINT"] == "${{ vars.SIM_ENTRYPOINT }}" + assert ( + job["env"]["SIMULATION_ENTRYPOINT_URL"] + == "${{ secrets.SIMULATION_ENTRYPOINT_URL }}" + ) + step_commands = [step.get("run", "") for step in job["steps"]] + assert step_commands.count(CHECK_COMMAND) == 1 + check_position = step_commands.index(CHECK_COMMAND) + bundle_check_position = step_commands.index( + "bash .github/check-policyengine-bundle-supported.sh" + ) + assert ( + job["steps"][check_position - 1].get("uses", "").startswith("actions/checkout@") + ) + assert check_position < bundle_check_position