Check each release calls its own environment's Simulation Entrypoint

.github/scripts/check_simulation_entrypoint_environment.sh

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+# Check that a deployment environment calls its own Simulation Entrypoint:
+# staging must call the staging service and production the production
+# service. The entrypoint names the Cloud Run revision that served a request
+# in its x-policyengine-simulation-revision response header, and a Cloud Run
+# revision name is its service name followed by a five-digit number and a
+# three-character suffix.
+#
+# The old gateway (SIM_ENTRYPOINT=old_gateway_direct) is one shared Modal app
+# that reports no environment, so it is not checked.
+#
+# Required env vars: DEPLOYMENT_ENVIRONMENT (staging or production),
+# SIM_ENTRYPOINT, and the URL variable that SIM_ENTRYPOINT selects.
+# Exit codes: 0 when the environments match, 1 when they do not or the
+# response names no revision, 2 for an unknown DEPLOYMENT_ENVIRONMENT, and
+# curl's own exit code when the request fails.
+
+set -euo pipefail
+
+source .github/scripts/simulation_entrypoint_env.sh
+
+deployment_environment="${DEPLOYMENT_ENVIRONMENT:-}"
+case "${deployment_environment}" in
+  staging)
+    expected_service="policyengine-simulation-entry-staging"
+    ;;
+  production)
+    expected_service="policyengine-simulation-entry"
+    ;;
+  *)
+    printf 'DEPLOYMENT_ENVIRONMENT must be staging or production, not %q\n' \
+      "${deployment_environment}" >&2
+    exit 2
+    ;;
+esac
+
+if [[ "${SIM_ENTRYPOINT:-}" == "old_gateway_direct" ]]; then
+  printf 'SIM_ENTRYPOINT=old_gateway_direct: the shared old gateway reports no environment, so there is nothing to check\n'
+  exit 0
+fi
+
+simulation_url="$(simulation_entrypoint_selected_url "${SIM_ENTRYPOINT:-}")"
+simulation_url="${simulation_url%/}"
+
+health_response="$(
+  curl --fail --silent --show-error --include \
+    --connect-timeout 10 --max-time 60 \
+    "${simulation_url}/health"
+)"
+
+# With --include, curl prints every response's headers; the last revision
+# header belongs to the final response.
+served_revision="$(
+  awk '
+    tolower($1) == "x-policyengine-simulation-revision:" {
+      gsub("\r", "", $2)
+      print $2
+    }
+  ' <<<"${health_response}" |
+    tail -n 1
+)"
+
+if [[ -z "${served_revision}" ]]; then
+  printf '%s/health did not name a Simulation Entrypoint revision; the %s deployment must call the stable URL of %s\n' \
+    "${simulation_url}" "${deployment_environment}" "${expected_service}" >&2
+  exit 1
+fi
+
+if ! [[ "${served_revision}" =~ ^(.+)-[0-9]{5}-[a-z0-9]{3}$ ]]; then
+  printf '%s, served by %s, is not a Cloud Run revision name\n' \
+    "${served_revision}" "${simulation_url}" >&2
+  exit 1
+fi
+served_service="${BASH_REMATCH[1]}"
+
+if [[ "${served_service}" != "${expected_service}" ]]; then
+  printf 'The %s deployment calls %s, which revision %s of %s serves; it must call %s\n' \
+    "${deployment_environment}" "${simulation_url}" "${served_revision}" \
+    "${served_service}" "${expected_service}" >&2
+  exit 1
+fi
+
+printf 'The %s deployment calls %s, served by revision %s of %s\n' \
+  "${deployment_environment}" "${simulation_url}" "${served_revision}" \
+  "${served_service}"

.github/workflows/push.yml

@@ -73,11 +73,14 @@ jobs:
       && (github.event.head_commit.message == 'Update PolicyEngine API')
     environment: staging
     env:
+      DEPLOYMENT_ENVIRONMENT: staging
       SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }}
       SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
+      - name: Check the simulation service belongs to staging
+        run: bash .github/scripts/check_simulation_entrypoint_environment.sh
       - name: Install jq
         run: sudo apt-get install -y jq
       - name: Set up uv for installed bundle validation
@@ -438,11 +441,14 @@ jobs:
       && (github.event.head_commit.message == 'Update PolicyEngine API')
     environment: production
     env:
+      DEPLOYMENT_ENVIRONMENT: production
       SIM_ENTRYPOINT: ${{ vars.SIM_ENTRYPOINT }}
       SIMULATION_ENTRYPOINT_URL: ${{ secrets.SIMULATION_ENTRYPOINT_URL }}
     steps:
       - name: Checkout repo
         uses: actions/checkout@v4
+      - name: Check the simulation service belongs to production
+        run: bash .github/scripts/check_simulation_entrypoint_environment.sh
       - name: Install jq
         run: sudo apt-get install -y jq
       - name: Set up uv for installed bundle validation

changelog.d/simulation-entrypoint-environment-check.added.md

@@ -0,0 +1 @@
+Each release now checks, in its staging and production model-version alignment jobs, that the environment's simulation URL is served by that environment's own Simulation Entrypoint service, and fails otherwise; staging must call the staging simulation service, and production the production one.

docs/migration/cloud-run-operations.md

@@ -259,6 +259,21 @@ not block a direct-Modal deployment. The legacy `SIMULATION_API_URL` secret is
 intentionally unsupported: its value is opaque, cannot be audited, and must
 not silently choose a deployment upstream.
 
+Each environment calls its own Simulation Entrypoint. The `staging` secret
+holds the stable URL of the `policyengine-simulation-entry-staging` Cloud Run
+service, which calls the staging Modal environment; the `production` secret
+holds `https://simulation.api.policyengine.org`, served by
+`policyengine-simulation-entry`. In each release, the staging and production
+model-version alignment jobs first run
+`.github/scripts/check_simulation_entrypoint_environment.sh`. It reads the
+`x-policyengine-simulation-revision` header from the selected URL's `/health`
+response and fails the release unless the revision belongs to that
+environment's service, so a release whose `staging` secret points at
+production, or the reverse, stops before it deploys. Staging's live simulation
+tests therefore run against the staging simulation service and staging Modal
+workers. The old gateway is one shared Modal app that reports no environment,
+so `old_gateway_direct` is not checked.
+
 ## IAM and bootstrap constraints
 
 - The GitHub deploy service account holds `roles/run.developer`: it can deploy to

tests/unit/test_simulation_entrypoint_environment_script.py

@@ -0,0 +1,256 @@
+"""Tests for .github/scripts/check_simulation_entrypoint_environment.sh."""
+
+from __future__ import annotations
+
+import os
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+
+import pytest
+import yaml
+
+REPO = Path(__file__).resolve().parents[2]
+SCRIPT = REPO / ".github/scripts/check_simulation_entrypoint_environment.sh"
+PUSH_WORKFLOW = REPO / ".github/workflows/push.yml"
+CHECK_COMMAND = "bash .github/scripts/check_simulation_entrypoint_environment.sh"
+SIMULATION_URL = "https://simulation.example.test"
+STAGING_REVISION = "policyengine-simulation-entry-staging-00070-abc"
+PRODUCTION_REVISION = "policyengine-simulation-entry-00122-low"
+HEALTHY_BODY = '{"status":"healthy"}'
+
+
+@dataclass(frozen=True)
+class FakeCurlReply:
+    """What the fake curl prints, and the exit code it returns."""
+
+    output: str
+    exit_code: int = 0
+
+
+# Cloud Run's sample container answers with an HTML page and no revision header.
+SAMPLE_CONTAINER_RESPONSE = FakeCurlReply(
+    "HTTP/2 200\r\ncontent-type: text/html; charset=utf-8\r\n\r\n<html></html>"
+)
+# curl --fail exits with 22 when the server answers with an HTTP error.
+FAILED_REQUEST = FakeCurlReply("", exit_code=22)
+
+
+def _entrypoint_response(revision: str) -> FakeCurlReply:
+    return FakeCurlReply(
+        "HTTP/2 200\r\n"
+        "content-type: application/json\r\n"
+        f"x-policyengine-simulation-revision: {revision}\r\n"
+        "\r\n"
+        f"{HEALTHY_BODY}"
+    )
+
+
+def _install_fake_curl(tmp_path: Path, reply: FakeCurlReply) -> Path:
+    """Install a fake curl that records its arguments and gives ``reply``."""
+    fake_bin_directory = tmp_path / "bin"
+    fake_bin_directory.mkdir()
+    response_file = tmp_path / "response.txt"
+    response_file.write_text(reply.output, encoding="utf-8")
+    calls_file = tmp_path / "curl-calls.log"
+    fake_curl = fake_bin_directory / "curl"
+    fake_curl.write_text(
+        "#!/usr/bin/env bash\n"
+        f'echo "$*" >> "{calls_file}"\n'
+        f'cat "{response_file}"\n'
+        f"exit {reply.exit_code}\n",
+        encoding="utf-8",
+    )
+    fake_curl.chmod(0o755)
+    return fake_bin_directory
+
+
+def _run_check(
+    tmp_path: Path,
+    curl_reply: FakeCurlReply,
+    *,
+    deployment_environment: str,
+    sim_entrypoint: str = "cloud_run_simulation_entrypoint",
+    simulation_url: str = SIMULATION_URL,
+) -> tuple[subprocess.CompletedProcess[str], list[str]]:
+    fake_bin_directory = _install_fake_curl(tmp_path, curl_reply)
+    result = subprocess.run(
+        ["bash", str(SCRIPT)],
+        cwd=REPO,
+        env={
+            "PATH": f"{fake_bin_directory}{os.pathsep}{os.environ['PATH']}",
+            "DEPLOYMENT_ENVIRONMENT": deployment_environment,
+            "SIM_ENTRYPOINT": sim_entrypoint,
+            "SIMULATION_ENTRYPOINT_URL": simulation_url,
+            "OLD_SIMULATION_GATEWAY_URL": "https://old-gateway.example.test",
+        },
+        text=True,
+        capture_output=True,
+        check=False,
+    )
+    calls_file = tmp_path / "curl-calls.log"
+    curl_calls = calls_file.read_text().splitlines() if calls_file.exists() else []
+    return result, curl_calls
+
+
+@pytest.mark.parametrize(
+    ("deployment_environment", "revision"),
+    [("staging", STAGING_REVISION), ("production", PRODUCTION_REVISION)],
+)
+def test_accepts_the_simulation_service_of_the_same_environment(
+    tmp_path: Path, deployment_environment: str, revision: str
+) -> None:
+    result, curl_calls = _run_check(
+        tmp_path,
+        _entrypoint_response(revision),
+        deployment_environment=deployment_environment,
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert len(curl_calls) == 1
+    assert curl_calls[0].endswith(f"{SIMULATION_URL}/health")
+    assert revision in result.stdout
+
+
+def test_requests_health_once_without_a_doubled_slash(tmp_path: Path) -> None:
+    result, curl_calls = _run_check(
+        tmp_path,
+        _entrypoint_response(STAGING_REVISION),
+        deployment_environment="staging",
+        simulation_url=f"{SIMULATION_URL}/",
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert curl_calls[0].endswith(f" {SIMULATION_URL}/health")
+
+
+def test_rejects_staging_calling_the_production_simulation_service(
+    tmp_path: Path,
+) -> None:
+    result, _ = _run_check(
+        tmp_path,
+        _entrypoint_response(PRODUCTION_REVISION),
+        deployment_environment="staging",
+    )
+
+    assert result.returncode == 1
+    assert "it must call policyengine-simulation-entry-staging" in result.stderr
+    assert PRODUCTION_REVISION in result.stderr
+
+
+def test_rejects_production_calling_the_staging_simulation_service(
+    tmp_path: Path,
+) -> None:
+    result, _ = _run_check(
+        tmp_path,
+        _entrypoint_response(STAGING_REVISION),
+        deployment_environment="production",
+    )
+
+    assert result.returncode == 1
+    assert "it must call policyengine-simulation-entry\n" in result.stderr
+    assert STAGING_REVISION in result.stderr
+
+
+def test_rejects_a_url_that_names_no_entrypoint_revision(tmp_path: Path) -> None:
+    result, _ = _run_check(
+        tmp_path, SAMPLE_CONTAINER_RESPONSE, deployment_environment="staging"
+    )
+
+    assert result.returncode == 1
+    assert "did not name a Simulation Entrypoint revision" in result.stderr
+
+
+def test_rejects_a_revision_name_without_a_cloud_run_suffix(tmp_path: Path) -> None:
+    result, _ = _run_check(
+        tmp_path,
+        _entrypoint_response("policyengine-simulation-entry-staging"),
+        deployment_environment="staging",
+    )
+
+    assert result.returncode == 1
+    assert "is not a Cloud Run revision name" in result.stderr
+
+
+def test_fails_when_the_health_request_fails(tmp_path: Path) -> None:
+    result, curl_calls = _run_check(
+        tmp_path, FAILED_REQUEST, deployment_environment="staging"
+    )
+
+    assert result.returncode == 22
+    assert len(curl_calls) == 1
+
+
+def test_rejects_an_unknown_deployment_environment(tmp_path: Path) -> None:
+    result, curl_calls = _run_check(
+        tmp_path,
+        _entrypoint_response(STAGING_REVISION),
+        deployment_environment="preview",
+    )
+
+    assert result.returncode == 2
+    assert "DEPLOYMENT_ENVIRONMENT must be staging or production" in result.stderr
+    assert curl_calls == []
+
+
+def test_rejects_a_missing_simulation_url(tmp_path: Path) -> None:
+    result, curl_calls = _run_check(
+        tmp_path,
+        _entrypoint_response(STAGING_REVISION),
+        deployment_environment="staging",
+        simulation_url="",
+    )
+
+    assert result.returncode == 1
+    assert "SIMULATION_ENTRYPOINT_URL is required" in result.stderr
+    assert curl_calls == []
+
+
+def test_does_not_check_the_shared_old_gateway(tmp_path: Path) -> None:
+    result, curl_calls = _run_check(
+        tmp_path,
+        _entrypoint_response(PRODUCTION_REVISION),
+        deployment_environment="staging",
+        sim_entrypoint="old_gateway_direct",
+    )
+
+    assert result.returncode == 0, result.stderr
+    assert curl_calls == []
+    assert "old_gateway_direct" in result.stdout
+
+
+@pytest.mark.parametrize(
+    ("job_name", "deployment_environment"),
+    [
+        ("ensure-staging-model-version-aligns-with-sim-api", "staging"),
+        ("ensure-production-model-version-aligns-with-sim-api", "production"),
+    ],
+)
+def test_each_release_checks_its_own_simulation_service_first(
+    job_name: str, deployment_environment: str
+) -> None:
+    """Each version-alignment job fails if it reaches the other environment.
+
+    The check runs right after checkout, before the job asks the simulation
+    service which bundle it supports, so a crossed environment fails the
+    release instead of validating against the wrong service.
+    """
+    job = yaml.safe_load(PUSH_WORKFLOW.read_text(encoding="utf-8"))["jobs"][job_name]
+
+    assert job["environment"] == deployment_environment
+    assert job["env"]["DEPLOYMENT_ENVIRONMENT"] == deployment_environment
+    assert job["env"]["SIM_ENTRYPOINT"] == "${{ vars.SIM_ENTRYPOINT }}"
+    assert (
+        job["env"]["SIMULATION_ENTRYPOINT_URL"]
+        == "${{ secrets.SIMULATION_ENTRYPOINT_URL }}"
+    )
+    step_commands = [step.get("run", "") for step in job["steps"]]
+    assert step_commands.count(CHECK_COMMAND) == 1
+    check_position = step_commands.index(CHECK_COMMAND)
+    bundle_check_position = step_commands.index(
+        "bash .github/check-policyengine-bundle-supported.sh"
+    )
+    assert (
+        job["steps"][check_position - 1].get("uses", "").startswith("actions/checkout@")
+    )
+    assert check_position < bundle_check_position