diff --git a/.github/scripts/cloud-run-simulation-entry-smoke.sh b/.github/scripts/cloud-run-simulation-entry-smoke.sh index 84c0726da..d34aa0de5 100755 --- a/.github/scripts/cloud-run-simulation-entry-smoke.sh +++ b/.github/scripts/cloud-run-simulation-entry-smoke.sh @@ -25,6 +25,12 @@ previous_revision="${3:-}" # and model. # This is the identity the gateway reports from the routing registry; the # numerical release sentinel checks what the worker computes. +# REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION=true (candidate checks only): +# /ready must report the entrypoint's own database reachable and at the +# migration revision the image expects, which shows that the runtime secret +# reached the revision and that the migration job ran. Stable and rollback +# checks leave it unset, because an older image may face a newer schema. +require_database_revision="${REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION:-}" expected_identity="${EXPECTED_IDENTITY:-none}" expected_policyengine_version="${EXPECTED_POLICYENGINE_VERSION:-}" expected_app_name="${EXPECTED_APP_NAME:-}" @@ -39,6 +45,14 @@ case "${expected_identity}" in exit 2 ;; esac +case "${require_database_revision}" in + "" | true) ;; + *) + printf 'REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION must be true or empty, not %s\n' \ + "${require_database_revision}" >&2 + exit 2 + ;; +esac case "${expected_latest}" in "" | true) ;; *) @@ -63,6 +77,9 @@ if [ "${expected_identity}" = "required" ] || fi routing_max_attempts="${SIMULATION_ENTRYPOINT_ROUTING_MAX_ATTEMPTS:-24}" routing_retry_delay_seconds="${SIMULATION_ENTRYPOINT_ROUTING_RETRY_DELAY_SECONDS:-5}" +# A cold candidate may not reach its database within /ready's three-second +# wait; an unreachable database is asked about again this many times. +database_ready_max_attempts="${SIMULATION_DATABASE_READY_MAX_ATTEMPTS:-6}" if ! [[ "${routing_max_attempts}" =~ ^[1-9][0-9]*$ ]]; then printf 'SIMULATION_ENTRYPOINT_ROUTING_MAX_ATTEMPTS must be a positive integer\n' >&2 @@ -72,6 +89,10 @@ if ! [[ "${routing_retry_delay_seconds}" =~ ^[0-9]+$ ]]; then printf 'SIMULATION_ENTRYPOINT_ROUTING_RETRY_DELAY_SECONDS must be a non-negative integer\n' >&2 exit 2 fi +if ! [[ "${database_ready_max_attempts}" =~ ^[1-9][0-9]*$ ]]; then + printf 'SIMULATION_DATABASE_READY_MAX_ATTEMPTS must be a positive integer\n' >&2 + exit 2 +fi health_headers="$(mktemp)" health_body="$(mktemp)" @@ -131,8 +152,29 @@ while true; do attempt=$((attempt + 1)) done -curl --fail --silent --show-error "${base_url}/ready" | - jq -e '.status == "ready"' >/dev/null +ready_body="$(curl --fail --silent --show-error "${base_url}/ready")" +jq -e '.status == "ready"' <<<"${ready_body}" >/dev/null +if [ "${require_database_revision}" = "true" ]; then + database_attempt=1 + while [ "${database_attempt}" -lt "${database_ready_max_attempts}" ] && + jq -e '.simulation_database.reachable == false' <<<"${ready_body}" >/dev/null; do + printf '%s: the simulation database was not reached; asking again (%s/%s)\n' \ + "${base_url}" "${database_attempt}" "${database_ready_max_attempts}" >&2 + sleep "${routing_retry_delay_seconds}" + ready_body="$(curl --fail --silent --show-error "${base_url}/ready")" + database_attempt=$((database_attempt + 1)) + done +fi +if [ "${require_database_revision}" = "true" ] && + ! jq -e ' + .simulation_database.reachable == true + and (.simulation_database.revision | type == "string") + and .simulation_database.revision == .simulation_database.expected_revision + ' <<<"${ready_body}" >/dev/null; then + printf '%s: the simulation database is not at the expected revision: %s\n' \ + "${base_url}" "$(jq -c '.simulation_database' <<<"${ready_body}")" >&2 + exit 1 +fi curl --fail --silent --show-error "${base_url}/versions" --output "${versions_body}" jq -e 'type == "object"' "${versions_body}" >/dev/null diff --git a/.github/scripts/deploy-cloud-run-simulation-entry.sh b/.github/scripts/deploy-cloud-run-simulation-entry.sh index 4320514ff..0956ae9a5 100755 --- a/.github/scripts/deploy-cloud-run-simulation-entry.sh +++ b/.github/scripts/deploy-cloud-run-simulation-entry.sh @@ -58,6 +58,28 @@ resolve_secret_reference() { printf '%s:%s' "${reference}" "${highest_enabled_version}" } +runtime_secrets=() + +# Resolve one secret per assignment: an array assignment takes its exit status +# from its last command substitution only, so resolving several secrets in +# one would let an earlier failure pass unnoticed. +add_runtime_secret() { + local variable_name="$1" + local resolved_reference + resolved_reference="$(resolve_secret_reference "$2")" + runtime_secrets+=("${variable_name}=${resolved_reference}") +} + +# The Stage 12 comparison path's own wrapper, removed with it, so the main +# path's wrapper can change without touching the Stage 12 block. Both use the +# secret lookups above, which predate the split. +add_stage12_runtime_secret() { + local variable_name="$1" + local resolved_reference + resolved_reference="$(resolve_secret_reference "$2")" + runtime_secrets+=("${variable_name}=${resolved_reference}") +} + required_variables=( PROJECT_ID REGION @@ -77,6 +99,7 @@ required_variables=( OLD_GATEWAY_AUTH_AUDIENCE_VALUE OLD_GATEWAY_AUTH_CLIENT_ID_VALUE OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME + SIMULATION_DATABASE_URL_SECRET_NAME OBSERVABILITY_SERVICE_NAMESPACE OBSERVABILITY_TRACE_PROJECT_ID OTEL_EXPORTER_OTLP_ENDPOINT @@ -115,11 +138,11 @@ export SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE="${SIMULATION_ENTRYPOINT_ [[ "${SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE}" == "0" || "${SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE}" == "1" ]] \ || fail "SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE must be 0 or 1" -runtime_secrets=( - "OLD_GATEWAY_AUTH_CLIENT_SECRET=$( - resolve_secret_reference "${OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME}" - )" -) +add_runtime_secret OLD_GATEWAY_AUTH_CLIENT_SECRET \ + "${OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME}" +# The runtime role's URL for the entrypoint's own simulation schema. The +# migration job, which uses a separate migrator URL, runs before this deploy. +add_runtime_secret SIMULATION_DATABASE_URL "${SIMULATION_DATABASE_URL_SECRET_NAME}" if [[ "${DEPLOY_STAGE12_V2}" == "true" ]]; then stage12_variables=( @@ -138,11 +161,9 @@ if [[ "${DEPLOY_STAGE12_V2}" == "true" ]]; then || fail "STAGE12_ARTIFACT_BUCKET_VALUE is invalid" validate_secret_reference "${STAGE12_DATABASE_URL_SECRET_NAME}" - runtime_secrets+=( - "MODAL_TOKEN_ID=$(resolve_secret_reference "${MODAL_TOKEN_ID_SECRET_NAME}")" - "MODAL_TOKEN_SECRET=$(resolve_secret_reference "${MODAL_TOKEN_SECRET_SECRET_NAME}")" - "STAGE12_DATABASE_URL=$(resolve_secret_reference "${STAGE12_DATABASE_URL_SECRET_NAME}")" - ) + add_stage12_runtime_secret MODAL_TOKEN_ID "${MODAL_TOKEN_ID_SECRET_NAME}" + add_stage12_runtime_secret MODAL_TOKEN_SECRET "${MODAL_TOKEN_SECRET_SECRET_NAME}" + add_stage12_runtime_secret STAGE12_DATABASE_URL "${STAGE12_DATABASE_URL_SECRET_NAME}" fi runtime_environment_file="${RUNNER_TEMP}/simulation-entry-runtime-environment.yaml" diff --git a/.github/scripts/migrate-simulation-database.sh b/.github/scripts/migrate-simulation-database.sh new file mode 100755 index 000000000..88bc3d84e --- /dev/null +++ b/.github/scripts/migrate-simulation-database.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Bring the simulation schema to the chain's head, or require it to be there. +# Run from libs/policyengine-simulation-persistence, whose locked environment +# (with the migrations extra) provides Alembic. migrations/env.py reads the +# migrator URL from SIMULATION_MIGRATION_DATABASE_URL; this script never +# prints it. +# +# SIMULATION_MIGRATION_MODE: +# upgrade apply every pending revision, then require the database at the +# head (deployments from the default branch); +# verify apply nothing and only require the database at this checkout's +# head, so a deployment dispatched from another branch can never +# apply that branch's revisions. + +set -euo pipefail +set +x + +if [[ -z "${SIMULATION_MIGRATION_DATABASE_URL:-}" ]]; then + printf 'SIMULATION_MIGRATION_DATABASE_URL must name the simulation database\n' >&2 + exit 1 +fi + +mode="${SIMULATION_MIGRATION_MODE:-}" +case "${mode}" in + upgrade | verify) ;; + *) + printf 'SIMULATION_MIGRATION_MODE must be upgrade or verify, not %s\n' "${mode:-empty}" >&2 + exit 1 + ;; +esac + +alembic_config="../../migrations/alembic.ini" + +if [[ "${mode}" == "upgrade" ]]; then + uv run --frozen --extra migrations alembic -c "${alembic_config}" upgrade head +fi +uv run --frozen --extra migrations alembic -c "${alembic_config}" current --check-heads diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 0c27df2f1..ac46f768b 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -16,7 +16,7 @@ jobs: # every job passed. See docs/engineering/skills/github-prs.md. name: CI complete if: ${{ always() }} - needs: [test, lint, pyright, quality-guards, docker-build, test-integration] + needs: [test, simulation-database, lint, pyright, quality-guards, docker-build, test-integration] runs-on: ubuntu-latest permissions: contents: read @@ -51,6 +51,7 @@ jobs: - projects/policyengine-simulation-gateway - libs/policyengine-simulation-contract - libs/policyengine-simulation-observability + - libs/policyengine-simulation-persistence - libs/policyengine-stage12-persistence - libs/policyengine-fastapi @@ -75,6 +76,54 @@ jobs: working-directory: ${{ matrix.project }} run: uv run --frozen pytest tests/ -v + simulation-database: + name: Simulation database + # Runs the simulation schema's migration chain (upgrade, alembic check, + # downgrade, upgrade) and the persistence tests that need a database, on a + # fresh PostgreSQL. Full CI runs only on the lowest unmerged pull request + # in a stack. + if: ${{ !github.event.pull_request.stack || github.event.pull_request.stack.base.ref == github.event.pull_request.base.ref }} + runs-on: ubuntu-latest + services: + postgres: + image: postgres:17 + env: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: simulation_ci + POSTGRES_DB: postgres + ports: + - 5432:5432 + options: >- + --health-cmd="pg_isready -U postgres" + --health-interval=5s + --health-timeout=5s + --health-retries=20 + env: + SIMULATION_TEST_DATABASE_URL: postgresql://postgres:simulation_ci@127.0.0.1:5432/postgres + # A missing server fails the tests instead of skipping them. + SIMULATION_REQUIRE_DATABASE_TESTS: "1" + + steps: + - uses: actions/checkout@v6 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: '3.13' + + - name: Install uv + uses: astral-sh/setup-uv@v8.1.0 + with: + enable-cache: true + + - name: Install dependencies + working-directory: libs/policyengine-simulation-persistence + run: uv sync --extra test --frozen + + - name: Run the migration chain and persistence tests on PostgreSQL + working-directory: libs/policyengine-simulation-persistence + run: uv run --frozen pytest tests/ -v -m postgres + lint: name: Lint # Full CI runs only on the lowest unmerged pull request in a stack. @@ -157,6 +206,7 @@ jobs: package: - libs/policyengine-simulation-contract - libs/policyengine-simulation-observability + - libs/policyengine-simulation-persistence - libs/policyengine-stage12-persistence - libs/policyengine-fastapi - projects/policyengine-simulation-entry diff --git a/.github/workflows/simulation-deploy.reusable.yml b/.github/workflows/simulation-deploy.reusable.yml index 9de0a3eff..15990f525 100644 --- a/.github/workflows/simulation-deploy.reusable.yml +++ b/.github/workflows/simulation-deploy.reusable.yml @@ -153,10 +153,40 @@ jobs: GATEWAY_AUTH_REQUIRED: ${{ vars.GATEWAY_AUTH_REQUIRED }} run: ../../.github/scripts/modal-sync-secrets.sh "${{ inputs.modal_environment }}" + migrate_simulation_database: + # Applies the simulation schema's Alembic chain before the entrypoint + # candidate deploys. The previous revision keeps serving meanwhile, so + # every migration must work with the release before it. + name: Migrate the simulation database schema + needs: prepare + runs-on: ubuntu-latest + environment: ${{ inputs.release_environment }} + steps: + - uses: actions/checkout@v6 + + - uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - uses: astral-sh/setup-uv@v8.1.0 + + - name: Install migration dependencies + working-directory: libs/policyengine-simulation-persistence + run: uv sync --frozen --extra migrations + + - name: Apply simulation schema migrations + working-directory: libs/policyengine-simulation-persistence + env: + SIMULATION_MIGRATION_DATABASE_URL: ${{ secrets.SIMULATION_MIGRATION_DATABASE_URL }} + # Only a deployment from main applies revisions. One dispatched from + # another branch requires the database to be at its chain's head. + SIMULATION_MIGRATION_MODE: ${{ github.ref == 'refs/heads/main' && 'upgrade' || 'verify' }} + run: ../../.github/scripts/migrate-simulation-database.sh + deploy_entrypoint: name: Deploy and unit test Cloud Run entrypoint - if: ${{ always() && needs.prepare.result == 'success' && ((inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'success') || (!inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'skipped')) }} - needs: [prepare, configure_stage12_v2] + if: ${{ always() && needs.prepare.result == 'success' && needs.migrate_simulation_database.result == 'success' && ((inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'success') || (!inputs.deploy_stage12_v2 && needs.configure_stage12_v2.result == 'skipped')) }} + needs: [prepare, configure_stage12_v2, migrate_simulation_database] runs-on: ubuntu-latest environment: ${{ inputs.release_environment }} permissions: @@ -237,6 +267,7 @@ jobs: OLD_GATEWAY_AUTH_AUDIENCE_VALUE: ${{ secrets.OLD_GATEWAY_AUTH_AUDIENCE }} OLD_GATEWAY_AUTH_CLIENT_ID_VALUE: ${{ secrets.OLD_GATEWAY_AUTH_CLIENT_ID }} OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME: ${{ vars.OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME }} + SIMULATION_DATABASE_URL_SECRET_NAME: ${{ vars.SIMULATION_DATABASE_URL_SECRET_NAME }} STAGE12_ENABLED_VALUE: ${{ vars.STAGE12_ENABLED }} STAGE12_ARTIFACT_BUCKET_VALUE: ${{ vars.STAGE12_ARTIFACT_BUCKET }} STAGE12_DATABASE_URL_SECRET_NAME: ${{ vars.STAGE12_DATABASE_URL_SECRET_NAME }} @@ -744,6 +775,8 @@ jobs: EXPECTED_UK_DATASET_SHA256: ${{ inputs.deploy_existing_stack && needs.prepare.outputs.uk_dataset_sha256 || '' }} # Latest must route to this release only when it should advance it. EXPECTED_LATEST: ${{ inputs.deploy_existing_stack && needs.update_routing.outputs.advances_latest == 'true' && 'true' || '' }} + # The candidate reaches its own database at the migrated revision. + REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION: "true" run: .github/scripts/cloud-run-simulation-entry-smoke.sh "${{ needs.deploy_entrypoint.outputs.candidate_url }}" "${{ needs.deploy_entrypoint.outputs.revision }}" - name: Download the numerical release sentinel diff --git a/AGENTS.md b/AGENTS.md index 12ba40fb5..7c410cfdf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -120,7 +120,11 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`. ### Stage 12 persistence -- Checked: `sim.stage12.deployment-scope`, `sim.stage12.deployment-uses-api-schema`, `sim.stage12.no-migration-chain`, `sim.stage12.no-production-user-associations`, `sim.stage12.no-schema-ddl`, `sim.stage12.persistence-typed-dml-only`, `sim.stage12.production-lock-has-db-runtime` +- Checked: `sim.stage12.deployment-scope`, `sim.stage12.deployment-uses-api-schema`, `sim.stage12.no-production-user-associations`, `sim.stage12.no-schema-ddl`, `sim.stage12.persistence-typed-dml-only`, `sim.stage12.production-lock-has-db-runtime` + +### Simulation persistence + +- Checked: `sim.persistence.executor-independent`, `sim.persistence.migrate-before-entrypoint`, `sim.persistence.migration-cycle`, `sim.persistence.no-modal-database-credentials`, `sim.persistence.one-migration-chain` ### Deployment pipeline diff --git a/Makefile b/Makefile index 94484d22c..7a0aea572 100644 --- a/Makefile +++ b/Makefile @@ -89,6 +89,7 @@ test: projects/policyengine-simulation-gateway \ libs/policyengine-simulation-contract \ libs/policyengine-simulation-observability \ + libs/policyengine-simulation-persistence \ libs/policyengine-stage12-persistence \ libs/policyengine-fastapi; do \ echo "Testing $$proj..."; \ diff --git a/docs/engineering/skills/database-access.md b/docs/engineering/skills/database-access.md index 1ed83a886..8448cb941 100644 --- a/docs/engineering/skills/database-access.md +++ b/docs/engineering/skills/database-access.md @@ -21,31 +21,32 @@ It builds on the Stage 12 rules: - `sim.stage12.no-schema-ddl`: the simulation runtimes and the Stage 12 persistence library contain no schema DDL. Its `DDL_PATTERN` regex is the one this rule matches against every source directory's string literals. -- `sim.stage12.no-migration-chain`: this repository has no migration chain - of its own; policyengine-api migrates the Stage 12 tables. +- `sim.persistence.one-migration-chain`: the repository's one Alembic chain, + at `migrations/`, owns the Simulation Entrypoint's `simulation` schema; + the Public API migrates the Stage 12 tables. - `sim.stage12.persistence-typed-dml-only` and `sim.stage12.deployment-uses-api-schema`: the Stage 12 persistence library uses typed SQLAlchemy Core DML only, with no raw SQL text. ## Who owns the schema -- **Today**, policyengine-api's Alembic migrations own the only tables this - repository uses, the Stage 12 comparison tables - (`sim.stage12.no-migration-chain`). This repository's code reads and - writes their rows, and deployment validation compares their live schema - with its SQLAlchemy mappings; nothing here changes them. -- **Once #847 merges**, this repository's own Alembic chain at `migrations/` - owns the `simulation` schema, and the Stage 12 tables stay with - policyengine-api. +- **The `simulation` schema** belongs to this repository's own Alembic chain + at `migrations/` (`sim.persistence.one-migration-chain`). The tables are + declared in `libs/policyengine-simulation-persistence`, whose README + describes the chain, the database roles and how to add a revision. +- **The Stage 12 comparison tables** belong to the Public API's Alembic + migrations. This repository's code reads and writes their rows, and + deployment validation compares their live schema with its SQLAlchemy + mappings; nothing here changes them. -The rule holds either way: whichever chain owns a table, only that chain's -migrations change it. +Either way, only the chain that owns a table changes it. ## What the guard checks `projects/policyengine-simulation-entry/tests/test_database_access_boundaries.py` parses every Python file under `projects/*/src` (including the executor's -`src/modal`), `libs/*/src` and `.github/scripts`, and runs in the entry +`src/modal`), `libs/*/src`, `.github/scripts` and the repository's +`migrations/` chain, and runs in the entry project's `Test` job. The checks live in `database_access_support.py` next to it. It fails on: diff --git a/docs/engineering/skills/type-checking.md b/docs/engineering/skills/type-checking.md index 56a444b58..b1b8abbd2 100644 --- a/docs/engineering/skills/type-checking.md +++ b/docs/engineering/skills/type-checking.md @@ -11,9 +11,9 @@ Every uv package is type-checked with Pyright on every pull request, by the - `sim.quality.pyright-libraries-and-entry`, `sim.quality.pyright-gateway`, `sim.quality.pyright-executor` and `sim.quality.pyright-apis-integ`: the - contract, observability, Stage 12 persistence and fastapi libraries, the - simulation entry, the gateway, the executor and the generated-client - integration tests have no Pyright errors. + contract, observability, simulation persistence, Stage 12 persistence and + fastapi libraries, the simulation entry, the gateway, the executor and the + generated-client integration tests have no Pyright errors. - `sim.quality.every-package-type-checked`: a Pyright rule lists every uv package under `libs/` and `projects/`. When you add a package, give it a `[tool.pyright] include`, add it to a Pyright rule's `enforcer.projects`, and diff --git a/docs/generated/rules.md b/docs/generated/rules.md index b2f783267..5d470dda9 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -8,11 +8,11 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf | Tier | enforced | ratcheted | advisory | doc-only | Total | | --- | ---: | ---: | ---: | ---: | ---: | -| mechanical | 53 | 2 | 0 | 14 | 69 | +| mechanical | 57 | 2 | 0 | 14 | 73 | | metric | 1 | 0 | 0 | 0 | 1 | | judgment | 0 | 0 | 0 | 0 | 0 | -**Blocking:** 56 of 70 rules. +**Blocking:** 60 of 74 rules. ## Gaps @@ -42,7 +42,6 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf - `sim.contracts.clients-publish-deployed-commit` - `sim.contracts.full-integration-beta-only` - `sim.contracts.gateway-openapi-golden` -- `sim.deploy.container-context-minimal` - `sim.deploy.gcloud-without-terraform` - `sim.deploy.numerical-sentinel-gate` - `sim.deploy.permission-enforcement-wired` @@ -53,11 +52,9 @@ Every rule that governs `policyengine-sim-api` is listed here with how it is enf - `sim.deploy.shell-scripts-valid` - `sim.deploy.smoke-tests-strict` - `sim.deploy.traffic-changes-exact` -- `sim.deploy.workflow-shell-safety` - `sim.process.rules-registry` - `sim.stage12.deployment-scope` - `sim.stage12.deployment-uses-api-schema` -- `sim.stage12.no-migration-chain` - `sim.stage12.no-production-user-associations` - `sim.stage12.persistence-typed-dml-only` - `sim.stage12.production-lock-has-db-runtime` @@ -103,7 +100,6 @@ flowchart LR | --- | --- | --- | --- | --- | --- | --- | --- | | `sim.stage12.deployment-scope` | Main-branch deploys roll out Stage 12 in both environments; a Stage 12-only deploy runs candidate tests and the requested promotion but never redeploys existing Modal resources or runs the existing stack's complete integration suite; the Stage 12 worker can request a Google OIDC token. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_promotion.py`, `projects/policyengine-simulation-entry/tests/test_deployment_assets_release_gates.py` | `pr.yml:test` | **unproven** | | `sim.stage12.deployment-uses-api-schema` | Stage 12 deployment uses the API-owned schema directly, and the Stage 12 runtime it deploys uses typed DML without raw SQL or schema changes. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | -| `sim.stage12.no-migration-chain` | This repository has no schema migration chain (no Alembic); Stage 12 tables are migrated by policyengine-api. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | **unproven** | | `sim.stage12.no-production-user-associations` | Stage 12 runtime modules never create production user associations (user_simulations or user_reports). | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | **unproven** | | `sim.stage12.no-schema-ddl` | Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py` | `pr.yml:test` | yes | | `sim.stage12.persistence-typed-dml-only` | The Stage 12 persistence library and runtime use typed SQLAlchemy Core DML only: no raw SQL text, driver-level SQL or schema operations. | mechanical | enforced | yes | pytest `libs/policyengine-stage12-persistence/tests/test_source_boundaries.py` | `pr.yml:test` | **unproven** | @@ -116,9 +112,8 @@ flowchart LR j_CI_pr_yml_test["CI pr.yml:test"] r_sim_stage12_deployment_uses_api_schema("sim.stage12.deployment-uses-api-schema (enforced)") e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py["pytest projects/policyengine-simulation-entry/tests/test_deployment_assets.py"] - r_sim_stage12_no_migration_chain("sim.stage12.no-migration-chain (enforced)") - e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py["pytest projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py"] r_sim_stage12_no_production_user_associations("sim.stage12.no-production-user-associations (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py["pytest projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py"] r_sim_stage12_no_schema_ddl("sim.stage12.no-schema-ddl (enforced)") r_sim_stage12_persistence_typed_dml_only("sim.stage12.persistence-typed-dml-only (enforced)") e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py["pytest libs/policyengine-stage12-persistence/tests/test_source_boundaries.py"] @@ -128,9 +123,8 @@ flowchart LR e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_promotion_py__projects_policyengine_simulation_entry_tests_test_deployment_assets_release_gates_py --> j_CI_pr_yml_test r_sim_stage12_deployment_uses_api_schema --> e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_py --> j_CI_pr_yml_test - r_sim_stage12_no_migration_chain --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py - e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py --> j_CI_pr_yml_test r_sim_stage12_no_production_user_associations --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py + e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py --> j_CI_pr_yml_test r_sim_stage12_no_schema_ddl --> e_pytest_projects_policyengine_simulation_entry_tests_test_stage12_schema_authority_py r_sim_stage12_persistence_typed_dml_only --> e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py e_pytest_libs_policyengine_stage12_persistence_tests_test_source_boundaries_py --> j_CI_pr_yml_test @@ -138,11 +132,43 @@ flowchart LR e_pytest_projects_policyengine_simulation_entry_tests_test_deployment_assets_traffic_py --> j_CI_pr_yml_test ``` +## Simulation persistence + +| Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | +| --- | --- | --- | --- | --- | --- | --- | --- | +| `sim.persistence.executor-independent` | The Modal executor and gateway never declare, lock or import policyengine-simulation-persistence; only the Simulation Entrypoint and the migration job install it. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py` | `pr.yml:test` | yes | +| `sim.persistence.migrate-before-entrypoint` | Each deployment migrates the simulation schema in its own job, the only one that receives SIMULATION_MIGRATION_DATABASE_URL, before the entrypoint candidate deploys, applying revisions only from main and otherwise requiring the database at the chain's head; the entrypoint receives SIMULATION_DATABASE_URL as a pinned Secret Manager version; and the candidate smoke requires the database at the image's revision. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_simulation_database_deployment.py` | `pr.yml:test` | yes | +| `sim.persistence.migration-cycle` | On a fresh PostgreSQL, the simulation chain upgrades to its head, passes alembic check, downgrades to base and upgrades again, run by a non-superuser migrator, and the runtime role can read the revision but cannot change the schema. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_migration_cycle.py` | `pr.yml:simulation-database` | yes | +| `sim.persistence.no-modal-database-credentials` | No Modal secret, Modal deployment or Modal application receives a database URL other than the comparison path's own STAGE12_DATABASE_URL, which Modal may hold until the comparison path is removed; in particular never SIMULATION_DATABASE_URL or SIMULATION_MIGRATION_DATABASE_URL. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py` | `pr.yml:test` | yes | +| `sim.persistence.one-migration-chain` | The simulation schema changes only through the one Alembic chain at migrations/, configured by migrations/alembic.ini and versioned in simulation.alembic_version; no other Alembic configuration, environment or revision directory exists. | mechanical | enforced | yes | pytest `libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py` | `pr.yml:test` | yes | + +```mermaid +flowchart LR + r_sim_persistence_executor_independent("sim.persistence.executor-independent (enforced)") + e_pytest_libs_policyengine_simulation_persistence_tests_test_ownership_boundaries_py["pytest libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py"] + j_CI_pr_yml_test["CI pr.yml:test"] + r_sim_persistence_migrate_before_entrypoint("sim.persistence.migrate-before-entrypoint (enforced)") + e_pytest_projects_policyengine_simulation_entry_tests_test_simulation_database_deployment_py["pytest projects/policyengine-simulation-entry/tests/test_simulation_database_deployment.py"] + r_sim_persistence_migration_cycle("sim.persistence.migration-cycle (enforced)") + e_pytest_libs_policyengine_simulation_persistence_tests_test_migration_cycle_py["pytest libs/policyengine-simulation-persistence/tests/test_migration_cycle.py"] + j_CI_pr_yml_simulation_database["CI pr.yml:simulation-database"] + r_sim_persistence_no_modal_database_credentials("sim.persistence.no-modal-database-credentials (enforced)") + r_sim_persistence_one_migration_chain("sim.persistence.one-migration-chain (enforced)") + r_sim_persistence_executor_independent --> e_pytest_libs_policyengine_simulation_persistence_tests_test_ownership_boundaries_py + e_pytest_libs_policyengine_simulation_persistence_tests_test_ownership_boundaries_py --> j_CI_pr_yml_test + r_sim_persistence_migrate_before_entrypoint --> e_pytest_projects_policyengine_simulation_entry_tests_test_simulation_database_deployment_py + e_pytest_projects_policyengine_simulation_entry_tests_test_simulation_database_deployment_py --> j_CI_pr_yml_test + r_sim_persistence_migration_cycle --> e_pytest_libs_policyengine_simulation_persistence_tests_test_migration_cycle_py + e_pytest_libs_policyengine_simulation_persistence_tests_test_migration_cycle_py --> j_CI_pr_yml_simulation_database + r_sim_persistence_no_modal_database_credentials --> e_pytest_libs_policyengine_simulation_persistence_tests_test_ownership_boundaries_py + r_sim_persistence_one_migration_chain --> e_pytest_libs_policyengine_simulation_persistence_tests_test_ownership_boundaries_py +``` + ## Deployment pipeline | Rule | Statement | Tier | Status | Blocking | Enforced by | CI | Proof | | --- | --- | --- | --- | --- | --- | --- | --- | -| `sim.deploy.container-context-minimal` | Container builds exclude local environments and unrelated projects, and the entrypoint container builds for the requested target platform. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py`, `projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.container-context-minimal` | Container builds exclude local environments and unrelated projects but include every local library the entrypoint installs, and the entrypoint container builds for the requested target platform. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py`, `projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py` | `pr.yml:test` | yes | | `sim.deploy.docker-images-build` | The simulation entry and executor Docker images build. | mechanical | enforced | yes | CI job `docker-build` | `pr.yml:docker-build` | — | | `sim.deploy.gcloud-without-terraform` | Deployment uses the gcloud-based workflow, not Terraform. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | | `sim.deploy.image-smoke-covers-every-image-lib` | The image smoke path filter includes every library mounted into a Modal image, so a change to any of them runs the smoke. | mechanical | doc-only | no | nothing | — | — | @@ -156,7 +182,7 @@ flowchart LR | `sim.deploy.shell-scripts-valid` | Deployment shell scripts have valid shell syntax. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | | `sim.deploy.smoke-tests-strict` | Candidate and stable smoke tests reject a revision mismatch without retrying, and fail if the previous revision does not converge. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets.py` | `pr.yml:test` | **unproven** | | `sim.deploy.traffic-changes-exact` | Traffic changes target an exact, validated revision, are reversible, and refuse to run over an intervening promotion. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py` | `pr.yml:test` | **unproven** | -| `sim.deploy.workflow-shell-safety` | Cloud Run deployment never interpolates GitHub expressions into shell, escapes environment values, and pins secret versions. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py` | `pr.yml:test` | **unproven** | +| `sim.deploy.workflow-shell-safety` | Cloud Run deployment never interpolates GitHub expressions into shell, escapes environment values, pins secret versions, stops before deploying when any runtime secret cannot be resolved, and resolves the Stage 12 secrets with the Stage 12 block's own helper. | mechanical | enforced | yes | pytest `projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py` | `pr.yml:test` | yes | ```mermaid flowchart LR @@ -310,7 +336,7 @@ flowchart LR | `sim.quality.pyright-apis-integ` | The generated-client integration-test project, apis-integ, has no Pyright errors, checked with the pinned Pyright version in the project's own locked environment. | mechanical | enforced | yes | pyright in `projects/policyengine-apis-integ` | `pr.yml:pyright` | yes | | `sim.quality.pyright-executor` | The simulation executor has no Pyright errors, checked with the pinned Pyright version in the executor's own locked environment. | mechanical | enforced | yes | pyright in `projects/policyengine-simulation-executor` | `pr.yml:pyright` | yes | | `sim.quality.pyright-gateway` | The simulation gateway has no Pyright errors, checked with the pinned Pyright version in the gateway's own locked environment. | mechanical | enforced | yes | pyright in `projects/policyengine-simulation-gateway` | `pr.yml:pyright` | yes | -| `sim.quality.pyright-libraries-and-entry` | The contract, observability, Stage 12 persistence and fastapi libraries and the simulation entry have no Pyright errors, checked with the pinned Pyright version in each package's own locked environment. | mechanical | enforced | yes | pyright in `libs/policyengine-simulation-contract`, `libs/policyengine-simulation-observability`, `libs/policyengine-stage12-persistence`, `libs/policyengine-fastapi`, `projects/policyengine-simulation-entry` | `pr.yml:pyright` | yes | +| `sim.quality.pyright-libraries-and-entry` | The contract, observability, simulation persistence, Stage 12 persistence and fastapi libraries and the simulation entry have no Pyright errors, checked with the pinned Pyright version in each package's own locked environment. | mechanical | enforced | yes | pyright in `libs/policyengine-simulation-contract`, `libs/policyengine-simulation-observability`, `libs/policyengine-simulation-persistence`, `libs/policyengine-stage12-persistence`, `libs/policyengine-fastapi`, `projects/policyengine-simulation-entry` | `pr.yml:pyright` | yes | | `sim.quality.python-file-length` | No tracked Python file has more than 800 lines, or more than 1,500 lines if it is test code: a file under a tests or fixtures directory, or a conftest.py. | metric | enforced | yes | quality guard `file-length` | `pr.yml:quality-guards` | yes | | `sim.quality.ruff-format-src` | Source code under projects/*/src and libs/*/src is formatted with ruff format. | mechanical | enforced | yes | ruff format | `pr.yml:lint` | — | | `sim.quality.ruff-format-tests` | Test code is formatted with ruff format too; CI currently checks only src directories. | mechanical | doc-only | no | nothing | — | — | @@ -343,7 +369,7 @@ flowchart LR r_sim_quality_pyright_gateway("sim.quality.pyright-gateway (enforced)") e_pyright_in_projects_policyengine_simulation_gateway["pyright in projects/policyengine-simulation-gateway"] r_sim_quality_pyright_libraries_and_entry("sim.quality.pyright-libraries-and-entry (enforced)") - e_pyright_in_libs_policyengine_simulation_contract__libs_policyengine_simulation_observability__libs_policyengine_stage12_persistence__libs_policyengine_fastapi__projects_policyengine_simulation_entry["pyright in libs/policyengine-simulation-contract, libs/policyengine-simulation-observability, libs/policyengine-stage12-persistence, libs/policyengine-fastapi, projects/policyengine-simulation-entry"] + e_pyright_in_libs_policyengine_simulation_contract__libs_policyengine_simulation_observability__libs_policyengine_simulation_persistence__libs_policyengine_stage12_persistence__libs_policyengine_fastapi__projects_policyengine_simulation_entry["pyright in libs/policyengine-simulation-contract, libs/policyengine-simulation-observability, libs/policyengine-simulation-persistence, libs/policyengine-stage12-persistence, libs/policyengine-fastapi, projects/policyengine-simulation-entry"] r_sim_quality_python_file_length("sim.quality.python-file-length (enforced)") e_quality_guard_file_length["quality guard file-length"] r_sim_quality_ruff_format_src("sim.quality.ruff-format-src (enforced)") @@ -377,8 +403,8 @@ flowchart LR e_pyright_in_projects_policyengine_simulation_executor --> j_CI_pr_yml_pyright r_sim_quality_pyright_gateway --> e_pyright_in_projects_policyengine_simulation_gateway e_pyright_in_projects_policyengine_simulation_gateway --> j_CI_pr_yml_pyright - r_sim_quality_pyright_libraries_and_entry --> e_pyright_in_libs_policyengine_simulation_contract__libs_policyengine_simulation_observability__libs_policyengine_stage12_persistence__libs_policyengine_fastapi__projects_policyengine_simulation_entry - e_pyright_in_libs_policyengine_simulation_contract__libs_policyengine_simulation_observability__libs_policyengine_stage12_persistence__libs_policyengine_fastapi__projects_policyengine_simulation_entry --> j_CI_pr_yml_pyright + r_sim_quality_pyright_libraries_and_entry --> e_pyright_in_libs_policyengine_simulation_contract__libs_policyengine_simulation_observability__libs_policyengine_simulation_persistence__libs_policyengine_stage12_persistence__libs_policyengine_fastapi__projects_policyengine_simulation_entry + e_pyright_in_libs_policyengine_simulation_contract__libs_policyengine_simulation_observability__libs_policyengine_simulation_persistence__libs_policyengine_stage12_persistence__libs_policyengine_fastapi__projects_policyengine_simulation_entry --> j_CI_pr_yml_pyright r_sim_quality_python_file_length --> e_quality_guard_file_length e_quality_guard_file_length --> j_CI_pr_yml_quality_guards r_sim_quality_ruff_format_src --> e_ruff_format diff --git a/libs/policyengine-simulation-persistence/README.md b/libs/policyengine-simulation-persistence/README.md new file mode 100644 index 000000000..84fb4b975 --- /dev/null +++ b/libs/policyengine-simulation-persistence/README.md @@ -0,0 +1,147 @@ +# PolicyEngine simulation persistence + +SQLAlchemy Core tables and database connections for the Simulation +Entrypoint's own Postgres schema, `simulation`. The Simulation Entrypoint owns +every table in that schema; no other service creates, changes or writes them. + +The temporary Stage 12 comparison tables are not part of this schema: they are +owned and migrated by the Public API, and `libs/policyengine-stage12-persistence` +writes them with DML only. + +## Tables + +| Table | Holds | +| --- | --- | +| `simulations` | What to run, independent of model versions: kind (`economy` or `household`), country, policy (null for current law), dynamic, and either a household or a region code (`state/ca`) with an optional dataset name (null means the region's default dataset in each catalog version) | +| `reports` | Which two simulations a report compares (an economy report needs a reform; a household report may have only a baseline), both of the report's own country and kind, with its year and options | + +Both are definitions: the runtime role may read and insert them but not update +or delete them. Rows migrated from the Public API's v1 database keep their v1 +IDs in the unique `legacy_simulation_id` and `legacy_report_id` columns, and +the stores' `create_migrated` methods return the existing row when the same v1 +ID is migrated again. + +Foreign keys into the Public API's tables in the same database (`policies` and +`households`, each together with its country, and `dynamics`) are +`ON DELETE RESTRICT`. Creating them needs `REFERENCES` on those tables for the +migrator role; the runtime role needs no privilege on them. + +## Ownership rules + +These rules are registered in `rules/registry.toml` and checked by +`tests/test_ownership_boundaries.py`: + +- **One migration chain** (`sim.persistence.one-migration-chain`). Schema + changes are made only by the Alembic chain in the repository's + `migrations/` directory, configured by `migrations/alembic.ini`. It keeps its + version table at `simulation.alembic_version`. No other Alembic + configuration, environment or revision directory exists in the repository. + Runtime code never changes a schema + (`sim.boundaries.sqlalchemy-and-alembic-only`). +- **The executor does not depend on persistence** + (`sim.persistence.executor-independent`). The Modal executor and gateway + never declare, lock or import this library. Only the Simulation Entrypoint + and the migration job install it. +- **No database credentials in Modal** + (`sim.persistence.no-modal-database-credentials`). No Modal secret, Modal + deployment or Modal application source receives or names a database URL: + neither a variable named like `*DATABASE_URL*` nor a written-out + `postgresql://` URL, whether in a step's, job's or workflow's environment + or in a script such a step runs. The one exception is the Stage 12 + comparison's own `STAGE12_DATABASE_URL` (and the name of the secret that + holds it), until the comparison path is removed. In particular, + `SIMULATION_DATABASE_URL` and `SIMULATION_MIGRATION_DATABASE_URL` never + reach Modal. + +## Database roles + +| Role | Used by | Privileges | +| --- | --- | --- | +| `simulation_migrator` | The deploy workflow's migration job, through `SIMULATION_MIGRATION_DATABASE_URL` | Creates the `simulation` schema if it is missing, and owns every object in it; `REFERENCES` on the Public API tables its foreign keys point to | +| `simulation_runtime` | The Simulation Entrypoint, and the one-time migration provisioning command an operator runs, through `SIMULATION_DATABASE_URL` | `USAGE` on the schema, `SELECT` on the version table, and `SELECT`, `INSERT` and `UPDATE` on the tables the migrations create, except `UPDATE` on the definition tables (`simulations`, `reports`); never `DELETE`, and nothing on the Public API tables | + +The initial revision grants the runtime role its privileges, including default +privileges on tables the migrator creates later; revision 0002 revokes +`UPDATE` on the definition tables, which never change. Both roles must exist +before the first migration runs; an operator creates them. Before revision +0002 is deployed, the owner of the Public API's tables must also grant the +migrator `REFERENCES` on `policies`, `households` and `dynamics` (the +operator step grants it on `tax_benefit_model_versions`, `datasets` and +`regions` too, for later revisions); without it the migration fails and +applies nothing. The migrator needs either +`CREATE` on the database or a `simulation` schema created in advance and owned +by it: the environment creates the schema only when it is missing, because +PostgreSQL checks `CREATE` on the database even for +`CREATE SCHEMA IF NOT EXISTS`. + +## Deployment + +On every deployment, the migration job (`migrate_simulation_database` in +`.github/workflows/simulation-deploy.reusable.yml`) runs +`.github/scripts/migrate-simulation-database.sh` before the Simulation +Entrypoint's candidate revision is deployed +(`sim.persistence.migrate-before-entrypoint`). From `main` it runs +`alembic upgrade head` and then `alembic current --check-heads`; from any other +branch it runs only the head check, so a deployment dispatched from an unmerged +branch fails instead of applying that branch's revisions. It is the only job +that receives `SIMULATION_MIGRATION_DATABASE_URL`. If it fails, the entrypoint +is not deployed. The Simulation Entrypoint's readiness check (`GET /ready`) reports +the database's revision next to `HEAD_REVISION`, the revision its image +expects, and the candidate check requires the two to match. + +## Migration provisioning + +`src/policyengine_simulation_persistence/migration_provisioning/` holds the +command an operator runs once per environment during the one-time migration: +it creates the simulations and reports of the Public API's exported v1 report +definitions, with no run, and writes each v1 report's v2 report ID. It needs +the `provisioning` extra. Its README describes the input and output files and +how to run it. Item 5.04 removes it. + +## Connections + +`create_simulation_engine` builds the runtime engine from the canonical +`postgresql://` URL the deployment stores, selecting the psycopg 3 driver. It +keeps a small pool, checks connections before use, and never prepares +server-side statements, so the same URL works through Supabase's session and +transaction poolers. A malformed URL raises `ValueError` without repeating the +URL, which holds a password. + +`read_schema_revision` returns the migration revision the database has applied, +or `None` before the first migration. `HEAD_REVISION` is the newest revision +this code knows. + +## Adding a revision + +1. Change the tables in `src/policyengine_simulation_persistence/`. +2. Add a revision under `migrations/versions/` whose `down_revision` is the + current head, and set `HEAD_REVISION` in `schema.py` to its revision ID. +3. Run the tests against a disposable PostgreSQL server (below). They run the + chain's upgrade, `alembic check`, downgrade and upgrade cycle. + +The schema creation and every pending revision apply in one transaction, so a +failed migration leaves nothing applied. A statement that waits more than ten +seconds for a lock fails the migration instead of waiting longer, because +adding a foreign key locks the Public API table it refers to and the Public +API's writes would queue behind it; the next deployment tries again. The previous release keeps serving +traffic while a deployment migrates, and a rollback moves traffic back to an +older image without downgrading the schema, so every revision must work with +the code of the release before it: add columns and tables, and remove them +only in a later release once nothing reads them. + +Only one migration PR in this chain may merge at a time; another one open at +the same time rebases onto the new head after the first merges. + +## Tests + +```bash +uv sync --frozen --extra test +uv run --frozen pytest tests/ +``` + +Tests marked `postgres` need a disposable PostgreSQL server. Point +`SIMULATION_TEST_DATABASE_URL` at its superuser, for example +`postgresql://postgres:password@127.0.0.1:5432/postgres`; each test creates and +drops its own database. Without the variable those tests skip. CI's +`Simulation database` job runs them against `postgres:17` with +`SIMULATION_REQUIRE_DATABASE_TESTS=1`, which turns a skip into a failure. diff --git a/libs/policyengine-simulation-persistence/pyproject.toml b/libs/policyengine-simulation-persistence/pyproject.toml new file mode 100644 index 000000000..e00f062dd --- /dev/null +++ b/libs/policyengine-simulation-persistence/pyproject.toml @@ -0,0 +1,55 @@ +[project] +name = "policyengine-simulation-persistence" +version = "0.1.0" +readme = "README.md" +authors = [ + {name = "PolicyEngine", email = "hello@policyengine.org"}, +] +license = {file = "../../LICENSE"} +requires-python = ">=3.13,<3.14" +dependencies = [ + "psycopg[binary]>=3.2,<4", + "sqlalchemy>=2,<3", +] + +[project.optional-dependencies] +# The migration job installs Alembic; the Simulation Entrypoint runtime does not. +# 1.17.1 added `alembic current --check-heads`, which the migration job runs. +migrations = [ + "alembic>=1.17.1,<2", +] +# The one-time migration provisioning command (migration_provisioning) reads +# its input file with Pydantic; nothing else in the library needs it. +provisioning = [ + "pydantic>=2.11,<3", +] +test = [ + "alembic>=1.17.1,<2", + "pydantic>=2.11,<3", + "pytest>=8.3.4", + "pytest-cov>=6.1.1", + "pyyaml>=6.0.2", +] +build = [ + "pyright>=1.1.401", +] + +[build-system] +requires = ["hatchling>=1.26"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +sources = ["src"] +only-include = ["src/policyengine_simulation_persistence"] +packages = ["src/policyengine_simulation_persistence"] + +[tool.pytest.ini_options] +pythonpath = ["src"] +testpaths = ["tests"] +markers = [ + "postgres: needs a disposable PostgreSQL server named by SIMULATION_TEST_DATABASE_URL", +] + +[tool.pyright] +# The pr.yml pyright job checks these paths: python -m scripts.rules pyright. +include = ["src"] diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py new file mode 100644 index 000000000..5e87ac336 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/__init__.py @@ -0,0 +1,49 @@ +"""Tables and connections for the Simulation Entrypoint's ``simulation`` schema. + +Importing the package registers every table in ``simulation_metadata``, which +the migration environment compares with the database. +""" + +from policyengine_simulation_persistence.engine import ( + create_simulation_engine, + normalized_database_url, + read_schema_revision, +) +from policyengine_simulation_persistence.reports import ( + NewReport, + ReportKind, + ReportRecord, + ReportStore, +) +from policyengine_simulation_persistence.schema import ( + ALEMBIC_VERSION_TABLE, + HEAD_REVISION, + RUNTIME_ROLE, + SIMULATION_SCHEMA, + simulation_metadata, +) +from policyengine_simulation_persistence.simulations import ( + NewSimulation, + SimulationKind, + SimulationRecord, + SimulationStore, +) + +__all__ = [ + "ALEMBIC_VERSION_TABLE", + "HEAD_REVISION", + "RUNTIME_ROLE", + "SIMULATION_SCHEMA", + "NewReport", + "NewSimulation", + "ReportKind", + "ReportRecord", + "ReportStore", + "SimulationKind", + "SimulationRecord", + "SimulationStore", + "create_simulation_engine", + "normalized_database_url", + "read_schema_revision", + "simulation_metadata", +] diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py new file mode 100644 index 000000000..58e70b3dc --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/engine.py @@ -0,0 +1,76 @@ +"""Database connections for the Simulation Entrypoint's own schema.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from sqlalchemy import create_engine, inspect, select +from sqlalchemy.engine import URL, make_url +from sqlalchemy.exc import ArgumentError + +from policyengine_simulation_persistence.schema import ( + ALEMBIC_VERSION_TABLE, + SIMULATION_SCHEMA, + alembic_version, +) + +if TYPE_CHECKING: + from sqlalchemy import Connection, Engine + +CONNECT_TIMEOUT_SECONDS = 5 +# Each Cloud Run instance serves up to 80 concurrent requests. Each database +# operation holds a connection for one short transaction, so a small pool is +# enough. +POOL_SIZE = 2 +MAX_OVERFLOW = 3 + + +def normalized_database_url(database_url: str) -> URL: + """Return ``database_url`` with the psycopg 3 driver selected. + + The stored secrets use the canonical ``postgresql://`` scheme. Raise + ValueError, without echoing the value, when the URL is missing, malformed + or incomplete. + """ + if not database_url or database_url != database_url.strip(): + msg = "Simulation database URL is missing or malformed" + raise ValueError(msg) + try: + parsed = make_url(database_url) + except (ArgumentError, ValueError): + # Raised without the parser's message, which may quote the URL. + msg = "Simulation database URL is malformed" + raise ValueError(msg) from None + if parsed.drivername != "postgresql": + msg = "Simulation database URL must use postgresql://" + raise ValueError(msg) + if not parsed.username or not parsed.host or not parsed.database: + msg = "Simulation database URL is incomplete" + raise ValueError(msg) + return parsed.set(drivername="postgresql+psycopg") + + +def create_simulation_engine(database_url: str) -> Engine: + """Build the engine the Simulation Entrypoint uses; it creates no objects.""" + return create_engine( + normalized_database_url(database_url), + pool_size=POOL_SIZE, + max_overflow=MAX_OVERFLOW, + pool_pre_ping=True, + connect_args={ + "connect_timeout": CONNECT_TIMEOUT_SECONDS, + # Never use server-side prepared statements, so the same engine + # works through Supabase's session and transaction poolers. + "prepare_threshold": None, + }, + ) + + +def read_schema_revision(connection: Connection) -> str | None: + """Return the applied Alembic revision, or None before the first migration.""" + if not inspect(connection).has_table( + ALEMBIC_VERSION_TABLE, + schema=SIMULATION_SCHEMA, + ): + return None + return connection.execute(select(alembic_version.c.version_num)).scalar() diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/README.md b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/README.md new file mode 100644 index 000000000..7d56b1390 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/README.md @@ -0,0 +1,226 @@ +# Migration provisioning + +During the one-time migration, the Public API exports the v1 reports that +account rows refer to. This package reads that export and creates each +report's simulations in `simulation.simulations` and the report in +`simulation.reports`, with no run. The first time a migrated report is read, +the Public API calculates it on the current catalog version. + +It is temporary: item 5.04 removes this package. + +## Input file: report definitions, schema version 1 + +One JSON object. The models in `definitions.py` are the format's definition; +this section describes them. + +| Field | Type | Meaning | +| --- | --- | --- | +| `schema_version` | `1` | The format's version. Any other value is rejected. | +| `reports` | array | One definition per v1 report. Each `legacy_report_id` appears once. | + +Each report definition: + +| Field | Type | Meaning | +| --- | --- | --- | +| `kind` | `"economy"` or `"household"` | The report's kind, and its simulations' kind. | +| `legacy_report_id` | positive integer, at most 2^63 − 1 | The v1 report ID. | +| `country` | two lowercase letters | The report's country, and its simulations'. | +| `year` | integer, 1900 to 2200 | The year the report calculates. | +| `options` | JSON object | The report's options, stored as given. Numbers are read as integers or 64-bit floats, so a decimal with more digits than a float holds is stored rounded. | +| `baseline` | simulation | The baseline simulation. | +| `reform` | simulation, or `null` for a household report | The reform simulation. An economy report must have one. | + +Each simulation of an economy report: + +| Field | Type | Meaning | +| --- | --- | --- | +| `legacy_simulation_id` | positive integer, at most 2^63 − 1 | The v1 simulation ID. | +| `policy_id` | UUID or `null` | The v2 policy that `legacy_policy_mappings` maps the v1 policy to, or `null` for current law. | +| `region_code` | text, 1 to 255 characters, no whitespace | The v2 region code the Public API's region reference files map the v1 region to: `us`, `state/ca`, `country/northern_ireland`, `constituency/E14001063`. | +| `dataset_name` | text, 1 to 255 characters, no whitespace, or `null` | A catalog dataset name, or `null` for the region's default dataset in each catalog version. | + +Each simulation of a household report: + +| Field | Type | Meaning | +| --- | --- | --- | +| `legacy_simulation_id` | positive integer, at most 2^63 − 1 | The v1 simulation ID. | +| `policy_id` | UUID or `null` | As for an economy simulation. | +| `household_id` | UUID | The v2 household that `legacy_household_mappings` maps the v1 household to. | + +Rules the file must follow, checked before anything is written: + +- Every field is required, including those that may be `null`, and no other + field is allowed. +- Numbers are JSON numbers and IDs are JSON strings; nothing is converted from + text. +- Nothing holds a value the database cannot store: no text, including the + options' keys and values, contains a NUL character, and the options hold no + NaN or Infinity. +- An economy report's two simulations have the same `region_code` and + `dataset_name`; a household report's have the same `household_id`. +- A v1 simulation that several reports use is described the same way in each. +- A v1 economy report with a single simulation, which v1 calculates as that + policy over itself, names the same simulation as `baseline` and `reform`. + +The export resolves every v2 reference itself. This package reads no Public API +table and no region reference file, and the runtime role it connects as has no +privilege on the Public API's tables. + +Example: + +```json +{ + "schema_version": 1, + "reports": [ + { + "kind": "economy", + "legacy_report_id": 4512, + "country": "us", + "year": 2026, + "options": {}, + "baseline": { + "legacy_simulation_id": 9001, + "policy_id": null, + "region_code": "state/ca", + "dataset_name": null + }, + "reform": { + "legacy_simulation_id": 9002, + "policy_id": "0b6f1c9e-3f43-4d6a-9d51-6f3a2c1b8e07", + "region_code": "state/ca", + "dataset_name": null + } + }, + { + "kind": "household", + "legacy_report_id": 4513, + "country": "uk", + "year": 2025, + "options": {}, + "baseline": { + "legacy_simulation_id": 9003, + "policy_id": null, + "household_id": "5d0c3a48-2b8e-4f1e-a7c2-9e4b6d1f0a33" + }, + "reform": null + } + ] +} +``` + +## Running it + +An operator runs it from a checkout of `main`, once per environment, after +the Public API's migration command has written the export. It is not an HTTP +route and runs nowhere else. + +1. Install the library with its `provisioning` extra: + + ```bash + cd libs/policyengine-simulation-persistence + uv sync --frozen --extra provisioning + ``` + +2. Put the runtime role's URL in `SIMULATION_DATABASE_URL`, the variable the + Simulation Entrypoint reads. The deploy mounts the newest enabled version + of the Secret Manager secret that the `SIMULATION_DATABASE_URL_SECRET_NAME` + variable of sim's GitHub environment names (`beta` for staging, `prod` for + production): a secret name in the project named by + `SIMULATION_ENTRYPOINT_GCP_PROJECT_ID`, or a full + `projects//secrets/` reference. Only the entrypoint's runtime + service account can read it today, so the operator needs read access to it + first. `latest` below names the newest version; if that version is + disabled, give the newest enabled version's number instead. + + ```bash + SIMULATION_DATABASE_URL="$(gcloud secrets versions access latest \ + --secret="$SECRET_NAME" --project="$PROJECT_ID")" + export SIMULATION_DATABASE_URL + ``` + +3. Dry run, then review its counts and failures: + + ```bash + uv run --frozen --extra provisioning \ + python -m policyengine_simulation_persistence.migration_provisioning \ + --definitions report-definitions.json --dry-run + ``` + +4. Run, writing the report mapping to a new file: + + ```bash + uv run --frozen --extra provisioning \ + python -m policyengine_simulation_persistence.migration_provisioning \ + --definitions report-definitions.json --output report-mapping.json + ``` + +Before it reads or writes a row, the command checks: + +- the definitions file, listing up to 20 errors; +- for a run, that the output file does not exist yet and that its directory + exists and can be written to; +- that `SIMULATION_DATABASE_URL` is set, uses `postgresql://` and logs in as + the runtime role, `simulation_runtime`, never the migrator; +- that the database is at the schema revision this code expects. + +For each definition it creates the simulations, keyed by +`legacy_simulation_id`, and then the report, keyed by `legacy_report_id`, +through the stores' `create_migrated` methods. A row that already exists is +reused if it matches the definition, so running the same export again creates +nothing and maps the same reports. A row that exists with a different +definition, or a definition the database rejects (a policy or household the +Public API does not have, one of another country, or another value it rejects +as data), fails that report alone; the other reports continue. The file check +already rejects the values it knows the database cannot store. + +Before it creates any of a report's rows, a run reads every row the report +needs that already exists and checks it against the definition, so a report +that conflicts with an existing row creates nothing. Each row is committed on +its own: if the database rejects a report after some of its simulations were +created, they stay, keyed by their v1 IDs, and a later run reuses them. + +The dry run only reads. It counts the simulations and reports a run would +create and those that already exist, and reports conflicts with existing rows. +The runtime role cannot read the Public API's tables, so a dry run does not +find a policy or household the database would reject; the run reports those. + +Both print a JSON summary: `dry_run`, `new_simulations`, +`existing_simulations`, `new_reports`, `existing_reports` (in a dry run, the +`new_` counts are what a run would create) and `failures`. + +| Exit status | Meaning | +| --- | --- | +| 0 | Every report was provisioned, or in a dry run would be. | +| 1 | Some reports failed, or in a dry run would fail. The summary lists them, and so does a run's output file; the other reports are provisioned. | +| 2 | The command refused to start. Nothing was written. | +| 3 | The database stopped the command partway, or a run's output file could not be written. A run's rows created before the stop are kept, and running the command again, with a new output file, reuses them. A dry run writes nothing. | + +## Output file: report mapping, schema version 1 + +A run writes it at the end; `mapping.py` defines it. The Public API's migration +command reads it and sets each account row's v2 report ID. + +| Field | Type | Meaning | +| --- | --- | --- | +| `schema_version` | `1` | The format's version. | +| `reports` | array | Each provisioned report: `legacy_report_id`, its v2 `report_id`, and `created` (`false` when an earlier run had created it). | +| `failures` | array | Each failed report: `legacy_report_id` and a one-line `reason`. | + +```json +{ + "schema_version": 1, + "reports": [ + { + "legacy_report_id": 4512, + "report_id": "6c1d7f0a-8f5e-4a3b-9c2d-1e4f7a9b3c55", + "created": true + } + ], + "failures": [ + { + "legacy_report_id": 4513, + "reason": "the database rejected v1 simulation 9003: insert or update on table \"simulations\" violates foreign key constraint \"fk_simulations_household_country\"" + } + ] +} +``` diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/__init__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/__init__.py new file mode 100644 index 000000000..ba6a20ceb --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/__init__.py @@ -0,0 +1,52 @@ +"""One-time migration provisioning: v1 report definitions into the simulation schema. + +An operator runs this once per environment, during the one-time migration. It +reads the Public API's export of v1 report definitions and creates each +report's simulations and the report itself, with no run. Item 5.04 removes this +package. ``README.md`` next to it describes the input file and how to run it. + +It needs the library's ``provisioning`` extra, which installs Pydantic. The +library's own package does not import it. +""" + +from policyengine_simulation_persistence.migration_provisioning.definitions import ( + REPORT_DEFINITIONS_SCHEMA_VERSION, + EconomyReportDefinition, + EconomySimulationDefinition, + HouseholdReportDefinition, + HouseholdSimulationDefinition, + ReportDefinition, + ReportDefinitionsFile, + SimulationDefinition, +) +from policyengine_simulation_persistence.migration_provisioning.mapping import ( + REPORT_MAPPING_SCHEMA_VERSION, + FailedReport, + MappedReport, + ReportMappingFile, +) +from policyengine_simulation_persistence.migration_provisioning.provisioning import ( + ProvisionedReport, + ProvisioningFailure, + ProvisioningResult, + provision_reports, +) + +__all__ = [ + "REPORT_DEFINITIONS_SCHEMA_VERSION", + "REPORT_MAPPING_SCHEMA_VERSION", + "EconomyReportDefinition", + "EconomySimulationDefinition", + "FailedReport", + "HouseholdReportDefinition", + "HouseholdSimulationDefinition", + "MappedReport", + "ProvisionedReport", + "ProvisioningFailure", + "ProvisioningResult", + "ReportDefinition", + "ReportDefinitionsFile", + "ReportMappingFile", + "SimulationDefinition", + "provision_reports", +] diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/__main__.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/__main__.py new file mode 100644 index 000000000..f3e079bf0 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/__main__.py @@ -0,0 +1,6 @@ +"""Run the migration provisioning command: see ``command.py`` and ``README.md``.""" + +from policyengine_simulation_persistence.migration_provisioning.command import main + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/command.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/command.py new file mode 100644 index 000000000..5b84de5be --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/command.py @@ -0,0 +1,279 @@ +"""The command an operator runs to provision the one-time migration's reports. + +``python -m policyengine_simulation_persistence.migration_provisioning +--definitions FILE (--output FILE | --dry-run)`` + +It connects with the URL in ``SIMULATION_DATABASE_URL``, which must log in as +the runtime role, to a database at the revision this code expects. Before it +connects it checks the definitions file and, for a run, that the output file +does not exist yet and that its directory exists and can be written to. A run +writes the output file at the end; a dry run writes nothing. Both print a JSON +summary on standard output. + +Exit status: 0 when every report was provisioned (or, in a dry run, would be); +1 when some reports failed (or would fail), which the summary and a run's +output file list; 2 when the command refused to start, and nothing was +written; 3 when the database stopped the command partway, or a run's output +file could not be written. Rows a run created before a stop are kept, and +running the command again reuses them; a dry run writes nothing. +""" + +from __future__ import annotations + +import argparse +import json +import os +import sys +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import TYPE_CHECKING + +from pydantic import ValidationError +from sqlalchemy import func, select +from sqlalchemy.exc import SQLAlchemyError + +from policyengine_simulation_persistence.engine import ( + create_simulation_engine, + read_schema_revision, +) +from policyengine_simulation_persistence.migration_provisioning.definitions import ( + ReportDefinitionsFile, +) +from policyengine_simulation_persistence.migration_provisioning.mapping import ( + ReportMappingFile, +) +from policyengine_simulation_persistence.migration_provisioning.provisioning import ( + database_error_summary, + provision_reports, +) +from policyengine_simulation_persistence.reports import ReportStore +from policyengine_simulation_persistence.schema import HEAD_REVISION, RUNTIME_ROLE +from policyengine_simulation_persistence.simulations import SimulationStore + +if TYPE_CHECKING: + from collections.abc import Mapping, Sequence + + from sqlalchemy import Engine + + from policyengine_simulation_persistence.migration_provisioning import ( + ProvisioningResult, + ) + +DATABASE_URL_VARIABLE = "SIMULATION_DATABASE_URL" +EXIT_PROVISIONED = 0 +EXIT_REPORTS_FAILED = 1 +EXIT_REFUSED = 2 +EXIT_STOPPED = 3 +# A definitions file with more errors lists only the first ones. +LISTED_ERROR_LIMIT = 20 + + +class CommandRefusedError(Exception): + """The command cannot start; the message says why.""" + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + prog="python -m policyengine_simulation_persistence.migration_provisioning", + description=( + "Create the simulations and reports of exported v1 report " + "definitions, with no run, and write each v1 report's v2 report ID. " + f"Connects with the runtime role's URL in {DATABASE_URL_VARIABLE}." + ), + ) + parser.add_argument( + "--definitions", + type=Path, + required=True, + help="the Public API's export of v1 report definitions (JSON)", + ) + parser.add_argument( + "--output", + type=Path, + help="where a run writes the report mapping; it must not exist yet", + ) + parser.add_argument( + "--dry-run", + action="store_true", + help="read the database and report what a run would do, writing nothing", + ) + return parser + + +@dataclass(frozen=True) +class _Arguments: + definitions: Path + # Where a run writes the report mapping; None for a dry run. + output: Path | None + + @property + def dry_run(self) -> bool: + return self.output is None + + +def _parse_arguments(argv: Sequence[str] | None) -> _Arguments: + parser = _parser() + parsed = parser.parse_args(argv) + if parsed.dry_run and parsed.output is not None: + parser.error("--dry-run writes no file, so it takes no --output") + if not parsed.dry_run and parsed.output is None: + parser.error("a run needs --output, or use --dry-run") + return _Arguments(definitions=parsed.definitions, output=parsed.output) + + +def _check_output_path(output: Path) -> None: + if output.exists(): + msg = f"{output} already exists; choose a new output file" + raise CommandRefusedError(msg) + if not output.parent.is_dir(): + msg = f"{output.parent} is not a directory" + raise CommandRefusedError(msg) + if not os.access(output.parent, os.W_OK | os.X_OK): + msg = f"cannot write to {output.parent}" + raise CommandRefusedError(msg) + + +def _validation_lines(error: ValidationError) -> list[str]: + details = error.errors(include_url=False, include_input=False) + lines = [ + " " + + (".".join(str(part) for part in detail["loc"]) or "file") + + ": " + + detail["msg"] + for detail in details[:LISTED_ERROR_LIMIT] + ] + if len(details) > LISTED_ERROR_LIMIT: + lines.append(f" and {len(details) - LISTED_ERROR_LIMIT} more") + return lines + + +def _read_definitions(path: Path) -> ReportDefinitionsFile: + try: + content = path.read_bytes() + except OSError as error: + msg = f"cannot read {path}: {error.strerror}" + raise CommandRefusedError(msg) from error + try: + return ReportDefinitionsFile.model_validate_json(content) + except ValidationError as error: + msg = "\n".join( + [ + f"{path} is not a valid report definitions file:", + *_validation_lines(error), + ] + ) + raise CommandRefusedError(msg) from error + + +def _runtime_engine(environ: Mapping[str, str]) -> Engine: + database_url = environ.get(DATABASE_URL_VARIABLE, "") + if not database_url: + msg = f"{DATABASE_URL_VARIABLE} is not set" + raise CommandRefusedError(msg) + try: + return create_simulation_engine(database_url) + except ValueError as error: + # The message never repeats the URL, which holds a password. + raise CommandRefusedError(str(error)) from None + + +def _check_database(engine: Engine) -> None: + try: + with engine.connect() as connection: + role = connection.execute(select(func.current_user())).scalar_one() + revision = read_schema_revision(connection) + except SQLAlchemyError as error: + msg = ( + "cannot connect to the simulation database: " + f"{database_error_summary(error)}" + ) + raise CommandRefusedError(msg) from None + if role != RUNTIME_ROLE: + msg = ( + f"connected as {role}, " + f"but migration provisioning connects as {RUNTIME_ROLE}" + ) + raise CommandRefusedError(msg) + if revision != HEAD_REVISION: + msg = ( + f"the simulation database is at revision {revision}, " + f"but this code expects {HEAD_REVISION}" + ) + raise CommandRefusedError(msg) + + +def _summary(result: ProvisioningResult) -> str: + return json.dumps( + { + "dry_run": result.dry_run, + "new_simulations": result.new_simulations, + "existing_simulations": result.existing_simulations, + "new_reports": result.new_reports, + "existing_reports": result.existing_reports, + "failures": [asdict(failure) for failure in result.failures], + }, + indent=2, + ) + + +def _write_mapping(output: Path, result: ProvisioningResult) -> None: + content = ReportMappingFile.from_result(result).model_dump_json(indent=2) + # Mode "x" fails rather than replace a file created since the check. + with output.open("x", encoding="utf-8") as mapping_file: + mapping_file.write(content + "\n") + + +def main( + argv: Sequence[str] | None = None, environ: Mapping[str, str] | None = None +) -> int: + """Run the command and return its exit status.""" + arguments = _parse_arguments(argv) + try: + if arguments.output is not None: + _check_output_path(arguments.output) + definitions = _read_definitions(arguments.definitions) + engine = _runtime_engine(os.environ if environ is None else environ) + except CommandRefusedError as refusal: + sys.stderr.write(f"{refusal}\n") + return EXIT_REFUSED + try: + _check_database(engine) + result = provision_reports( + definitions, + SimulationStore(engine), + ReportStore(engine), + dry_run=arguments.dry_run, + ) + except CommandRefusedError as refusal: + sys.stderr.write(f"{refusal}\n") + return EXIT_REFUSED + except SQLAlchemyError as error: + summary = database_error_summary(error) + if arguments.dry_run: + message = ( + f"the database stopped the dry run: {summary}\n" + "The dry run wrote nothing.\n" + ) + else: + message = ( + f"the database stopped the run: {summary}\n" + "No output file was written. The rows created so far are kept, " + "and running the command again reuses them.\n" + ) + sys.stderr.write(message) + return EXIT_STOPPED + finally: + engine.dispose() + sys.stdout.write(_summary(result) + "\n") + if arguments.output is not None: + try: + _write_mapping(arguments.output, result) + except OSError as error: + sys.stderr.write( + f"cannot write {arguments.output}: {error.strerror}\n" + "The run finished, but no output file was written. Running the " + "command again with a new --output reuses every row it created " + "and writes the mapping.\n" + ) + return EXIT_STOPPED + return EXIT_REPORTS_FAILED if result.failures else EXIT_PROVISIONED diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/definitions.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/definitions.py new file mode 100644 index 000000000..e77ea3625 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/definitions.py @@ -0,0 +1,272 @@ +"""The export file of v1 report definitions, schema version 1. + +The Public API's one-time migration command exports one definition per v1 +report that an account row refers to. Each definition already holds v2 +references: the v2 policy and household IDs that ``legacy_policy_mappings`` +and ``legacy_household_mappings`` map the v1 IDs to, and the region code that +the Public API's region reference files map the v1 region to. This module +reads no Public API table and no reference file. ``README.md`` next to it +describes the format with an example. + +Every field is required, even where null is allowed, so a field the export +leaves out is an error rather than a silent default. Numbers are never +converted from text. Values the database cannot store, such as text holding a +NUL character or NaN in the options, are rejected here, before anything is +written. +""" + +from __future__ import annotations + +import math +from abc import ABC, abstractmethod +from collections import Counter +from typing import TYPE_CHECKING, Annotated, Literal +from uuid import UUID + +from pydantic import ( + AfterValidator, + BaseModel, + ConfigDict, + Field, + JsonValue, + model_validator, +) + +from policyengine_simulation_persistence.reports import ( + FIRST_YEAR, + LAST_YEAR, + NewReport, + ReportKind, +) +from policyengine_simulation_persistence.simulations import ( + NewSimulation, + SimulationKind, +) + +if TYPE_CHECKING: + from typing import Self + +REPORT_DEFINITIONS_SCHEMA_VERSION = 1 +# The legacy ID columns are bigint, so a larger ID could not be stored. +LARGEST_LEGACY_ID = 2**63 - 1 + +type LegacyId = Annotated[int, Field(gt=0, le=LARGEST_LEGACY_ID)] + + +def _reject_nul(text: str) -> str: + if "\x00" in text: + msg = "text must not contain a NUL character, which the database cannot store" + raise ValueError(msg) + return text + + +def _check_storable(value: object) -> None: + if isinstance(value, float) and not math.isfinite(value): + msg = "options must not hold NaN or Infinity, which JSON does not allow" + raise ValueError(msg) + if isinstance(value, str): + _reject_nul(value) + elif isinstance(value, dict): + for key, item in value.items(): + _reject_nul(str(key)) + _check_storable(item) + elif isinstance(value, list): + for item in value: + _check_storable(item) + + +def _storable_options(options: dict[str, JsonValue]) -> dict[str, JsonValue]: + _check_storable(options) + return options + + +type StorableText = Annotated[str, AfterValidator(_reject_nul)] +type StorableOptions = Annotated[ + dict[str, JsonValue], AfterValidator(_storable_options) +] + + +class _DefinitionModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class EconomySimulationDefinition(_DefinitionModel): + """One v1 economy simulation: a policy applied to a region.""" + + legacy_simulation_id: LegacyId + # The v2 policy ID, or null for current law. + policy_id: UUID | None + # The v2 region code, such as ``state/ca`` or ``constituency/E14001063``. + region_code: StorableText + # A dataset name in the catalog, or null for the region's default dataset. + dataset_name: StorableText | None + + def new_simulation(self, country: str) -> NewSimulation: + """Return the simulation to create for this v1 simulation.""" + return NewSimulation( + kind=SimulationKind.ECONOMY, + country=country, + policy_id=self.policy_id, + region_code=self.region_code, + dataset_name=self.dataset_name, + legacy_simulation_id=self.legacy_simulation_id, + ) + + +class HouseholdSimulationDefinition(_DefinitionModel): + """One v1 household simulation: a policy applied to a household.""" + + legacy_simulation_id: LegacyId + # The v2 policy ID, or null for current law. + policy_id: UUID | None + # The v2 household ID. + household_id: UUID + + def new_simulation(self, country: str) -> NewSimulation: + """Return the simulation to create for this v1 simulation.""" + return NewSimulation( + kind=SimulationKind.HOUSEHOLD, + country=country, + policy_id=self.policy_id, + household_id=self.household_id, + legacy_simulation_id=self.legacy_simulation_id, + ) + + +type SimulationDefinition = EconomySimulationDefinition | HouseholdSimulationDefinition + + +class _ReportDefinition(_DefinitionModel, ABC): + legacy_report_id: LegacyId + country: str + year: Annotated[int, Field(ge=FIRST_YEAR, le=LAST_YEAR)] + options: StorableOptions + + @abstractmethod + def simulations(self) -> tuple[SimulationDefinition, ...]: + """Return the baseline, then the reform if the report has one.""" + + @abstractmethod + def report_kind(self) -> ReportKind: + """Return the kind of the report and of its simulations.""" + + def new_simulation(self, simulation: SimulationDefinition) -> NewSimulation: + """Return the simulation to create for one of this report's simulations.""" + return simulation.new_simulation(self.country) + + def new_report( + self, baseline_simulation_id: UUID, reform_simulation_id: UUID | None + ) -> NewReport: + """Return the report to create once its simulations exist.""" + return NewReport( + kind=self.report_kind(), + country=self.country, + baseline_simulation_id=baseline_simulation_id, + reform_simulation_id=reform_simulation_id, + year=self.year, + options=self.options, + legacy_report_id=self.legacy_report_id, + ) + + def _check_simulations(self) -> None: + # The simulation tables' own checks, run before anything is written. + for simulation in self.simulations(): + self.new_simulation(simulation) + + +class EconomyReportDefinition(_ReportDefinition): + """A v1 economy report: a reform and a baseline policy on one region.""" + + kind: Literal["economy"] + baseline: EconomySimulationDefinition + reform: EconomySimulationDefinition + + def simulations(self) -> tuple[SimulationDefinition, ...]: + """Return the baseline, then the reform.""" + return (self.baseline, self.reform) + + def report_kind(self) -> ReportKind: + """Return ``economy``.""" + return ReportKind.ECONOMY + + @model_validator(mode="after") + def _check_population(self) -> Self: + self._check_simulations() + population = (self.baseline.region_code, self.baseline.dataset_name) + if (self.reform.region_code, self.reform.dataset_name) != population: + msg = ( + "an economy report's simulations must have " + "the same region_code and dataset_name" + ) + raise ValueError(msg) + return self + + +class HouseholdReportDefinition(_ReportDefinition): + """A v1 household report: one household under one policy, or under two.""" + + kind: Literal["household"] + baseline: HouseholdSimulationDefinition + reform: HouseholdSimulationDefinition | None + + def simulations(self) -> tuple[SimulationDefinition, ...]: + """Return the baseline, then the reform if there is one.""" + if self.reform is None: + return (self.baseline,) + return (self.baseline, self.reform) + + def report_kind(self) -> ReportKind: + """Return ``household``.""" + return ReportKind.HOUSEHOLD + + @model_validator(mode="after") + def _check_population(self) -> Self: + self._check_simulations() + if self.reform is not None and ( + self.reform.household_id != self.baseline.household_id + ): + msg = "a household report's simulations must have the same household_id" + raise ValueError(msg) + return self + + +type ReportDefinition = Annotated[ + EconomyReportDefinition | HouseholdReportDefinition, + Field(discriminator="kind"), +] + + +def _listed(legacy_ids: set[int]) -> str: + return ", ".join(str(legacy_id) for legacy_id in sorted(legacy_ids)) + + +class ReportDefinitionsFile(_DefinitionModel): + """The whole export: its schema version and every report definition.""" + + schema_version: Literal[1] + reports: list[ReportDefinition] + + @model_validator(mode="after") + def _check_legacy_ids(self) -> Self: + report_counts = Counter(report.legacy_report_id for report in self.reports) + repeated_reports = { + legacy_id for legacy_id, count in report_counts.items() if count > 1 + } + if repeated_reports: + msg = f"v1 reports appear more than once: {_listed(repeated_reports)}" + raise ValueError(msg) + descriptions: dict[int, NewSimulation] = {} + conflicting: set[int] = set() + for report in self.reports: + for simulation in report.simulations(): + new_simulation = report.new_simulation(simulation) + legacy_id = simulation.legacy_simulation_id + if descriptions.setdefault(legacy_id, new_simulation) != new_simulation: + conflicting.add(legacy_id) + if conflicting: + msg = ( + "v1 simulations are described differently in different reports: " + f"{_listed(conflicting)}" + ) + raise ValueError(msg) + return self diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/mapping.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/mapping.py new file mode 100644 index 000000000..ae0adf4b3 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/mapping.py @@ -0,0 +1,69 @@ +"""The file a run writes: each v1 report's v2 report ID, schema version 1. + +The Public API's one-time migration command reads it and sets each account +row's v2 report ID from it. ``README.md`` next to this module describes it. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Literal +from uuid import UUID + +from pydantic import BaseModel, ConfigDict, PositiveInt + +if TYPE_CHECKING: + from policyengine_simulation_persistence.migration_provisioning import ( + ProvisioningResult, + ) + +REPORT_MAPPING_SCHEMA_VERSION = 1 + + +class _MappingModel(BaseModel): + model_config = ConfigDict(extra="forbid", frozen=True, strict=True) + + +class MappedReport(_MappingModel): + """A v1 report and the v2 report that holds its definition.""" + + legacy_report_id: PositiveInt + report_id: UUID + # False when an earlier run had already created the report. + created: bool + + +class FailedReport(_MappingModel): + """A v1 report the run could not provision, and why.""" + + legacy_report_id: PositiveInt + reason: str + + +class ReportMappingFile(_MappingModel): + """Every provisioned v1 report's v2 report ID, and every failed report.""" + + schema_version: Literal[1] + reports: list[MappedReport] + failures: list[FailedReport] + + @classmethod + def from_result(cls, result: ProvisioningResult) -> ReportMappingFile: + """Describe a run's result, in the order of the export's definitions.""" + return cls( + schema_version=REPORT_MAPPING_SCHEMA_VERSION, + reports=[ + MappedReport( + legacy_report_id=provisioned.legacy_report_id, + report_id=provisioned.report_id, + created=provisioned.created, + ) + for provisioned in result.reports + ], + failures=[ + FailedReport( + legacy_report_id=failure.legacy_report_id, + reason=failure.reason, + ) + for failure in result.failures + ], + ) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/provisioning.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/provisioning.py new file mode 100644 index 000000000..9fa0f41b4 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/migration_provisioning/provisioning.py @@ -0,0 +1,338 @@ +"""Create the simulations and reports of exported v1 report definitions. + +Each definition's simulations are created first, keyed by +``legacy_simulation_id``, then its report, keyed by ``legacy_report_id``, +through the stores' ``create_migrated`` methods. A row that already exists is +reused if it matches the definition, so running the same export again creates +nothing. A row that exists with a different definition, or a definition the +database rejects (such as a policy the Public API does not have), fails that +report alone; the other reports continue. So does a value the database rejects +as data; the file check already rejects those it knows, such as text holding a +NUL character. + +Before it creates any of a report's rows, a run reads every row the report +needs that already exists and checks it against the definition, so a conflict +creates nothing. The stores commit each row on its own: a report the database +rejects after some of its simulations were created leaves those simulations in +place, keyed by their v1 IDs, and a later run reuses them. + +A dry run only reads: it counts the rows a run would create and the rows that +already exist, and reports the definitions that conflict with existing rows. It +cannot see the Public API's tables, so it does not find a policy or household +the database would reject. +""" + +from __future__ import annotations + +from dataclasses import dataclass, fields +from dataclasses import field as dataclass_field +from decimal import Decimal +from typing import TYPE_CHECKING, overload + +from sqlalchemy.exc import DataError, DBAPIError, IntegrityError, SQLAlchemyError + +if TYPE_CHECKING: + from collections.abc import Callable + from uuid import UUID + + from policyengine_simulation_persistence.migration_provisioning import ( + ReportDefinition, + ReportDefinitionsFile, + SimulationDefinition, + ) + from policyengine_simulation_persistence.reports import ( + NewReport, + ReportRecord, + ReportStore, + ) + from policyengine_simulation_persistence.simulations import ( + NewSimulation, + SimulationRecord, + SimulationStore, + ) + + +@dataclass(frozen=True) +class ProvisionedReport: + """A v1 report and the v2 report that now holds its definition.""" + + legacy_report_id: int + report_id: UUID + # False when an earlier run had already created the report. + created: bool + + +@dataclass(frozen=True) +class ProvisioningFailure: + """A v1 report whose definition could not be provisioned, and why.""" + + legacy_report_id: int + reason: str + + +@dataclass +class ProvisioningResult: + """What one run did, or would do, in the order of the export's definitions. + + In a dry run, ``reports`` lists only the reports that already exist, and + the ``new_`` counts are the rows a run would create. + """ + + dry_run: bool + reports: list[ProvisionedReport] = dataclass_field(default_factory=list) + failures: list[ProvisioningFailure] = dataclass_field(default_factory=list) + # A simulation several reports use is counted once. + new_simulations: int = 0 + existing_simulations: int = 0 + new_reports: int = 0 + existing_reports: int = 0 + + +class DefinitionRejectedError(Exception): + """One report definition cannot be provisioned; the reason says why.""" + + def __init__(self, reason: str) -> None: + """Keep ``reason``, a single line of text.""" + super().__init__(reason) + self.reason = reason + + +def _as_stored(value: object) -> object: + """Return ``value`` with every JSON number as the exact decimal stored. + + JSONB keeps a number as an exact decimal, so the float 1e23 reads back as + the integer 100000000000000000000000, which is not equal to the float. A + float is stored as its shortest decimal form, which ``repr`` gives. + """ + if isinstance(value, bool): + return value + if isinstance(value, int): + return Decimal(value) + if isinstance(value, float): + return Decimal(repr(value)) + if isinstance(value, dict): + return {key: _as_stored(item) for key, item in value.items()} + if isinstance(value, list): + return [_as_stored(item) for item in value] + return value + + +@overload +def _check_matches( + record: SimulationRecord, expected: NewSimulation, subject: str +) -> None: ... + + +@overload +def _check_matches(record: ReportRecord, expected: NewReport, subject: str) -> None: ... + + +def _check_matches( + record: SimulationRecord | ReportRecord, + expected: NewSimulation | NewReport, + subject: str, +) -> None: + differing = [ + field.name + for field in fields(expected) + if _as_stored(getattr(record, field.name)) + != _as_stored(getattr(expected, field.name)) + ] + if differing: + msg = f"{subject} is already migrated with a different {', '.join(differing)}" + raise DefinitionRejectedError(msg) + + +def database_error_summary(error: SQLAlchemyError) -> str: + """Return the first line of the database's own message for ``error``. + + The driver's message names the constraint or the connection problem, and + never the password. SQLAlchemy's longer text adds the statement and its + parameters. + """ + original = error.orig if isinstance(error, DBAPIError) else None + lines = str(original if original is not None else error).strip().splitlines() + return lines[0] if lines else type(error).__name__ + + +def _create[RecordT](create: Callable[[], RecordT], subject: str) -> RecordT: + try: + return create() + except (IntegrityError, DataError) as error: + # A constraint the row breaks, or a value the database rejects as data + # that the file check does not know of. + msg = f"the database rejected {subject}: {database_error_summary(error)}" + raise DefinitionRejectedError(msg) from error + + +class _Provisioner: + def __init__( + self, + simulation_store: SimulationStore, + report_store: ReportStore, + *, + dry_run: bool, + ) -> None: + self.simulation_store = simulation_store + self.report_store = report_store + self.dry_run = dry_run + self.result = ProvisioningResult(dry_run=dry_run) + # v2 simulation IDs by v1 simulation ID: rows found or created so far. + self.simulation_ids: dict[int, UUID] = {} + # v1 simulation IDs a dry run found missing, which a run would create. + self.simulations_to_create: set[int] = set() + + def provision(self, definition: ReportDefinition) -> None: + try: + known_ids = [ + self._existing_simulation_id(definition, simulation) + for simulation in definition.simulations() + ] + existing_report = self._existing_report(definition, known_ids) + if existing_report is not None: + self._map_report(definition, existing_report.id, created=False) + elif self.dry_run: + self._count_creations(definition, known_ids) + else: + self._create_rows(definition, known_ids) + except DefinitionRejectedError as rejection: + self.result.failures.append( + ProvisioningFailure( + legacy_report_id=definition.legacy_report_id, + reason=rejection.reason, + ) + ) + + def _existing_simulation_id( + self, definition: ReportDefinition, simulation: SimulationDefinition + ) -> UUID | None: + """Return the existing simulation's v2 ID, or None if a run creates it. + + Reads only, and rejects the report if the row differs from the + definition. + """ + legacy_id = simulation.legacy_simulation_id + if legacy_id in self.simulation_ids: + return self.simulation_ids[legacy_id] + if legacy_id in self.simulations_to_create: + return None + record = self.simulation_store.get_by_legacy_simulation_id(legacy_id) + if record is None: + return None + _check_matches( + record, definition.new_simulation(simulation), f"v1 simulation {legacy_id}" + ) + self.simulation_ids[legacy_id] = record.id + self.result.existing_simulations += 1 + return record.id + + def _existing_report( + self, definition: ReportDefinition, known_ids: list[UUID | None] + ) -> ReportRecord | None: + """Return the existing report, if it matches the definition. Reads only.""" + legacy_id = definition.legacy_report_id + subject = f"v1 report {legacy_id}" + record = self.report_store.get_by_legacy_report_id(legacy_id) + if record is None: + return None + simulation_ids = [ + simulation_id for simulation_id in known_ids if simulation_id is not None + ] + if len(simulation_ids) < len(known_ids): + msg = ( + f"{subject} is already migrated, with simulations this run would create" + ) + raise DefinitionRejectedError(msg) + baseline_id, reform_id = _baseline_and_reform(simulation_ids) + _check_matches(record, definition.new_report(baseline_id, reform_id), subject) + return record + + def _count_creations( + self, definition: ReportDefinition, known_ids: list[UUID | None] + ) -> None: + for simulation, known_id in zip( + definition.simulations(), known_ids, strict=True + ): + legacy_id = simulation.legacy_simulation_id + if known_id is None and legacy_id not in self.simulations_to_create: + self.simulations_to_create.add(legacy_id) + self.result.new_simulations += 1 + self.result.new_reports += 1 + + def _create_rows( + self, definition: ReportDefinition, known_ids: list[UUID | None] + ) -> None: + simulation_ids = [ + known_id + if known_id is not None + else self._created_simulation_id(definition, simulation) + for simulation, known_id in zip( + definition.simulations(), known_ids, strict=True + ) + ] + subject = f"v1 report {definition.legacy_report_id}" + new_report = definition.new_report(*_baseline_and_reform(simulation_ids)) + record = _create(lambda: self.report_store.create_migrated(new_report), subject) + # Another writer may have created the report since it was read. + _check_matches(record, new_report, subject) + self._map_report(definition, record.id, created=True) + + def _created_simulation_id( + self, definition: ReportDefinition, simulation: SimulationDefinition + ) -> UUID: + legacy_id = simulation.legacy_simulation_id + if legacy_id in self.simulation_ids: + # The same v1 simulation is this report's baseline and reform. + return self.simulation_ids[legacy_id] + subject = f"v1 simulation {legacy_id}" + new_simulation = definition.new_simulation(simulation) + record = _create( + lambda: self.simulation_store.create_migrated(new_simulation), subject + ) + _check_matches(record, new_simulation, subject) + self.simulation_ids[legacy_id] = record.id + self.result.new_simulations += 1 + return record.id + + def _map_report( + self, definition: ReportDefinition, report_id: UUID, *, created: bool + ) -> None: + """Add the report to the mapping and count it as new or existing.""" + if created: + self.result.new_reports += 1 + else: + self.result.existing_reports += 1 + self.result.reports.append( + ProvisionedReport( + legacy_report_id=definition.legacy_report_id, + report_id=report_id, + created=created, + ) + ) + + +def _baseline_and_reform(simulation_ids: list[UUID]) -> tuple[UUID, UUID | None]: + """Split v2 simulation IDs listed in the order of ``simulations()``. + + That order is the baseline, then the reform if the report has one. + """ + baseline_id, *reform_ids = simulation_ids + return baseline_id, reform_ids[0] if reform_ids else None + + +def provision_reports( + definitions: ReportDefinitionsFile, + simulation_store: SimulationStore, + report_store: ReportStore, + *, + dry_run: bool = False, +) -> ProvisioningResult: + """Create every definition's simulations and report, and return the result. + + With ``dry_run``, write nothing and return what a run would do. Both stores + must connect as the runtime role. + """ + provisioner = _Provisioner(simulation_store, report_store, dry_run=dry_run) + for definition in definitions.reports: + provisioner.provision(definition) + return provisioner.result diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/public_api.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/public_api.py new file mode 100644 index 000000000..65f709a1f --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/public_api.py @@ -0,0 +1,43 @@ +"""The Public API tables that rows in the simulation schema refer to. + +The Public API owns these tables and creates and changes them with its own +migration chain, in the same database as the ``simulation`` schema. They are +described here only so the simulation tables can declare foreign keys to them: +each lists just the key columns those foreign keys use, in a metadata object of +its own, so the simulation chain never compares, creates or changes them. + +The tables have no schema here because PostgreSQL reports a foreign key into +``public`` without a schema when ``public`` is on the search path; naming one +would make every schema comparison report the foreign keys as changed. The +migrations name ``public`` explicitly. + +Every foreign key into these tables is ``ON DELETE RESTRICT``: the Public API +cannot delete a row a simulation uses. Creating the foreign keys needs +``REFERENCES`` on each table for the migrator role; the runtime role needs no +privilege on them, because PostgreSQL runs a foreign key's check as the owner +of the referenced table. +""" + +from __future__ import annotations + +from sqlalchemy import Column, MetaData, String, Table, Uuid + +public_api_metadata = MetaData() + +policies = Table( + "policies", + public_api_metadata, + Column("id", Uuid(as_uuid=True), primary_key=True), + Column("country_id", String(2), nullable=False), +) +households = Table( + "households", + public_api_metadata, + Column("id", Uuid(as_uuid=True), primary_key=True), + Column("country_id", String(2), nullable=False), +) +dynamics = Table( + "dynamics", + public_api_metadata, + Column("id", Uuid(as_uuid=True), primary_key=True), +) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/reports.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/reports.py new file mode 100644 index 000000000..99faccf40 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/reports.py @@ -0,0 +1,229 @@ +"""``simulation.reports``: which two simulations a report compares. + +A report is an economy or household comparison a user can save: its baseline +simulation, its reform simulation (required for an economy report; a household +report may have only a baseline), the year and further options. It is a +definition and never changes. Both simulations must have the report's country +and kind, which composite foreign keys into ``simulation.simulations`` check. + +A report's calculations are its report runs (item 1.06); a report with none has +not been calculated yet. A report migrated from the Public API's v1 database +keeps its v1 report ID in ``legacy_report_id``. +""" + +from __future__ import annotations + +from dataclasses import dataclass, fields +from dataclasses import field as dataclass_field +from enum import StrEnum +from typing import TYPE_CHECKING, Any +from uuid import UUID, uuid4 + +from sqlalchemy import ( + BigInteger, + CheckConstraint, + Column, + DateTime, + ForeignKeyConstraint, + Index, + Integer, + Table, + Text, + Uuid, + func, + select, + text, +) +from sqlalchemy.dialects.postgresql import JSONB +from sqlalchemy.dialects.postgresql import insert as postgres_insert + +from policyengine_simulation_persistence.schema import simulation_metadata +from policyengine_simulation_persistence.simulations import ( + check_country, + check_legacy_id, + insert_or_get_existing, + parse_kind, + simulations, +) + +if TYPE_CHECKING: + from datetime import datetime + + from sqlalchemy import ColumnElement, Engine, RowMapping + +FIRST_YEAR = 1900 +LAST_YEAR = 2200 + + +class ReportKind(StrEnum): + """What a report compares; its simulations have the same kind.""" + + ECONOMY = "economy" + HOUSEHOLD = "household" + + +_KIND_VALUES = ", ".join(f"'{kind.value}'" for kind in ReportKind) +_SIMULATION_KEY = [simulations.c.id, simulations.c.country, simulations.c.kind] + +reports = Table( + "reports", + simulation_metadata, + Column("id", Uuid(as_uuid=True), primary_key=True), + Column("kind", Text, nullable=False), + Column("country", Text, nullable=False), + Column("baseline_simulation_id", Uuid(as_uuid=True), nullable=False), + Column("reform_simulation_id", Uuid(as_uuid=True), nullable=True), + Column("year", Integer, nullable=False), + Column("options", JSONB, nullable=False, server_default=text("'{}'::jsonb")), + Column("legacy_report_id", BigInteger, nullable=True, unique=True), + Column( + "created_at", + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + ), + CheckConstraint(f"kind IN ({_KIND_VALUES})", name="kind"), + CheckConstraint("country ~ '^[a-z]{2}$'", name="country"), + CheckConstraint(f"year BETWEEN {FIRST_YEAR} AND {LAST_YEAR}", name="year"), + CheckConstraint("jsonb_typeof(options) = 'object'", name="options"), + CheckConstraint( + "kind = 'household' OR reform_simulation_id IS NOT NULL", + name="economy_has_reform", + ), + CheckConstraint( + "legacy_report_id IS NULL OR legacy_report_id > 0", + name="legacy_report_id", + ), + ForeignKeyConstraint( + ["baseline_simulation_id", "country", "kind"], + _SIMULATION_KEY, + name="fk_reports_baseline_simulation", + ondelete="RESTRICT", + ), + ForeignKeyConstraint( + ["reform_simulation_id", "country", "kind"], + _SIMULATION_KEY, + name="fk_reports_reform_simulation", + ondelete="RESTRICT", + ), +) +Index("ix_reports_baseline_simulation_id", reports.c.baseline_simulation_id) +Index("ix_reports_reform_simulation_id", reports.c.reform_simulation_id) + + +@dataclass(frozen=True) +class NewReport: + """A report to create. + + The database checks that both simulations exist with the report's country + and kind, and that an economy report has a reform; the fields are checked + here first. + """ + + kind: ReportKind + country: str + baseline_simulation_id: UUID + year: int + reform_simulation_id: UUID | None = None + options: dict[str, Any] = dataclass_field(default_factory=dict) + legacy_report_id: int | None = None + + def __post_init__(self) -> None: + """Reject malformed fields without touching the database.""" + # A frozen dataclass sets its own field through object.__setattr__. + object.__setattr__(self, "kind", parse_kind(self.kind, ReportKind)) + check_country(self.country) + if not FIRST_YEAR <= self.year <= LAST_YEAR: + msg = f"year must be between {FIRST_YEAR} and {LAST_YEAR}" + raise ValueError(msg) + if not isinstance(self.options, dict): + msg = "options must be a JSON object" + raise TypeError(msg) + check_legacy_id("legacy_report_id", self.legacy_report_id) + + +@dataclass(frozen=True) +class ReportRecord: + """One ``simulation.reports`` row.""" + + id: UUID + kind: ReportKind + country: str + baseline_simulation_id: UUID + reform_simulation_id: UUID | None + year: int + options: dict[str, Any] + legacy_report_id: int | None + created_at: datetime + + @classmethod + def from_row(cls, row: RowMapping) -> ReportRecord: + """Build the record from a selected ``reports`` row.""" + values = {field.name: row[field.name] for field in fields(cls)} + values["kind"] = ReportKind(values["kind"]) + return cls(**values) + + +def _new_report_values(new_report: NewReport) -> dict[str, Any]: + return { + "id": uuid4(), + **{field.name: getattr(new_report, field.name) for field in fields(new_report)}, + "kind": new_report.kind.value, + } + + +class ReportStore: + """Typed SQLAlchemy Core reads and inserts for report rows.""" + + def __init__(self, engine: Engine) -> None: + """Use ``engine``, which connects as the runtime role.""" + self.engine = engine + + def create(self, new_report: NewReport) -> ReportRecord: + """Insert one report and return it.""" + with self.engine.begin() as connection: + row = ( + connection.execute( + postgres_insert(reports) + .values(**_new_report_values(new_report)) + .returning(*reports.c) + ) + .mappings() + .one() + ) + return ReportRecord.from_row(row) + + def create_migrated(self, new_report: NewReport) -> ReportRecord: + """Create a report for a v1 report, or return the existing one. + + The v1 report ID must be set. An existing row is returned as it is, + even if ``new_report`` describes it differently, so running the + one-time migration again creates nothing new. The report has no run + until one is added. + """ + if new_report.legacy_report_id is None: + msg = "a migrated report needs its legacy_report_id" + raise ValueError(msg) + with self.engine.begin() as connection: + row = insert_or_get_existing( + connection, + reports, + _new_report_values(new_report), + reports.c.legacy_report_id, + ) + return ReportRecord.from_row(row) + + def get(self, report_id: UUID) -> ReportRecord | None: + """Return the report with this ID, or None.""" + return self._one(reports.c.id == report_id) + + def get_by_legacy_report_id(self, legacy_report_id: int) -> ReportRecord | None: + """Return the report migrated from this v1 report, or None.""" + return self._one(reports.c.legacy_report_id == legacy_report_id) + + def _one(self, condition: ColumnElement[bool]) -> ReportRecord | None: + with self.engine.connect() as connection: + row = ( + connection.execute(select(reports).where(condition)).mappings().first() + ) + return None if row is None else ReportRecord.from_row(row) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/schema.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/schema.py new file mode 100644 index 000000000..7e4080638 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/schema.py @@ -0,0 +1,44 @@ +"""Names and metadata of the Simulation Entrypoint's own Postgres schema. + +The Simulation Entrypoint owns every table in the ``simulation`` schema. One +Alembic chain, at the repository's ``migrations/`` directory, creates and +changes them, and records its revision in ``simulation.alembic_version``. +""" + +from __future__ import annotations + +from sqlalchemy import Column, MetaData, String, Table + +SIMULATION_SCHEMA = "simulation" +ALEMBIC_VERSION_TABLE = "alembic_version" +# The login role the Simulation Entrypoint runtime connects as. Migrations grant +# it schema usage and table DML; it never owns or changes schema objects. +RUNTIME_ROLE = "simulation_runtime" +# The newest revision in migrations/versions. Each migration PR updates it, and +# tests/test_schema.py fails when it differs from the chain's head. The +# readiness check reports it next to the database's revision. +HEAD_REVISION = "0002" + +NAMING_CONVENTION = { + "ix": "ix_%(table_name)s_%(column_0_N_name)s", + "uq": "uq_%(table_name)s_%(column_0_N_name)s", + "ck": "ck_%(table_name)s_%(constraint_name)s", + "fk": "fk_%(table_name)s_%(column_0_name)s_%(referred_table_name)s", + "pk": "pk_%(table_name)s", +} + +# Every table the Simulation Entrypoint owns is registered here. +simulation_metadata = MetaData( + schema=SIMULATION_SCHEMA, + naming_convention=NAMING_CONVENTION, +) + +# Alembic creates and owns this table; it is described here, outside +# simulation_metadata, only so the runtime can read the applied revision +# without depending on Alembic. +_alembic_metadata = MetaData(schema=SIMULATION_SCHEMA) +alembic_version = Table( + ALEMBIC_VERSION_TABLE, + _alembic_metadata, + Column("version_num", String(32), primary_key=True), +) diff --git a/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/simulations.py b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/simulations.py new file mode 100644 index 000000000..0ad0e7629 --- /dev/null +++ b/libs/policyengine-simulation-persistence/src/policyengine_simulation_persistence/simulations.py @@ -0,0 +1,314 @@ +"""``simulation.simulations``: what to run, independent of model versions. + +A simulation is one policy applied to one population: a household, or an +economy named by a region code (``state/ca``) and an optional dataset name. It +is a definition, not an execution, and never changes once created. A null +policy means current law, and a null dataset name means the region's default +dataset in whichever catalog version a run uses, so the same simulation can +run again on a newer catalog version. + +Foreign keys into the Public API's tables (``public_api``) tie the policy and +the household to the simulation's country, and name an optional dynamic. The +region and dataset are not foreign keys here, because their rows belong to one +catalog version; each run resolves them in its own. + +A simulation migrated from the Public API's v1 database keeps its v1 +simulation ID in ``legacy_simulation_id``. That database is separate, so the v1 +ID is a plain unique column, not a foreign key. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, fields +from enum import StrEnum +from typing import TYPE_CHECKING, Any +from uuid import UUID, uuid4 + +from sqlalchemy import ( + BigInteger, + CheckConstraint, + Column, + DateTime, + ForeignKeyConstraint, + Index, + String, + Table, + Text, + UniqueConstraint, + Uuid, + func, + select, +) +from sqlalchemy.dialects.postgresql import insert as postgres_insert + +from policyengine_simulation_persistence import public_api +from policyengine_simulation_persistence.schema import simulation_metadata + +if TYPE_CHECKING: + from datetime import datetime + + from sqlalchemy import ColumnElement, Connection, Engine, RowMapping + +COUNTRY_PATTERN = re.compile(r"[a-z]{2}") +# Region codes and dataset names are the Public API's catalog keys: up to 255 +# characters, with no whitespace. +CATALOG_KEY_PATTERN = re.compile(r"\S{1,255}") +CATALOG_KEY_LENGTH = 255 + + +class SimulationKind(StrEnum): + """What a simulation runs on: one household or a whole economy.""" + + ECONOMY = "economy" + HOUSEHOLD = "household" + + +_KIND_VALUES = ", ".join(f"'{kind.value}'" for kind in SimulationKind) + +simulations = Table( + "simulations", + simulation_metadata, + Column("id", Uuid(as_uuid=True), primary_key=True), + Column("kind", Text, nullable=False), + Column("country", Text, nullable=False), + Column("policy_id", Uuid(as_uuid=True), nullable=True), + Column("dynamic_id", Uuid(as_uuid=True), nullable=True), + Column("household_id", Uuid(as_uuid=True), nullable=True), + Column("region_code", String(CATALOG_KEY_LENGTH), nullable=True), + Column("dataset_name", String(CATALOG_KEY_LENGTH), nullable=True), + Column("legacy_simulation_id", BigInteger, nullable=True, unique=True), + Column( + "created_at", + DateTime(timezone=True), + nullable=False, + server_default=func.now(), + ), + CheckConstraint(f"kind IN ({_KIND_VALUES})", name="kind"), + CheckConstraint("country ~ '^[a-z]{2}$'", name="country"), + CheckConstraint( + "(kind = 'household' AND household_id IS NOT NULL " + "AND region_code IS NULL AND dataset_name IS NULL) " + "OR (kind = 'economy' AND household_id IS NULL " + "AND region_code IS NOT NULL)", + name="population_matches_kind", + ), + CheckConstraint( + "region_code IS NULL OR region_code ~ '^\\S+$'", name="region_code" + ), + CheckConstraint( + "dataset_name IS NULL OR dataset_name ~ '^\\S+$'", name="dataset_name" + ), + CheckConstraint( + "legacy_simulation_id IS NULL OR legacy_simulation_id > 0", + name="legacy_simulation_id", + ), + # Reports refer to a simulation together with its country and kind, so a + # report can only compare simulations of its own country and kind. + UniqueConstraint("id", "country", "kind"), + ForeignKeyConstraint( + ["policy_id", "country"], + [public_api.policies.c.id, public_api.policies.c.country_id], + name="fk_simulations_policy_country", + ondelete="RESTRICT", + ), + ForeignKeyConstraint( + ["dynamic_id"], + [public_api.dynamics.c.id], + name="fk_simulations_dynamic", + ondelete="RESTRICT", + ), + ForeignKeyConstraint( + ["household_id", "country"], + [public_api.households.c.id, public_api.households.c.country_id], + name="fk_simulations_household_country", + ondelete="RESTRICT", + ), +) +# A Public API delete of a policy, household or dynamic checks these for +# references. +Index("ix_simulations_policy_id", simulations.c.policy_id) +Index("ix_simulations_household_id", simulations.c.household_id) +Index("ix_simulations_dynamic_id", simulations.c.dynamic_id) + + +def _check_catalog_key(field_name: str, value: str | None) -> None: + if value is not None and not CATALOG_KEY_PATTERN.fullmatch(value): + msg = f"{field_name} must be 1 to 255 characters with no whitespace" + raise ValueError(msg) + + +def check_legacy_id(field_name: str, value: int | None) -> None: + """Reject a v1 ID that is not a positive integer.""" + if value is not None and value <= 0: + msg = f"{field_name} must be a positive v1 ID" + raise ValueError(msg) + + +def parse_kind[KindT: StrEnum](value: object, kind_type: type[KindT]) -> KindT: + """Return the ``kind_type`` member for ``value``, which may be its text. + + Raise ValueError naming ``kind`` when the value is not one of its members. + """ + try: + return kind_type(value) + except ValueError: + msg = f"kind must be one of {', '.join(kind_type)}" + raise ValueError(msg) from None + + +def check_country(country: str) -> None: + """Reject a country that is not a lowercase two-letter code.""" + if not COUNTRY_PATTERN.fullmatch(country): + msg = "country must be a lowercase two-letter code" + raise ValueError(msg) + + +def insert_or_get_existing( + connection: Connection, + table: Table, + values: dict[str, Any], + unique_column: Column[Any], +) -> RowMapping: + """Insert a row, or return the row that already has its unique value.""" + created = ( + connection.execute( + postgres_insert(table) + .values(**values) + .on_conflict_do_nothing(index_elements=[unique_column]) + .returning(*table.c) + ) + .mappings() + .first() + ) + if created is not None: + return created + return ( + connection.execute( + select(table).where(unique_column == values[unique_column.name]) + ) + .mappings() + .one() + ) + + +@dataclass(frozen=True) +class NewSimulation: + """A simulation to create. + + The database checks that the population matches the kind and that the + policy, household and dynamic exist for the country; the fields are + checked here first. + """ + + kind: SimulationKind + country: str + policy_id: UUID | None = None + dynamic_id: UUID | None = None + household_id: UUID | None = None + region_code: str | None = None + dataset_name: str | None = None + legacy_simulation_id: int | None = None + + def __post_init__(self) -> None: + """Reject malformed fields without touching the database.""" + # A frozen dataclass sets its own field through object.__setattr__. + object.__setattr__(self, "kind", parse_kind(self.kind, SimulationKind)) + check_country(self.country) + _check_catalog_key("region_code", self.region_code) + _check_catalog_key("dataset_name", self.dataset_name) + check_legacy_id("legacy_simulation_id", self.legacy_simulation_id) + + +@dataclass(frozen=True) +class SimulationRecord: + """One ``simulation.simulations`` row.""" + + id: UUID + kind: SimulationKind + country: str + policy_id: UUID | None + dynamic_id: UUID | None + household_id: UUID | None + region_code: str | None + dataset_name: str | None + legacy_simulation_id: int | None + created_at: datetime + + @classmethod + def from_row(cls, row: RowMapping) -> SimulationRecord: + """Build the record from a selected ``simulations`` row.""" + values = {field.name: row[field.name] for field in fields(cls)} + values["kind"] = SimulationKind(values["kind"]) + return cls(**values) + + +def _new_simulation_values(new_simulation: NewSimulation) -> dict[str, Any]: + return { + "id": uuid4(), + **{ + field.name: getattr(new_simulation, field.name) + for field in fields(new_simulation) + }, + "kind": new_simulation.kind.value, + } + + +class SimulationStore: + """Typed SQLAlchemy Core reads and inserts for simulation rows.""" + + def __init__(self, engine: Engine) -> None: + """Use ``engine``, which connects as the runtime role.""" + self.engine = engine + + def create(self, new_simulation: NewSimulation) -> SimulationRecord: + """Insert one simulation and return it.""" + with self.engine.begin() as connection: + row = ( + connection.execute( + postgres_insert(simulations) + .values(**_new_simulation_values(new_simulation)) + .returning(*simulations.c) + ) + .mappings() + .one() + ) + return SimulationRecord.from_row(row) + + def create_migrated(self, new_simulation: NewSimulation) -> SimulationRecord: + """Create a simulation for a v1 simulation, or return the existing one. + + The v1 simulation ID must be set. An existing row is returned as it is, + even if ``new_simulation`` describes it differently, so running the + one-time migration again creates nothing new. + """ + if new_simulation.legacy_simulation_id is None: + msg = "a migrated simulation needs its legacy_simulation_id" + raise ValueError(msg) + with self.engine.begin() as connection: + row = insert_or_get_existing( + connection, + simulations, + _new_simulation_values(new_simulation), + simulations.c.legacy_simulation_id, + ) + return SimulationRecord.from_row(row) + + def get(self, simulation_id: UUID) -> SimulationRecord | None: + """Return the simulation with this ID, or None.""" + return self._one(simulations.c.id == simulation_id) + + def get_by_legacy_simulation_id( + self, legacy_simulation_id: int + ) -> SimulationRecord | None: + """Return the simulation migrated from this v1 simulation, or None.""" + return self._one(simulations.c.legacy_simulation_id == legacy_simulation_id) + + def _one(self, condition: ColumnElement[bool]) -> SimulationRecord | None: + with self.engine.connect() as connection: + row = ( + connection.execute(select(simulations).where(condition)) + .mappings() + .first() + ) + return None if row is None else SimulationRecord.from_row(row) diff --git a/libs/policyengine-simulation-persistence/tests/conftest.py b/libs/policyengine-simulation-persistence/tests/conftest.py new file mode 100644 index 000000000..12a4821b6 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/conftest.py @@ -0,0 +1,152 @@ +"""Disposable PostgreSQL databases for the migration and store tests. + +Tests marked ``postgres`` need a PostgreSQL server whose superuser URL is in +``SIMULATION_TEST_DATABASE_URL``. Each test gets a new database owned by a +non-superuser migrator role, so migrations run with the privileges they have in +production. Without the variable those tests skip, unless +``SIMULATION_REQUIRE_DATABASE_TESTS=1`` (set in CI), where they fail instead. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from typing import TYPE_CHECKING +from uuid import uuid4 + +import pytest +from alembic import command +from migration_support import ( + MIGRATION_DATABASE_URL_VARIABLE, + MIGRATOR_ROLE, + TEST_ROLE_PASSWORD, + alembic_config, +) +from policyengine_simulation_persistence.schema import RUNTIME_ROLE +from public_api_support import ( + PublicApiRows, + create_public_api_tables, + insert_public_api_rows, +) +from sqlalchemy import create_engine, text +from sqlalchemy.engine import URL, make_url + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy import Connection + +DATABASE_URL_VARIABLE = "SIMULATION_TEST_DATABASE_URL" +REQUIRE_DATABASE_VARIABLE = "SIMULATION_REQUIRE_DATABASE_TESTS" + + +@dataclass(frozen=True) +class DisposableDatabase: + """One empty database and the URLs of the roles that use it.""" + + name: str + migrator_url: str + runtime_url: str + superuser_url: str + + +def _canonical(url: URL) -> str: + return url.set(drivername="postgresql").render_as_string(hide_password=False) + + +def _ensure_login_role(connection: Connection, role_name: str) -> None: + connection.execute( + text( + "DO $$ BEGIN " + f"IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = '{role_name}') " + f"THEN CREATE ROLE {role_name} LOGIN PASSWORD '{TEST_ROLE_PASSWORD}'; " + "END IF; END $$" + ) + ) + + +@pytest.fixture(scope="session") +def postgres_server_url() -> URL: + """Return the superuser URL of the test server, or skip without one.""" + configured = os.environ.get(DATABASE_URL_VARIABLE, "") + if not configured: + message = f"{DATABASE_URL_VARIABLE} names no PostgreSQL server" + if os.environ.get(REQUIRE_DATABASE_VARIABLE) == "1": + pytest.fail(message) + pytest.skip(message) + return make_url(configured).set(drivername="postgresql+psycopg") + + +@pytest.fixture +def references_granted() -> bool: + """Whether the migrator gets REFERENCES on the Public API tables. + + The operator grants it in production; a test overrides this fixture to + leave it out. + """ + return True + + +@pytest.fixture +def disposable_database( + postgres_server_url: URL, + monkeypatch: pytest.MonkeyPatch, + references_granted, +) -> Iterator[DisposableDatabase]: + """Create a database owned by the migrator, and drop it afterwards. + + It holds only the Public API tables the simulation schema refers to + (``public_api_support``). ``SIMULATION_MIGRATION_DATABASE_URL`` points at + it, as in the deploy job. + """ + database_name = f"simulation_test_{uuid4().hex[:12]}" + server_engine = create_engine(postgres_server_url, isolation_level="AUTOCOMMIT") + try: + with server_engine.connect() as connection: + _ensure_login_role(connection, MIGRATOR_ROLE) + _ensure_login_role(connection, RUNTIME_ROLE) + connection.execute( + text(f'CREATE DATABASE "{database_name}" OWNER {MIGRATOR_ROLE}') + ) + database = DisposableDatabase( + name=database_name, + migrator_url=_canonical( + postgres_server_url.set( + username=MIGRATOR_ROLE, + password=TEST_ROLE_PASSWORD, + database=database_name, + ) + ), + runtime_url=_canonical( + postgres_server_url.set( + username=RUNTIME_ROLE, + password=TEST_ROLE_PASSWORD, + database=database_name, + ) + ), + superuser_url=_canonical(postgres_server_url.set(database=database_name)), + ) + create_public_api_tables( + database.superuser_url, grant_references=references_granted + ) + monkeypatch.setenv(MIGRATION_DATABASE_URL_VARIABLE, database.migrator_url) + yield database + finally: + with server_engine.connect() as connection: + connection.execute( + text(f'DROP DATABASE IF EXISTS "{database_name}" WITH (FORCE)') + ) + server_engine.dispose() + + +@pytest.fixture +def migrated_database(disposable_database: DisposableDatabase) -> DisposableDatabase: + """Return a disposable database with the whole chain applied.""" + command.upgrade(alembic_config(), "head") + return disposable_database + + +@pytest.fixture +def public_api_rows(migrated_database: DisposableDatabase) -> PublicApiRows: + """Insert one row of each kind the simulation tables refer to.""" + return insert_public_api_rows(migrated_database.superuser_url) diff --git a/libs/policyengine-simulation-persistence/tests/migration_provisioning_support.py b/libs/policyengine-simulation-persistence/tests/migration_provisioning_support.py new file mode 100644 index 000000000..195c93a6a --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/migration_provisioning_support.py @@ -0,0 +1,125 @@ +"""Builders for the migration provisioning tests: exported v1 report definitions.""" + +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any +from uuid import UUID + +from policyengine_simulation_persistence.migration_provisioning import ( + ReportDefinitionsFile, +) + +if TYPE_CHECKING: + from public_api_support import PublicApiRows + +US_REFORM_POLICY_ID = UUID("00000000-0000-4000-8000-000000000001") +US_HOUSEHOLD_ID = UUID("00000000-0000-4000-8000-000000000002") + + +def economy_simulation_definition( + legacy_simulation_id: int, **overrides: object +) -> dict[str, Any]: + """Return a current-law California economy simulation, as exported.""" + definition: dict[str, Any] = { + "legacy_simulation_id": legacy_simulation_id, + "policy_id": None, + "region_code": "state/ca", + "dataset_name": None, + } + definition.update(overrides) + return definition + + +def household_simulation_definition( + legacy_simulation_id: int, **overrides: object +) -> dict[str, Any]: + """Return a current-law simulation of the US household, as exported.""" + definition: dict[str, Any] = { + "legacy_simulation_id": legacy_simulation_id, + "policy_id": None, + "household_id": str(US_HOUSEHOLD_ID), + } + definition.update(overrides) + return definition + + +def economy_report_definition( + legacy_report_id: int = 4512, **overrides: object +) -> dict[str, Any]: + """Return a 2026 US economy report of a reform in California, as exported.""" + definition: dict[str, Any] = { + "kind": "economy", + "legacy_report_id": legacy_report_id, + "country": "us", + "year": 2026, + "options": {}, + "baseline": economy_simulation_definition(9001), + "reform": economy_simulation_definition( + 9002, policy_id=str(US_REFORM_POLICY_ID) + ), + } + definition.update(overrides) + return definition + + +def household_report_definition( + legacy_report_id: int = 4513, **overrides: object +) -> dict[str, Any]: + """Return a 2025 report of the US household under current law, as exported.""" + definition: dict[str, Any] = { + "kind": "household", + "legacy_report_id": legacy_report_id, + "country": "us", + "year": 2025, + "options": {}, + "baseline": household_simulation_definition(9003), + "reform": None, + } + definition.update(overrides) + return definition + + +def economy_report_for_rows( + rows: PublicApiRows, legacy_report_id: int = 4512, **overrides: object +) -> dict[str, Any]: + """Return an economy report of the test database's US policy in California.""" + values: dict[str, object] = { + "reform": economy_simulation_definition(9002, policy_id=str(rows.us_policy_id)) + } + values.update(overrides) + return economy_report_definition(legacy_report_id, **values) + + +def household_report_for_rows( + rows: PublicApiRows, legacy_report_id: int = 4513, **overrides: object +) -> dict[str, Any]: + """Return a report comparing the test database's US policy with current law. + + Both simulations use the test database's US household. + """ + household_id = str(rows.us_household_id) + values: dict[str, object] = { + "baseline": household_simulation_definition(9003, household_id=household_id), + "reform": household_simulation_definition( + 9004, household_id=household_id, policy_id=str(rows.us_policy_id) + ), + } + values.update(overrides) + return household_report_definition(legacy_report_id, **values) + + +def definitions_json( + *report_definitions: dict[str, Any], schema_version: int = 1 +) -> str: + """Return an export file holding ``report_definitions``.""" + return json.dumps( + {"schema_version": schema_version, "reports": list(report_definitions)} + ) + + +def parse_definitions(*report_definitions: dict[str, Any]) -> ReportDefinitionsFile: + """Parse an export file holding ``report_definitions``.""" + return ReportDefinitionsFile.model_validate_json( + definitions_json(*report_definitions) + ) diff --git a/libs/policyengine-simulation-persistence/tests/migration_support.py b/libs/policyengine-simulation-persistence/tests/migration_support.py new file mode 100644 index 000000000..8efb3d13e --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/migration_support.py @@ -0,0 +1,75 @@ +"""Paths and names shared by the migration and store tests.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +from alembic import command +from alembic.config import Config +from policyengine_simulation_persistence import ( + create_simulation_engine, + read_schema_revision, +) + +if TYPE_CHECKING: + from collections.abc import Callable + +REPOSITORY_ROOT = Path(__file__).resolve().parents[3] +ALEMBIC_CONFIG_PATH = REPOSITORY_ROOT / "migrations" / "alembic.ini" +MIGRATION_DATABASE_URL_VARIABLE = "SIMULATION_MIGRATION_DATABASE_URL" +MIGRATOR_ROLE = "simulation_migrator" +TEST_ROLE_PASSWORD = "simulation-test-password" + + +def alembic_config() -> Config: + """Return the repository's Alembic configuration.""" + return Config(str(ALEMBIC_CONFIG_PATH)) + + +def chain_config(script_location: Path) -> Config: + """Return a configuration for the chain at ``script_location``.""" + config = Config() + config.set_main_option("script_location", str(script_location)) + return config + + +def _step_failure(name: str, step: Callable[[], object]) -> str | None: + try: + step() + except Exception as error: # noqa: BLE001 # reported as the cycle's failure + detail = str(error).strip().splitlines() + return f"{name} failed: {type(error).__name__}" + ( + f": {detail[0]}" if detail else "" + ) + return None + + +def migration_cycle_failures(config: Config, *, check: bool) -> list[str]: + """Upgrade, downgrade and upgrade the chain again; return what failed. + + With ``check``, ``alembic check`` must also find nothing to change after + each upgrade. The cycle stops at the first failure. + """ + steps: list[tuple[str, Callable[[], object]]] = [ + ("upgrade", lambda: command.upgrade(config, "head")) + ] + if check: + steps.append(("check", lambda: command.check(config))) + steps.append(("downgrade", lambda: command.downgrade(config, "base"))) + steps.extend(steps[: 2 if check else 1]) + for name, step in steps: + failure = _step_failure(name, step) + if failure is not None: + return [failure] + return [] + + +def applied_revision(database_url: str) -> str | None: + """Return the revision the database at ``database_url`` has applied.""" + engine = create_simulation_engine(database_url) + try: + with engine.connect() as connection: + return read_schema_revision(connection) + finally: + engine.dispose() diff --git a/libs/policyengine-simulation-persistence/tests/ownership_checks.py b/libs/policyengine-simulation-persistence/tests/ownership_checks.py new file mode 100644 index 000000000..6ae57384a --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/ownership_checks.py @@ -0,0 +1,273 @@ +"""Checks behind the simulation schema ownership rules in rules/registry.toml. + +Each check takes the files it inspects, so the rule tests run it on the +repository and the proof tests run it on the fixtures under +``rules/fixtures//``. Runtime code that changes a schema is caught by +sim.boundaries.sqlalchemy-and-alembic-only, not here. +""" + +from __future__ import annotations + +import ast +import configparser +import re +from pathlib import Path +from typing import TYPE_CHECKING + +import tomllib +import yaml + +if TYPE_CHECKING: + from collections.abc import Iterator, Mapping + +CHAIN_DIRECTORY = Path("migrations") +CHAIN_CONFIG = CHAIN_DIRECTORY / "alembic.ini" +CHAIN_REVISIONS = CHAIN_DIRECTORY / "versions" +SIMULATION_SCHEMA = "simulation" +IGNORED_DIRECTORY_NAMES = frozenset( + {".git", ".venv", "node_modules", "__pycache__", ".pytest_cache", ".ruff_cache"} +) +# Proof fixtures deliberately contain bad layouts. +IGNORED_RELATIVE_DIRECTORIES = (Path("rules") / "fixtures",) +PERSISTENCE_DISTRIBUTION = "policyengine-simulation-persistence" +PERSISTENCE_PACKAGE = "policyengine_simulation_persistence" +# Any variable named like a database URL (DATABASE_URL itself included), or a +# PostgreSQL URL written out. +DATABASE_CREDENTIAL = re.compile( + r"\b[A-Z0-9_]*DATABASE_URL[A-Z0-9_]*\b|\bpostgres(?:ql)?(?:\+\w+)?://" +) +# The comparison path's own database URL, and the name of the Secret Manager +# secret that holds it, may reach Modal until the comparison path is removed. +COMPARISON_PATH_CREDENTIALS = frozenset( + {"STAGE12_DATABASE_URL", "STAGE12_DATABASE_URL_SECRET_NAME"} +) +MODAL_SECRET_COMMAND = re.compile(r"\bmodal\s+secret\s+create\b") +# Deploying an application or running a temporary one both start Modal +# containers that a step's values may reach. +MODAL_DEPLOY_COMMAND = re.compile(r"\bmodal\s+(?:deploy|run)\b") + + +def _repository_files(root: Path, pattern: str) -> Iterator[Path]: + for path in sorted(root.rglob(pattern)): + relative = path.relative_to(root) + if IGNORED_DIRECTORY_NAMES.intersection(relative.parts): + continue + if any( + relative.is_relative_to(ignored) for ignored in IGNORED_RELATIVE_DIRECTORIES + ): + continue + yield path + + +def _configures_simulation_version_table(environment_source: str) -> bool: + for node in ast.walk(ast.parse(environment_source)): + if not ( + isinstance(node, ast.Call) + and isinstance(node.func, ast.Attribute) + and node.func.attr == "configure" + ): + continue + for keyword in node.keywords: + if keyword.arg != "version_table_schema": + continue + value = keyword.value + if isinstance(value, ast.Name) and value.id == "SIMULATION_SCHEMA": + return True + if isinstance(value, ast.Constant) and value.value == SIMULATION_SCHEMA: + return True + return False + + +def _layout_violations(root: Path) -> list[str]: + violations: list[str] = [] + configurations = [ + path.relative_to(root) for path in _repository_files(root, "alembic*.ini") + ] + if configurations != [CHAIN_CONFIG]: + violations.append( + f"expected only {CHAIN_CONFIG} as an Alembic configuration, found " + f"{[str(path) for path in configurations]}" + ) + for versions in _repository_files(root, "versions"): + relative = versions.relative_to(root) + if ( + versions.is_dir() + and relative != CHAIN_REVISIONS + and any(versions.glob("*.py")) + ): + violations.append(f"revision directory outside the chain: {relative}") + for environment in _repository_files(root, "env.py"): + relative = environment.relative_to(root) + if relative != CHAIN_DIRECTORY / "env.py" and "alembic" in ( + environment.read_text(encoding="utf-8") + ): + violations.append(f"Alembic environment outside the chain: {relative}") + return violations + + +def migration_chain_violations(root: Path) -> list[str]: + """Return how ``root`` departs from one simulation chain at ``migrations/``.""" + root = root.resolve() + violations = _layout_violations(root) + config_path = root / CHAIN_CONFIG + if not config_path.is_file(): + return violations + parser = configparser.ConfigParser(interpolation=None) + parser.read(config_path, encoding="utf-8") + script_location = parser.get("alembic", "script_location", fallback="") + resolved = Path(script_location.replace("%(here)s", str(config_path.parent))) + if not resolved.is_absolute(): + resolved = root / resolved + if resolved.resolve() != (root / CHAIN_DIRECTORY).resolve(): + violations.append(f"script_location {script_location!r} is not migrations/") + environment_path = root / CHAIN_DIRECTORY / "env.py" + if not environment_path.is_file() or not _configures_simulation_version_table( + environment_path.read_text(encoding="utf-8") + ): + violations.append( + "migrations/env.py must keep its version table in the simulation schema" + ) + return violations + + +def declares_persistence_dependency(pyproject_text: str) -> bool: + """Return whether a pyproject declares the persistence library anywhere.""" + project = tomllib.loads(pyproject_text).get("project", {}) + requirements = [ + *project.get("dependencies", []), + *( + requirement + for group in project.get("optional-dependencies", {}).values() + for requirement in group + ), + ] + return any( + re.split(r"[\s\[<>=!~;]", requirement, maxsplit=1)[0] + == PERSISTENCE_DISTRIBUTION + for requirement in requirements + ) + + +def persistence_dependency_violations(project_root: Path) -> list[str]: + """Return where a project depends on or imports the persistence library.""" + violations: list[str] = [] + pyproject = project_root / "pyproject.toml" + if declares_persistence_dependency(pyproject.read_text(encoding="utf-8")): + violations.append(f"{pyproject} declares {PERSISTENCE_DISTRIBUTION}") + lock = project_root / "uv.lock" + if lock.is_file() and f'name = "{PERSISTENCE_DISTRIBUTION}"' in lock.read_text( + encoding="utf-8" + ): + violations.append(f"{lock} locks {PERSISTENCE_DISTRIBUTION}") + violations.extend( + f"{path} imports {PERSISTENCE_PACKAGE}" + for path in sorted((project_root / "src").rglob("*.py")) + if PERSISTENCE_PACKAGE in path.read_text(encoding="utf-8") + ) + return violations + + +def _foreign_credentials(text: str) -> list[str]: + """Return the database credentials in ``text`` other than the comparison path's.""" + return sorted( + { + match + for match in DATABASE_CREDENTIAL.findall(text) + if match not in COMPARISON_PATH_CREDENTIALS + } + ) + + +def _environment_text(environment: object) -> str: + if not isinstance(environment, dict): + return "" + return "\n".join(f"{key}={value}" for key, value in environment.items()) + + +def _step_text(step: Mapping[str, object]) -> str: + return "\n".join( + str(value) + for value in (step.get("run", ""), step.get("uses", ""), step.get("with", "")) + ) + + +def _step_reaches_modal(step_text: str, modal_scripts: set[str]) -> bool: + return bool( + MODAL_SECRET_COMMAND.search(step_text) + or MODAL_DEPLOY_COMMAND.search(step_text) + or any(name in step_text for name in modal_scripts) + ) + + +def _workflow_violations( + workflow_name: str, workflow_text: str, modal_scripts: set[str] +) -> list[str]: + workflow = yaml.safe_load(workflow_text) or {} + workflow_environment = _environment_text(workflow.get("env")) + violations: list[str] = [] + for job_name, job in (workflow.get("jobs") or {}).items(): + job_environment = _environment_text(job.get("env")) + for step in job.get("steps") or []: + step_text = _step_text(step) + if not _step_reaches_modal(step_text, modal_scripts): + continue + # Workflow-level and job-level env reach every step of the job. + carried = _foreign_credentials( + "\n".join( + ( + workflow_environment, + job_environment, + _environment_text(step.get("env")), + step_text, + ) + ) + ) + if carried: + violations.append( + f"{workflow_name} job {job_name} step " + f"{step.get('name', '')!r} gives Modal a database " + f"credential other than the comparison path's: " + f"{', '.join(carried)}" + ) + return violations + + +def modal_credential_violations( + workflow_texts: Mapping[str, str], + script_texts: Mapping[str, str], +) -> list[str]: + """Return Modal secret or deploy steps that carry a foreign database URL. + + ``workflow_texts`` and ``script_texts`` map a file name to its contents. A + step counts as giving Modal a value when it creates a Modal secret, deploys + a Modal application or runs a temporary one, directly or through a script + that does, and + every value in its step, job or workflow environment reaches it. Only the + comparison path's own database URL may reach Modal. + """ + modal_scripts = { + name + for name, text in script_texts.items() + if MODAL_SECRET_COMMAND.search(text) or MODAL_DEPLOY_COMMAND.search(text) + } + violations = [ + f"{name} gives Modal a database credential other than the comparison " + f"path's: {', '.join(carried)}" + for name in sorted(modal_scripts) + if (carried := _foreign_credentials(script_texts[name])) + ] + for workflow_name, workflow_text in sorted(workflow_texts.items()): + violations.extend( + _workflow_violations(workflow_name, workflow_text, modal_scripts) + ) + return violations + + +def modal_source_violations(sources: Mapping[str, str]) -> list[str]: + """Return Modal project sources that name a foreign database URL variable.""" + return [ + f"{name} names {variable}" + for name, source in sorted(sources.items()) + for variable in _foreign_credentials(source) + if not variable.endswith("://") + ] diff --git a/libs/policyengine-simulation-persistence/tests/public_api_support.py b/libs/policyengine-simulation-persistence/tests/public_api_support.py new file mode 100644 index 000000000..cf144c68d --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/public_api_support.py @@ -0,0 +1,126 @@ +"""The Public API tables the simulation schema refers to, reduced to their keys. + +In production these tables belong to the Public API's own migration chain, in +the same database as the simulation schema. The tests create only the columns +and unique constraints the simulation schema's foreign keys use, with the +Public API's constraint names, owned by the superuser, and grant the migrator +``REFERENCES`` on them, as the operator does in production. The runtime role +gets no privilege on them. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING +from uuid import UUID, uuid4 + +from migration_support import MIGRATOR_ROLE +from sqlalchemy import create_engine, text + +if TYPE_CHECKING: + from sqlalchemy import Engine + +PUBLIC_API_TABLE_STATEMENTS = ( + "CREATE TABLE public.policies (" + " id uuid PRIMARY KEY," + " country_id varchar(2) NOT NULL," + " CONSTRAINT uq_policies_id_country UNIQUE (id, country_id))", + "CREATE TABLE public.households (" + " id uuid PRIMARY KEY," + " country_id varchar(2) NOT NULL," + " CONSTRAINT uq_households_id_country UNIQUE (id, country_id))", + "CREATE TABLE public.dynamics (id uuid PRIMARY KEY)", +) +REFERENCED_TABLES = ("policies", "households", "dynamics") + + +def superuser_engine(superuser_url: str, **options: object) -> Engine: + """Return an engine that connects to the test database as the superuser.""" + return create_engine( + superuser_url.replace("postgresql://", "postgresql+psycopg://", 1), + **options, + ) + + +def create_public_api_tables(superuser_url: str, *, grant_references: bool) -> None: + """Create the referenced Public API tables in a new test database.""" + engine = superuser_engine(superuser_url, isolation_level="AUTOCOMMIT") + grant = ( + "GRANT REFERENCES ON " + + ", ".join(f"public.{table}" for table in REFERENCED_TABLES) + + f" TO {MIGRATOR_ROLE}" + ) + statements = ( + (*PUBLIC_API_TABLE_STATEMENTS, grant) + if grant_references + else PUBLIC_API_TABLE_STATEMENTS + ) + try: + with engine.connect() as connection: + for statement in statements: + connection.execute(text(statement)) + finally: + engine.dispose() + + +@dataclass(frozen=True) +class PublicApiRows: + """Public API rows a simulation can refer to.""" + + us_policy_id: UUID + uk_policy_id: UUID + us_household_id: UUID + uk_household_id: UUID + dynamic_id: UUID + + +def insert_public_api_rows(superuser_url: str) -> PublicApiRows: + """Insert rows into the referenced Public API tables, as their owner.""" + rows = PublicApiRows( + us_policy_id=uuid4(), + uk_policy_id=uuid4(), + us_household_id=uuid4(), + uk_household_id=uuid4(), + dynamic_id=uuid4(), + ) + statements = ( + ( + "INSERT INTO public.policies (id, country_id) VALUES " + "(:us_policy, 'us'), (:uk_policy, 'uk')", + {"us_policy": rows.us_policy_id, "uk_policy": rows.uk_policy_id}, + ), + ( + "INSERT INTO public.households (id, country_id) VALUES " + "(:us_household, 'us'), (:uk_household, 'uk')", + { + "us_household": rows.us_household_id, + "uk_household": rows.uk_household_id, + }, + ), + ( + "INSERT INTO public.dynamics (id) VALUES (:dynamic)", + {"dynamic": rows.dynamic_id}, + ), + ) + engine = superuser_engine(superuser_url) + try: + with engine.begin() as connection: + for statement, parameters in statements: + connection.execute(text(statement), parameters) + finally: + engine.dispose() + return rows + + +def delete_public_api_row(superuser_url: str, table_name: str, row_id: UUID) -> None: + """Delete one Public API row as its owner, as the Public API would.""" + assert table_name in REFERENCED_TABLES + engine = superuser_engine(superuser_url) + try: + with engine.begin() as connection: + connection.execute( + text(f"DELETE FROM public.{table_name} WHERE id = :row_id"), # noqa: S608 # table names come from REFERENCED_TABLES + {"row_id": row_id}, + ) + finally: + engine.dispose() diff --git a/libs/policyengine-simulation-persistence/tests/report_support.py b/libs/policyengine-simulation-persistence/tests/report_support.py new file mode 100644 index 000000000..2ce21677e --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/report_support.py @@ -0,0 +1,56 @@ +"""Builders for the report tests.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from policyengine_simulation_persistence import ( + NewReport, + ReportKind, + ReportStore, + create_simulation_engine, +) +from simulation_support import ( + economy_simulation, + household_simulation, + simulation_store, +) + +if TYPE_CHECKING: + from policyengine_simulation_persistence import SimulationRecord + from public_api_support import PublicApiRows + + +def report_store(database) -> ReportStore: + """Return a store that connects as the runtime role.""" + return ReportStore(create_simulation_engine(database.runtime_url)) + + +def economy_pair( + database, rows: PublicApiRows +) -> tuple[SimulationRecord, SimulationRecord]: + """Create a current-law baseline and a reform of California's economy.""" + store = simulation_store(database) + baseline = store.create(economy_simulation(rows, policy_id=None)) + reform = store.create(economy_simulation(rows)) + return baseline, reform + + +def economy_report( + baseline: SimulationRecord, reform: SimulationRecord, **overrides: object +) -> NewReport: + """Return a 2026 US economy report comparing the two simulations.""" + values: dict[str, object] = { + "kind": ReportKind.ECONOMY, + "country": "us", + "baseline_simulation_id": baseline.id, + "reform_simulation_id": reform.id, + "year": 2026, + } + values.update(overrides) + return NewReport(**values) # type: ignore[arg-type] + + +def household_baseline(database, rows: PublicApiRows) -> SimulationRecord: + """Create a current-law simulation of the US household.""" + return simulation_store(database).create(household_simulation(rows)) diff --git a/libs/policyengine-simulation-persistence/tests/simulation_support.py b/libs/policyengine-simulation-persistence/tests/simulation_support.py new file mode 100644 index 000000000..030ee2e93 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/simulation_support.py @@ -0,0 +1,43 @@ +"""Builders shared by the simulation and report tests.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from policyengine_simulation_persistence import ( + NewSimulation, + SimulationKind, + SimulationStore, + create_simulation_engine, +) + +if TYPE_CHECKING: + from public_api_support import PublicApiRows + + +def economy_simulation(rows: PublicApiRows, **overrides: object) -> NewSimulation: + """Return a California economy simulation of the US reform.""" + values: dict[str, object] = { + "kind": SimulationKind.ECONOMY, + "country": "us", + "policy_id": rows.us_policy_id, + "region_code": "state/ca", + } + values.update(overrides) + return NewSimulation(**values) # type: ignore[arg-type] + + +def household_simulation(rows: PublicApiRows, **overrides: object) -> NewSimulation: + """Return a current-law simulation of the US household.""" + values: dict[str, object] = { + "kind": SimulationKind.HOUSEHOLD, + "country": "us", + "household_id": rows.us_household_id, + } + values.update(overrides) + return NewSimulation(**values) # type: ignore[arg-type] + + +def simulation_store(database) -> SimulationStore: + """Return a store that connects as the runtime role.""" + return SimulationStore(create_simulation_engine(database.runtime_url)) diff --git a/libs/policyengine-simulation-persistence/tests/test_definitions.py b/libs/policyengine-simulation-persistence/tests/test_definitions.py new file mode 100644 index 000000000..d15f35b35 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_definitions.py @@ -0,0 +1,100 @@ +"""New simulation and report definitions are checked before any write.""" + +from __future__ import annotations + +from uuid import uuid4 + +import pytest +from policyengine_simulation_persistence import ( + NewReport, + NewSimulation, + ReportKind, + SimulationKind, +) + + +@pytest.mark.parametrize( + ("field_name", "value"), + [ + ("country", "USA"), + ("country", "Us"), + ("region_code", ""), + ("region_code", " state/ca"), + ("region_code", "x" * 256), + ("dataset_name", ""), + ("dataset_name", "populace us"), + ("legacy_simulation_id", 0), + ], +) +def test_malformed_simulation_definitions_are_rejected(field_name, value) -> None: + values = { + "kind": SimulationKind.ECONOMY, + "country": "us", + "region_code": "state/ca", + field_name: value, + } + + with pytest.raises(ValueError, match=field_name): + NewSimulation(**values) + + +@pytest.mark.parametrize( + ("field_name", "value"), + [ + ("country", "gb "), + ("year", 1899), + ("year", 2201), + ("legacy_report_id", -4), + ], +) +def test_malformed_report_definitions_are_rejected(field_name, value) -> None: + values = { + "kind": ReportKind.ECONOMY, + "country": "us", + "baseline_simulation_id": uuid4(), + "reform_simulation_id": uuid4(), + "year": 2026, + field_name: value, + } + + with pytest.raises(ValueError, match=field_name): + NewReport(**values) + + +def test_report_options_must_be_a_json_object() -> None: + with pytest.raises(TypeError, match="options"): + NewReport( + kind=ReportKind.ECONOMY, + country="us", + baseline_simulation_id=uuid4(), + year=2026, + options=["not", "an", "object"], # type: ignore[arg-type] + ) + + +def test_a_kind_given_as_text_becomes_the_enum() -> None: + simulation = NewSimulation(kind="economy", country="us", region_code="state/ca") # type: ignore[arg-type] + report = NewReport( + kind="household", # type: ignore[arg-type] + country="us", + baseline_simulation_id=uuid4(), + year=2026, + ) + + assert simulation.kind is SimulationKind.ECONOMY + assert report.kind is ReportKind.HOUSEHOLD + + +def test_an_unknown_simulation_kind_is_rejected() -> None: + with pytest.raises(ValueError, match="kind"): + NewSimulation(kind="microsimulation", country="us", region_code="state/ca") # type: ignore[arg-type] + + +def test_an_unknown_report_kind_is_rejected() -> None: + with pytest.raises(ValueError, match="kind"): + NewReport( + kind="microsimulation", # type: ignore[arg-type] + country="us", + baseline_simulation_id=uuid4(), + year=2026, + ) diff --git a/libs/policyengine-simulation-persistence/tests/test_migration_cycle.py b/libs/policyengine-simulation-persistence/tests/test_migration_cycle.py new file mode 100644 index 000000000..b889e61a0 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_migration_cycle.py @@ -0,0 +1,268 @@ +"""Run the simulation migration chain against a fresh PostgreSQL database.""" + +from __future__ import annotations + +import re +import time + +import pytest +from alembic import command +from migration_support import ( + MIGRATOR_ROLE, + REPOSITORY_ROOT, + alembic_config, + applied_revision, + chain_config, + migration_cycle_failures, +) +from policyengine_simulation_persistence import ( + HEAD_REVISION, + SIMULATION_SCHEMA, + create_simulation_engine, + simulation_metadata, +) +from report_support import economy_pair, economy_report, report_store +from sqlalchemy import create_engine, inspect, text +from sqlalchemy.dialects import postgresql +from sqlalchemy.exc import OperationalError, ProgrammingError +from sqlalchemy.schema import CreateTable + +pytestmark = pytest.mark.postgres + + +def _superuser_engine(database, **options): + return create_engine( + database.superuser_url.replace("postgresql://", "postgresql+psycopg://", 1), + **options, + ) + + +def test_upgrade_check_downgrade_upgrade_cycle(disposable_database) -> None: + config = alembic_config() + + assert migration_cycle_failures(config, check=True) == [] + assert applied_revision(disposable_database.migrator_url) == HEAD_REVISION + + command.downgrade(config, "base") + assert applied_revision(disposable_database.migrator_url) is None + + +def test_rule_proof_migration_cycle(disposable_database) -> None: + """Proof for rule sim.persistence.migration-cycle (rules/registry.toml).""" + fixtures = ( + REPOSITORY_ROOT / "rules" / "fixtures" / "sim.persistence.migration-cycle" + ) + + assert ( + migration_cycle_failures(chain_config(fixtures / "good_chain"), check=False) + == [] + ) + assert migration_cycle_failures( + chain_config(fixtures / "bad_chain"), check=False + ) == [ + "upgrade failed: ProgrammingError: (psycopg.errors.DuplicateTable) relation " + '"proof_bad_table" already exists' + ] + + +def test_the_version_table_lives_only_in_the_simulation_schema( + disposable_database, +) -> None: + command.upgrade(alembic_config(), "head") + + engine = _superuser_engine(disposable_database) + try: + with engine.connect() as connection: + inspector = inspect(connection) + assert inspector.has_table("alembic_version", schema=SIMULATION_SCHEMA) + assert not inspector.has_table("alembic_version", schema="public") + schema_owner = connection.execute( + text( + "SELECT pg_get_userbyid(nspowner) FROM pg_namespace " + "WHERE nspname = :schema" + ), + {"schema": SIMULATION_SCHEMA}, + ).scalar() + finally: + engine.dispose() + assert schema_owner == MIGRATOR_ROLE + + +def test_an_unmigrated_database_reports_no_revision(disposable_database) -> None: + assert applied_revision(disposable_database.migrator_url) is None + + +def _precreate_schema_without_database_create(database) -> None: + """Set up the operator alternative to granting the migrator CREATE. + + The superuser takes over the database, so the migrator keeps only the + default CONNECT and TEMPORARY privileges, and creates the simulation + schema owned by the migrator. + """ + engine = _superuser_engine(database, isolation_level="AUTOCOMMIT") + try: + with engine.connect() as connection: + connection.execute( + text(f'ALTER DATABASE "{database.name}" OWNER TO CURRENT_USER') + ) + connection.execute( + text(f"CREATE SCHEMA {SIMULATION_SCHEMA} AUTHORIZATION {MIGRATOR_ROLE}") + ) + migrator_may_create = connection.execute( + text( + "SELECT has_database_privilege(:role, current_database(), 'CREATE')" + ), + {"role": MIGRATOR_ROLE}, + ).scalar() + finally: + engine.dispose() + assert migrator_may_create is False + + +def test_a_precreated_schema_needs_no_database_create_privilege( + disposable_database, +) -> None: + _precreate_schema_without_database_create(disposable_database) + + command.upgrade(alembic_config(), "head") + + assert applied_revision(disposable_database.migrator_url) == HEAD_REVISION + assert applied_revision(disposable_database.runtime_url) == HEAD_REVISION + + +def test_the_runtime_role_reads_the_revision_but_cannot_change_the_schema( + disposable_database, +) -> None: + command.upgrade(alembic_config(), "head") + + assert applied_revision(disposable_database.runtime_url) == HEAD_REVISION + engine = create_simulation_engine(disposable_database.runtime_url) + try: + with ( + pytest.raises(ProgrammingError, match="permission denied"), + engine.begin() as connection, + ): + connection.execute( + text("CREATE TABLE simulation.unowned (identifier integer)") + ) + with ( + pytest.raises(ProgrammingError, match="permission denied"), + engine.begin() as connection, + ): + connection.execute( + text("UPDATE simulation.alembic_version SET version_num = ''") + ) + finally: + engine.dispose() + + +def test_a_failed_revision_leaves_nothing_applied( + disposable_database, monkeypatch +) -> None: + """The schema and every revision apply in one transaction.""" + from alembic.runtime import migration + + def fail_after_creating_the_schema(*_arguments, **_keywords): + msg = "simulated failure inside the migration transaction" + raise RuntimeError(msg) + + monkeypatch.setattr( + migration.MigrationContext, + "run_migrations", + fail_after_creating_the_schema, + ) + + with pytest.raises(RuntimeError, match="simulated failure"): + command.upgrade(alembic_config(), "head") + + engine = _superuser_engine(disposable_database) + try: + with engine.connect() as connection: + assert not inspect(connection).has_schema(SIMULATION_SCHEMA) + finally: + engine.dispose() + + +def test_a_migration_waiting_for_a_lock_gives_up_instead_of_blocking( + disposable_database, +) -> None: + """Writers queue behind a waiting migration, so it must not wait long.""" + command.upgrade(alembic_config(), "head") + holder = _superuser_engine(disposable_database) + try: + with holder.begin() as connection: + connection.execute( + text("LOCK TABLE simulation.alembic_version IN ACCESS EXCLUSIVE MODE") + ) + started = time.monotonic() + with pytest.raises(OperationalError, match="lock timeout"): + command.upgrade(alembic_config(), "head") + waited = time.monotonic() - started + finally: + holder.dispose() + assert waited < 30 + + +@pytest.mark.parametrize("references_granted", [False]) +def test_the_chain_needs_references_on_the_public_api_tables( + disposable_database, +) -> None: + """Without the operator's REFERENCES grant, no foreign key can be created.""" + with pytest.raises(ProgrammingError, match="permission denied for table"): + command.upgrade(alembic_config(), "head") + + assert applied_revision(disposable_database.migrator_url) is None + + +@pytest.mark.parametrize("table_name", ["simulations", "reports"]) +def test_the_runtime_role_cannot_change_or_delete_a_definition( + migrated_database, public_api_rows, table_name +) -> None: + baseline, reform = economy_pair(migrated_database, public_api_rows) + report_store(migrated_database).create(economy_report(baseline, reform)) + engine = create_simulation_engine(migrated_database.runtime_url) + try: + for statement in ( + f"UPDATE simulation.{table_name} SET created_at = now()", # noqa: S608 # fixed table names + f"DELETE FROM simulation.{table_name}", # noqa: S608 # fixed table names + ): + with ( + pytest.raises(ProgrammingError, match="permission denied"), + engine.begin() as connection, + ): + connection.execute(text(statement)) + finally: + engine.dispose() + + +def _declared_constraint_names() -> set[str]: + """Return the names the table definitions give their constraints.""" + names: set[str] = set() + for table in simulation_metadata.sorted_tables: + ddl = str(CreateTable(table).compile(dialect=postgresql.dialect())) + names.update(re.findall(r"CONSTRAINT (\w+)", ddl)) + return names + + +def test_every_constraint_has_the_name_its_definition_gives_it( + migrated_database, +) -> None: + """Compare names here, since alembic check does not compare check constraints.""" + engine = _superuser_engine(migrated_database) + try: + with engine.connect() as connection: + database_names = set( + connection.execute( + text( + "SELECT conname FROM pg_constraint " + "JOIN pg_namespace ON pg_namespace.oid = connamespace " + "WHERE nspname = 'simulation' " + "AND contype IN ('c', 'f', 'p', 'u') " + "AND conrelid <> 'simulation.alembic_version'::regclass" + ) + ).scalars() + ) + finally: + engine.dispose() + + assert database_names == _declared_constraint_names() diff --git a/libs/policyengine-simulation-persistence/tests/test_migration_provisioning.py b/libs/policyengine-simulation-persistence/tests/test_migration_provisioning.py new file mode 100644 index 000000000..48c09b2b3 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_migration_provisioning.py @@ -0,0 +1,542 @@ +"""Migration provisioning creates migrated simulations and reports, once each.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any +from uuid import uuid4 + +import pytest +from migration_provisioning_support import ( + economy_report_for_rows, + economy_simulation_definition, + household_report_for_rows, + parse_definitions, +) +from policyengine_simulation_persistence import ( + ReportKind, + ReportStore, + SimulationStore, +) +from policyengine_simulation_persistence.migration_provisioning import ( + ProvisionedReport, + ProvisioningFailure, + provision_reports, +) +from public_api_support import superuser_engine +from report_support import report_store +from simulation_support import economy_simulation, simulation_store +from sqlalchemy import text +from sqlalchemy.exc import DataError + +if TYPE_CHECKING: + from conftest import DisposableDatabase + from policyengine_simulation_persistence.migration_provisioning import ( + ProvisioningResult, + ) + +pytestmark = pytest.mark.postgres + + +def _provision( + database: DisposableDatabase, + *report_definitions: dict[str, Any], + dry_run: bool = False, +) -> ProvisioningResult: + return provision_reports( + parse_definitions(*report_definitions), + simulation_store(database), + report_store(database), + dry_run=dry_run, + ) + + +def _row_counts(database: DisposableDatabase) -> tuple[int, int]: + engine = superuser_engine(database.superuser_url) + try: + with engine.connect() as connection: + return ( + connection.execute( + text("SELECT count(*) FROM simulation.simulations") + ).scalar_one(), + connection.execute( + text("SELECT count(*) FROM simulation.reports") + ).scalar_one(), + ) + finally: + engine.dispose() + + +def test_each_report_gets_its_simulations_and_then_the_report( + migrated_database, public_api_rows +) -> None: + result = _provision( + migrated_database, + economy_report_for_rows(public_api_rows, options={"target": "cliff"}), + household_report_for_rows(public_api_rows), + ) + + simulations = simulation_store(migrated_database) + reports = report_store(migrated_database) + baseline = simulations.get_by_legacy_simulation_id(9001) + reform = simulations.get_by_legacy_simulation_id(9002) + assert baseline is not None + assert reform is not None + assert (baseline.policy_id, baseline.region_code, baseline.dataset_name) == ( + None, + "state/ca", + None, + ) + assert reform.policy_id == public_api_rows.us_policy_id + economy = reports.get_by_legacy_report_id(4512) + assert economy is not None + assert economy.kind is ReportKind.ECONOMY + assert (economy.baseline_simulation_id, economy.reform_simulation_id) == ( + baseline.id, + reform.id, + ) + assert (economy.year, economy.options) == (2026, {"target": "cliff"}) + household = reports.get_by_legacy_report_id(4513) + assert household is not None + assert household.kind is ReportKind.HOUSEHOLD + assert result.reports == [ + ProvisionedReport(legacy_report_id=4512, report_id=economy.id, created=True), + ProvisionedReport(legacy_report_id=4513, report_id=household.id, created=True), + ] + assert result.failures == [] + assert (result.new_simulations, result.existing_simulations) == (4, 0) + assert (result.new_reports, result.existing_reports) == (2, 0) + assert result.dry_run is False + + +def test_running_again_creates_nothing_and_maps_the_same_reports( + migrated_database, public_api_rows +) -> None: + definitions = ( + economy_report_for_rows(public_api_rows), + household_report_for_rows(public_api_rows), + ) + first = _provision(migrated_database, *definitions) + counts_after_first_run = _row_counts(migrated_database) + + second = _provision(migrated_database, *definitions) + + assert _row_counts(migrated_database) == counts_after_first_run == (4, 2) + assert [mapped.report_id for mapped in second.reports] == [ + mapped.report_id for mapped in first.reports + ] + assert [mapped.created for mapped in second.reports] == [False, False] + assert (second.new_simulations, second.existing_simulations) == (0, 4) + assert (second.new_reports, second.existing_reports) == (0, 2) + assert second.failures == [] + + +def test_a_simulation_several_reports_use_is_created_once( + migrated_database, public_api_rows +) -> None: + result = _provision( + migrated_database, + economy_report_for_rows(public_api_rows, 4512), + economy_report_for_rows( + public_api_rows, + 4514, + reform=economy_simulation_definition( + 9005, policy_id=str(public_api_rows.us_policy_id), dataset_name="cps" + ), + baseline=economy_simulation_definition(9006, dataset_name="cps"), + ), + economy_report_for_rows(public_api_rows, 4515), + ) + + reports = report_store(migrated_database) + first = reports.get_by_legacy_report_id(4512) + third = reports.get_by_legacy_report_id(4515) + assert first is not None + assert third is not None + assert first.baseline_simulation_id == third.baseline_simulation_id + assert first.reform_simulation_id == third.reform_simulation_id + assert (result.new_simulations, result.new_reports) == (4, 3) + assert _row_counts(migrated_database) == (4, 3) + + +def test_a_single_simulation_economy_report_compares_it_with_itself( + migrated_database, public_api_rows +) -> None: + single = economy_simulation_definition( + 9001, policy_id=str(public_api_rows.us_policy_id) + ) + + result = _provision( + migrated_database, + economy_report_for_rows(public_api_rows, baseline=single, reform=single), + ) + + report = report_store(migrated_database).get_by_legacy_report_id(4512) + assert report is not None + assert report.baseline_simulation_id == report.reform_simulation_id + assert (result.new_simulations, result.new_reports) == (1, 1) + + +def test_a_simulation_already_migrated_differently_fails_its_report_only( + migrated_database, public_api_rows +) -> None: + simulation_store(migrated_database).create_migrated( + economy_simulation( + public_api_rows, + policy_id=None, + region_code="state/ny", + legacy_simulation_id=9001, + ) + ) + + result = _provision( + migrated_database, + economy_report_for_rows(public_api_rows), + household_report_for_rows(public_api_rows), + ) + + assert result.failures == [ + ProvisioningFailure( + legacy_report_id=4512, + reason=( + "v1 simulation 9001 is already migrated with a different region_code" + ), + ) + ] + assert report_store(migrated_database).get_by_legacy_report_id(4512) is None + assert [mapped.legacy_report_id for mapped in result.reports] == [4513] + assert (result.new_simulations, result.existing_simulations) == (2, 0) + + +def test_a_report_already_migrated_differently_fails( + migrated_database, public_api_rows +) -> None: + _provision(migrated_database, economy_report_for_rows(public_api_rows)) + + result = _provision( + migrated_database, economy_report_for_rows(public_api_rows, year=2027) + ) + + assert result.failures == [ + ProvisioningFailure( + legacy_report_id=4512, + reason="v1 report 4512 is already migrated with a different year", + ) + ] + assert result.reports == [] + assert (result.existing_simulations, result.existing_reports) == (2, 0) + + +@pytest.mark.parametrize( + ("reform_policy", "constraint"), + [ + ("missing", "fk_simulations_policy_country"), + ("uk", "fk_simulations_policy_country"), + ], + ids=["a policy the Public API does not have", "a policy of another country"], +) +def test_a_definition_the_database_rejects_fails_and_the_rest_continue( + migrated_database, public_api_rows, reform_policy, constraint +) -> None: + policy_id = uuid4() if reform_policy == "missing" else public_api_rows.uk_policy_id + + result = _provision( + migrated_database, + economy_report_for_rows( + public_api_rows, + reform=economy_simulation_definition(9002, policy_id=str(policy_id)), + ), + household_report_for_rows(public_api_rows), + ) + + (failure,) = result.failures + assert failure.legacy_report_id == 4512 + assert failure.reason.startswith("the database rejected v1 simulation 9002: ") + assert constraint in failure.reason + assert "\n" not in failure.reason + assert [mapped.legacy_report_id for mapped in result.reports] == [4513] + # The baseline was created before the reform failed; a later run reuses it. + assert (result.new_simulations, result.new_reports) == (3, 1) + + +def test_a_dry_run_writes_nothing_and_counts_what_a_run_would_create( + migrated_database, public_api_rows +) -> None: + result = _provision( + migrated_database, + economy_report_for_rows(public_api_rows, 4512), + economy_report_for_rows(public_api_rows, 4515), + household_report_for_rows(public_api_rows), + dry_run=True, + ) + + assert _row_counts(migrated_database) == (0, 0) + assert result.dry_run is True + assert result.reports == [] + assert result.failures == [] + assert (result.new_simulations, result.existing_simulations) == (4, 0) + assert (result.new_reports, result.existing_reports) == (3, 0) + + +def test_a_dry_run_after_a_run_maps_every_existing_report( + migrated_database, public_api_rows +) -> None: + definitions = ( + economy_report_for_rows(public_api_rows), + household_report_for_rows(public_api_rows), + ) + first = _provision(migrated_database, *definitions) + + dry_run = _provision(migrated_database, *definitions, dry_run=True) + + assert dry_run.reports == [ + ProvisionedReport( + legacy_report_id=mapped.legacy_report_id, + report_id=mapped.report_id, + created=False, + ) + for mapped in first.reports + ] + assert (dry_run.new_simulations, dry_run.existing_simulations) == (0, 4) + assert (dry_run.new_reports, dry_run.existing_reports) == (0, 2) + + +def test_a_dry_run_counts_a_report_whose_simulations_all_exist( + migrated_database, public_api_rows +) -> None: + _provision(migrated_database, economy_report_for_rows(public_api_rows, 4512)) + + result = _provision( + migrated_database, economy_report_for_rows(public_api_rows, 4515), dry_run=True + ) + + assert (result.new_simulations, result.existing_simulations) == (0, 2) + assert (result.new_reports, result.existing_reports) == (1, 0) + assert _row_counts(migrated_database) == (2, 1) + + +def test_a_dry_run_reports_rows_already_migrated_differently( + migrated_database, public_api_rows +) -> None: + _provision(migrated_database, economy_report_for_rows(public_api_rows)) + simulation_store(migrated_database).create_migrated( + economy_simulation( + public_api_rows, + policy_id=None, + region_code="state/ny", + legacy_simulation_id=9003, + ) + ) + + result = _provision( + migrated_database, + economy_report_for_rows(public_api_rows, year=2027), + household_report_for_rows(public_api_rows), + dry_run=True, + ) + + assert result.failures == [ + ProvisioningFailure( + legacy_report_id=4512, + reason="v1 report 4512 is already migrated with a different year", + ), + ProvisioningFailure( + legacy_report_id=4513, + reason=( + "v1 simulation 9003 is already migrated with a different kind, " + "household_id, region_code" + ), + ), + ] + assert _row_counts(migrated_database) == (3, 1) + + +def test_a_dry_run_reports_a_report_migrated_with_other_simulations( + migrated_database, public_api_rows +) -> None: + _provision(migrated_database, economy_report_for_rows(public_api_rows)) + + result = _provision( + migrated_database, + economy_report_for_rows( + public_api_rows, + baseline=economy_simulation_definition(9101), + reform=economy_simulation_definition( + 9102, policy_id=str(public_api_rows.us_policy_id) + ), + ), + dry_run=True, + ) + + assert result.failures == [ + ProvisioningFailure( + legacy_report_id=4512, + reason=( + "v1 report 4512 is already migrated, " + "with simulations this run would create" + ), + ) + ] + # A run creates nothing for a report that conflicts, so neither does this. + assert (result.new_simulations, result.new_reports) == (0, 0) + + +@pytest.mark.parametrize( + ("store_type", "subject"), + [(SimulationStore, "v1 simulation 9001"), (ReportStore, "v1 report 4512")], + ids=["a simulation", "a report"], +) +def test_a_value_the_database_cannot_store_fails_its_report_only( + migrated_database, public_api_rows, monkeypatch, store_type, subject +) -> None: + # The file check rejects the values known to be unstorable; any other + # value the database rejects as data fails its report in the same way. + create_migrated = store_type.create_migrated + + def reject_the_first_report(store, new_row): + if getattr(new_row, "legacy_simulation_id", None) == 9001 or ( + getattr(new_row, "legacy_report_id", None) == 4512 + ): + statement = "INSERT" + raise DataError( + statement, {}, ValueError("value out of range\nDETAIL: more") + ) + return create_migrated(store, new_row) + + monkeypatch.setattr(store_type, "create_migrated", reject_the_first_report) + + result = _provision( + migrated_database, + economy_report_for_rows(public_api_rows), + household_report_for_rows(public_api_rows), + ) + + assert result.failures == [ + ProvisioningFailure( + legacy_report_id=4512, + reason=f"the database rejected {subject}: value out of range", + ) + ] + assert [mapped.legacy_report_id for mapped in result.reports] == [4513] + + +def test_options_compare_as_the_json_numbers_the_database_stores( + migrated_database, public_api_rows +) -> None: + # JSONB keeps numbers as exact decimals, so 1e23 reads back as the integer + # 100000000000000000000000, which is not the float 1e23. + options = { + "threshold": 1e23, + "rate": 0.1, + "nested": {"limits": [1.2345678901234567e19, 1e300, 5e-324, -0.0, 7]}, + "flag": True, + } + definition = economy_report_for_rows(public_api_rows, options=options) + + first = _provision(migrated_database, definition) + second = _provision(migrated_database, definition) + + assert first.failures == [] + assert second.failures == [] + assert [mapped.created for mapped in first.reports + second.reports] == [ + True, + False, + ] + + +def test_a_later_run_reuses_the_simulations_a_failed_report_left( + migrated_database, public_api_rows +) -> None: + missing_policy = economy_simulation_definition(9002, policy_id=str(uuid4())) + first = _provision( + migrated_database, + economy_report_for_rows(public_api_rows, reform=missing_policy), + ) + baseline = simulation_store(migrated_database).get_by_legacy_simulation_id(9001) + + # The export is corrected and the command runs again. + second = _provision(migrated_database, economy_report_for_rows(public_api_rows)) + + report = report_store(migrated_database).get_by_legacy_report_id(4512) + assert [failure.legacy_report_id for failure in first.failures] == [4512] + assert baseline is not None + assert report is not None + assert report.baseline_simulation_id == baseline.id + assert second.failures == [] + assert (second.new_simulations, second.existing_simulations) == (1, 1) + assert (second.new_reports, second.existing_reports) == (1, 0) + + +def test_a_report_that_conflicts_with_an_existing_row_creates_nothing( + migrated_database, public_api_rows +) -> None: + # The runtime role cannot delete a row, so a run checks every existing row + # a report needs before it creates any of them. + simulation_store(migrated_database).create_migrated( + economy_simulation( + public_api_rows, region_code="state/ny", legacy_simulation_id=9002 + ) + ) + + result = _provision(migrated_database, economy_report_for_rows(public_api_rows)) + + assert result.failures == [ + ProvisioningFailure( + legacy_report_id=4512, + reason=( + "v1 simulation 9002 is already migrated with a different region_code" + ), + ) + ] + assert simulation_store(migrated_database).get_by_legacy_simulation_id(9001) is None + assert _row_counts(migrated_database) == (1, 0) + assert (result.new_simulations, result.existing_simulations) == (0, 0) + + +def test_a_run_creates_no_simulation_for_a_report_migrated_with_others( + migrated_database, public_api_rows +) -> None: + _provision(migrated_database, economy_report_for_rows(public_api_rows)) + + result = _provision( + migrated_database, + economy_report_for_rows( + public_api_rows, + baseline=economy_simulation_definition(9101), + reform=economy_simulation_definition( + 9102, policy_id=str(public_api_rows.us_policy_id) + ), + ), + ) + + assert result.failures == [ + ProvisioningFailure( + legacy_report_id=4512, + reason=( + "v1 report 4512 is already migrated, " + "with simulations this run would create" + ), + ) + ] + assert _row_counts(migrated_database) == (2, 1) + + +def test_a_dry_run_finds_an_existing_single_simulation_economy_report( + migrated_database, public_api_rows +) -> None: + single = economy_simulation_definition( + 9001, policy_id=str(public_api_rows.us_policy_id) + ) + definition = economy_report_for_rows( + public_api_rows, baseline=single, reform=single + ) + first = _provision(migrated_database, definition) + + dry_run = _provision(migrated_database, definition, dry_run=True) + + assert dry_run.failures == [] + assert dry_run.reports == [ + ProvisionedReport( + legacy_report_id=4512, report_id=first.reports[0].report_id, created=False + ) + ] + assert (dry_run.new_simulations, dry_run.existing_simulations) == (0, 1) diff --git a/libs/policyengine-simulation-persistence/tests/test_migration_provisioning_command.py b/libs/policyengine-simulation-persistence/tests/test_migration_provisioning_command.py new file mode 100644 index 000000000..2dc793e72 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_migration_provisioning_command.py @@ -0,0 +1,551 @@ +"""The migration provisioning command an operator runs.""" + +from __future__ import annotations + +import json +import os +import subprocess +import sys +from typing import TYPE_CHECKING, Any + +import pytest +from alembic import command +from migration_provisioning_support import ( + definitions_json, + economy_report_definition, + economy_report_for_rows, + household_report_for_rows, + household_simulation_definition, +) +from migration_support import alembic_config +from policyengine_simulation_persistence import ReportStore +from policyengine_simulation_persistence.migration_provisioning import ( + REPORT_MAPPING_SCHEMA_VERSION, + MappedReport, + ReportMappingFile, +) +from policyengine_simulation_persistence.migration_provisioning.command import ( + main, +) +from report_support import report_store +from simulation_support import simulation_store +from sqlalchemy.exc import OperationalError + +if TYPE_CHECKING: + from pathlib import Path + +MODULE = "policyengine_simulation_persistence.migration_provisioning" +URL_VARIABLE = "SIMULATION_DATABASE_URL" +# Nothing listens on port 1, so a connection is refused at once. +UNREACHABLE_URL = ( + "postgresql://simulation_runtime:not-a-real-password@127.0.0.1:1/postgres" +) + + +def _write_definitions(tmp_path: Path, *report_definitions: dict[str, Any]) -> Path: + path = tmp_path / "report-definitions.json" + path.write_text(definitions_json(*report_definitions)) + return path + + +def _run(definitions: Path, *options: str, database_url: str | None) -> int: + environ = {} if database_url is None else {URL_VARIABLE: database_url} + return main(["--definitions", str(definitions), *options], environ=environ) + + +def _summary(capsys: pytest.CaptureFixture[str]) -> dict[str, Any]: + return json.loads(capsys.readouterr().out) + + +def test_the_module_runs_as_a_command() -> None: + completed = subprocess.run( + [sys.executable, "-m", MODULE, "--help"], + capture_output=True, + text=True, + check=False, + ) + + assert completed.returncode == 0 + assert "--definitions" in completed.stdout + assert "--dry-run" in completed.stdout + + +def test_a_run_needs_an_output_file(tmp_path: Path) -> None: + definitions = _write_definitions(tmp_path) + + with pytest.raises(SystemExit) as exit_info: + _run(definitions, database_url=None) + + assert exit_info.value.code == 2 + + +def test_a_dry_run_takes_no_output_file(tmp_path: Path) -> None: + definitions = _write_definitions(tmp_path) + + with pytest.raises(SystemExit) as exit_info: + _run( + definitions, + "--dry-run", + "--output", + str(tmp_path / "out.json"), + database_url=None, + ) + + assert exit_info.value.code == 2 + + +def test_an_existing_output_file_is_never_overwritten( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + definitions = _write_definitions(tmp_path, economy_report_definition()) + output = tmp_path / "report-mapping.json" + output.write_text("earlier mapping") + + exit_code = _run(definitions, "--output", str(output), database_url=UNREACHABLE_URL) + + assert exit_code == 2 + assert output.read_text() == "earlier mapping" + assert "already exists" in capsys.readouterr().err + + +def test_the_output_file_needs_an_existing_directory( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + definitions = _write_definitions(tmp_path, economy_report_definition()) + + exit_code = _run( + definitions, + "--output", + str(tmp_path / "missing" / "report-mapping.json"), + database_url=UNREACHABLE_URL, + ) + + assert exit_code == 2 + assert "is not a directory" in capsys.readouterr().err + + +def test_an_invalid_definitions_file_is_refused_with_each_error( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + definitions = _write_definitions( + tmp_path, + economy_report_definition(country="USA"), + economy_report_definition(4600, reform=None), + ) + + exit_code = _run( + definitions, + "--output", + str(tmp_path / "report-mapping.json"), + database_url=UNREACHABLE_URL, + ) + + error_output = capsys.readouterr().err + assert exit_code == 2 + assert "is not a valid report definitions file" in error_output + assert "reports.0.economy: Value error, country must be" in error_output + assert "reports.1.economy.reform: Input should be an object" in error_output + assert not (tmp_path / "report-mapping.json").exists() + + +def test_a_file_with_many_errors_lists_the_first_twenty( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + definitions = _write_definitions( + tmp_path, + *( + economy_report_definition(legacy_report_id, country="USA") + for legacy_report_id in range(1, 26) + ), + ) + + exit_code = _run(definitions, "--dry-run", database_url=UNREACHABLE_URL) + + error_lines = capsys.readouterr().err.splitlines() + assert exit_code == 2 + assert len([line for line in error_lines if "country must be" in line]) == 20 + assert error_lines[-1] == " and 5 more" + + +def test_a_file_that_is_not_json_is_refused( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + definitions = tmp_path / "report-definitions.json" + definitions.write_text("{not json") + + exit_code = _run(definitions, "--dry-run", database_url=UNREACHABLE_URL) + + assert exit_code == 2 + assert "file: Invalid JSON" in capsys.readouterr().err + + +def test_a_missing_definitions_file_is_refused( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + exit_code = _run( + tmp_path / "missing.json", "--dry-run", database_url=UNREACHABLE_URL + ) + + assert exit_code == 2 + assert "cannot read" in capsys.readouterr().err + + +def test_the_database_url_comes_from_the_environment( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + definitions = _write_definitions(tmp_path, economy_report_definition()) + + exit_code = _run(definitions, "--dry-run", database_url=None) + + assert exit_code == 2 + assert f"{URL_VARIABLE} is not set" in capsys.readouterr().err + + +def test_a_malformed_database_url_is_refused_without_repeating_it( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + definitions = _write_definitions(tmp_path, economy_report_definition()) + + exit_code = _run( + definitions, + "--dry-run", + database_url="postgres://simulation_runtime:hunter2@db.example/postgres", + ) + + error_output = capsys.readouterr().err + assert exit_code == 2 + assert "must use postgresql://" in error_output + assert "hunter2" not in error_output + + +def test_an_unreachable_database_is_refused_without_its_password( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + definitions = _write_definitions(tmp_path, economy_report_definition()) + + exit_code = _run(definitions, "--dry-run", database_url=UNREACHABLE_URL) + + error_output = capsys.readouterr().err + assert exit_code == 2 + assert "cannot connect to the simulation database" in error_output + assert "not-a-real-password" not in error_output + assert "\n" not in error_output.rstrip("\n") + + +@pytest.mark.postgres +def test_a_run_writes_the_mapping_of_each_v1_report( + tmp_path: Path, migrated_database, public_api_rows, capsys +) -> None: + definitions = _write_definitions( + tmp_path, + economy_report_for_rows(public_api_rows), + household_report_for_rows(public_api_rows), + ) + output = tmp_path / "report-mapping.json" + + exit_code = _run( + definitions, "--output", str(output), database_url=migrated_database.runtime_url + ) + + reports = report_store(migrated_database) + economy = reports.get_by_legacy_report_id(4512) + household = reports.get_by_legacy_report_id(4513) + assert economy is not None + assert household is not None + mapping = ReportMappingFile.model_validate_json(output.read_text()) + assert mapping == ReportMappingFile( + schema_version=REPORT_MAPPING_SCHEMA_VERSION, + reports=[ + MappedReport(legacy_report_id=4512, report_id=economy.id, created=True), + MappedReport(legacy_report_id=4513, report_id=household.id, created=True), + ], + failures=[], + ) + assert exit_code == 0 + assert _summary(capsys) == { + "dry_run": False, + "new_simulations": 4, + "existing_simulations": 0, + "new_reports": 2, + "existing_reports": 0, + "failures": [], + } + + +@pytest.mark.postgres +def test_running_again_maps_the_same_reports( + tmp_path: Path, migrated_database, public_api_rows +) -> None: + definitions = _write_definitions(tmp_path, economy_report_for_rows(public_api_rows)) + first_output = tmp_path / "first.json" + second_output = tmp_path / "second.json" + _run( + definitions, + "--output", + str(first_output), + database_url=migrated_database.runtime_url, + ) + + exit_code = _run( + definitions, + "--output", + str(second_output), + database_url=migrated_database.runtime_url, + ) + + first = ReportMappingFile.model_validate_json(first_output.read_text()) + second = ReportMappingFile.model_validate_json(second_output.read_text()) + assert exit_code == 0 + assert [mapped.report_id for mapped in second.reports] == [ + mapped.report_id for mapped in first.reports + ] + assert [mapped.created for mapped in second.reports] == [False] + + +@pytest.mark.postgres +def test_failed_reports_are_listed_and_the_run_exits_with_one( + tmp_path: Path, migrated_database, public_api_rows, capsys +) -> None: + definitions = _write_definitions( + tmp_path, + economy_report_for_rows(public_api_rows), + household_report_for_rows( + public_api_rows, + baseline=household_simulation_definition( + 9003, household_id=str(public_api_rows.uk_household_id) + ), + reform=None, + ), + ) + output = tmp_path / "report-mapping.json" + + exit_code = _run( + definitions, "--output", str(output), database_url=migrated_database.runtime_url + ) + + mapping = ReportMappingFile.model_validate_json(output.read_text()) + assert exit_code == 1 + assert [mapped.legacy_report_id for mapped in mapping.reports] == [4512] + (failure,) = mapping.failures + assert failure.legacy_report_id == 4513 + assert "fk_simulations_household_country" in failure.reason + assert _summary(capsys)["failures"] == [ + {"legacy_report_id": 4513, "reason": failure.reason} + ] + + +@pytest.mark.postgres +def test_a_dry_run_prints_what_a_run_would_do( + tmp_path: Path, migrated_database, public_api_rows, capsys +) -> None: + definitions = _write_definitions(tmp_path, economy_report_for_rows(public_api_rows)) + + exit_code = _run( + definitions, "--dry-run", database_url=migrated_database.runtime_url + ) + + assert exit_code == 0 + assert _summary(capsys) == { + "dry_run": True, + "new_simulations": 2, + "existing_simulations": 0, + "new_reports": 1, + "existing_reports": 0, + "failures": [], + } + assert report_store(migrated_database).get_by_legacy_report_id(4512) is None + assert list(tmp_path.iterdir()) == [definitions] + + +@pytest.mark.postgres +def test_only_the_runtime_role_may_run_it( + tmp_path: Path, migrated_database, public_api_rows, capsys +) -> None: + definitions = _write_definitions(tmp_path, economy_report_for_rows(public_api_rows)) + output = tmp_path / "report-mapping.json" + + exit_code = _run( + definitions, + "--output", + str(output), + database_url=migrated_database.migrator_url, + ) + + assert exit_code == 2 + assert "connected as simulation_migrator" in capsys.readouterr().err + assert not output.exists() + assert report_store(migrated_database).get_by_legacy_report_id(4512) is None + + +@pytest.mark.postgres +def test_the_database_must_be_at_this_code_s_revision( + tmp_path: Path, disposable_database, capsys +) -> None: + command.upgrade(alembic_config(), "0001") + definitions = _write_definitions(tmp_path, economy_report_definition()) + + exit_code = _run( + definitions, "--dry-run", database_url=disposable_database.runtime_url + ) + + assert exit_code == 2 + assert ( + "the simulation database is at revision 0001, but this code expects 0002" + in capsys.readouterr().err + ) + + +@pytest.mark.postgres +def test_a_run_the_database_stops_writes_no_mapping_and_exits_with_three( + tmp_path: Path, + migrated_database, + public_api_rows, + capsys, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def lose_the_connection(store, new_report): + statement = "INSERT INTO simulation.reports" + raise OperationalError( + statement, {}, ConnectionError("server closed the connection") + ) + + monkeypatch.setattr(ReportStore, "create_migrated", lose_the_connection) + definitions = _write_definitions(tmp_path, economy_report_for_rows(public_api_rows)) + output = tmp_path / "report-mapping.json" + + exit_code = _run( + definitions, "--output", str(output), database_url=migrated_database.runtime_url + ) + + error_output = capsys.readouterr().err + assert exit_code == 3 + assert "server closed the connection" in error_output + assert "running the command again reuses them" in error_output + assert not output.exists() + # The simulations created before the stop stay, keyed by their v1 IDs. + assert ( + simulation_store(migrated_database).get_by_legacy_simulation_id(9001) + is not None + ) + + +@pytest.mark.postgres +def test_an_output_file_that_cannot_be_written_exits_with_three( + tmp_path: Path, + migrated_database, + public_api_rows, + capsys, + monkeypatch: pytest.MonkeyPatch, +) -> None: + from policyengine_simulation_persistence.migration_provisioning import ( + command as command_module, + ) + + output = tmp_path / "report-mapping.json" + provision_reports = command_module.provision_reports + + def provision_while_another_process_writes_the_output(*arguments, **options): + result = provision_reports(*arguments, **options) + output.write_text("written meanwhile") + return result + + monkeypatch.setattr( + command_module, + "provision_reports", + provision_while_another_process_writes_the_output, + ) + definitions = _write_definitions(tmp_path, economy_report_for_rows(public_api_rows)) + + exit_code = _run( + definitions, "--output", str(output), database_url=migrated_database.runtime_url + ) + + captured = capsys.readouterr() + assert exit_code == 3 + assert output.read_text() == "written meanwhile" + assert f"cannot write {output}" in captured.err + assert "with a new --output" in captured.err + assert json.loads(captured.out)["new_reports"] == 1 + + +@pytest.mark.postgres +def test_a_dry_run_the_database_stops_says_it_wrote_nothing( + tmp_path: Path, + migrated_database, + public_api_rows, + capsys, + monkeypatch: pytest.MonkeyPatch, +) -> None: + def lose_the_connection(store, legacy_report_id): + statement = "SELECT FROM simulation.reports" + raise OperationalError( + statement, {}, ConnectionError("server closed the connection") + ) + + monkeypatch.setattr(ReportStore, "get_by_legacy_report_id", lose_the_connection) + definitions = _write_definitions(tmp_path, economy_report_for_rows(public_api_rows)) + + exit_code = _run( + definitions, "--dry-run", database_url=migrated_database.runtime_url + ) + + error_output = capsys.readouterr().err + assert exit_code == 3 + assert "the database stopped the dry run: server closed the connection" in ( + error_output + ) + assert "The dry run wrote nothing." in error_output + assert "kept" not in error_output + + +@pytest.mark.postgres +def test_a_dry_run_that_finds_conflicts_exits_with_one( + tmp_path: Path, migrated_database, public_api_rows, capsys +) -> None: + first = _write_definitions(tmp_path, economy_report_for_rows(public_api_rows)) + _run( + first, + "--output", + str(tmp_path / "report-mapping.json"), + database_url=migrated_database.runtime_url, + ) + capsys.readouterr() + changed = tmp_path / "changed" / "report-definitions.json" + changed.parent.mkdir() + changed.write_text( + definitions_json(economy_report_for_rows(public_api_rows, year=2027)) + ) + + exit_code = _run(changed, "--dry-run", database_url=migrated_database.runtime_url) + + assert exit_code == 1 + assert _summary(capsys)["failures"] == [ + { + "legacy_report_id": 4512, + "reason": "v1 report 4512 is already migrated with a different year", + } + ] + + +@pytest.mark.skipif( + os.geteuid() == 0, reason="the superuser may write to any directory" +) +def test_an_output_directory_it_cannot_write_to_is_refused_first( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + definitions = _write_definitions(tmp_path, economy_report_definition()) + read_only = tmp_path / "read-only" + read_only.mkdir() + read_only.chmod(0o500) + try: + exit_code = _run( + definitions, + "--output", + str(read_only / "report-mapping.json"), + database_url=UNREACHABLE_URL, + ) + finally: + read_only.chmod(0o700) + + assert exit_code == 2 + assert f"cannot write to {read_only}" in capsys.readouterr().err diff --git a/libs/policyengine-simulation-persistence/tests/test_migration_provisioning_definitions.py b/libs/policyengine-simulation-persistence/tests/test_migration_provisioning_definitions.py new file mode 100644 index 000000000..588e9c22c --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_migration_provisioning_definitions.py @@ -0,0 +1,409 @@ +"""The export file of v1 report definitions that migration provisioning reads.""" + +from __future__ import annotations + +import re +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import pytest +from migration_provisioning_support import ( + US_HOUSEHOLD_ID, + US_REFORM_POLICY_ID, + definitions_json, + economy_report_definition, + economy_simulation_definition, + household_report_definition, + household_simulation_definition, + parse_definitions, +) +from policyengine_simulation_persistence import ( + NewReport, + NewSimulation, + ReportKind, + SimulationKind, + migration_provisioning, +) +from policyengine_simulation_persistence.migration_provisioning import ( + REPORT_DEFINITIONS_SCHEMA_VERSION, + EconomyReportDefinition, + HouseholdReportDefinition, + ReportDefinitionsFile, + ReportMappingFile, +) +from pydantic import ValidationError + + +def _error_messages(error: pytest.ExceptionInfo[ValidationError]) -> str: + return " | ".join( + ".".join(str(part) for part in detail["loc"]) + ": " + detail["msg"] + for detail in error.value.errors() + ) + + +def test_an_economy_report_becomes_two_simulations_and_a_report() -> None: + definitions = parse_definitions( + economy_report_definition(options={"target": "cliff"}) + ) + + (report,) = definitions.reports + assert isinstance(report, EconomyReportDefinition) + assert [ + report.new_simulation(simulation) for simulation in report.simulations() + ] == [ + NewSimulation( + kind=SimulationKind.ECONOMY, + country="us", + region_code="state/ca", + legacy_simulation_id=9001, + ), + NewSimulation( + kind=SimulationKind.ECONOMY, + country="us", + policy_id=US_REFORM_POLICY_ID, + region_code="state/ca", + legacy_simulation_id=9002, + ), + ] + baseline_id, reform_id = uuid4(), uuid4() + assert report.new_report(baseline_id, reform_id) == NewReport( + kind=ReportKind.ECONOMY, + country="us", + baseline_simulation_id=baseline_id, + reform_simulation_id=reform_id, + year=2026, + options={"target": "cliff"}, + legacy_report_id=4512, + ) + + +def test_a_household_report_may_have_only_a_baseline() -> None: + definitions = parse_definitions(household_report_definition()) + + (report,) = definitions.reports + assert isinstance(report, HouseholdReportDefinition) + (baseline,) = report.simulations() + assert report.new_simulation(baseline) == NewSimulation( + kind=SimulationKind.HOUSEHOLD, + country="us", + household_id=US_HOUSEHOLD_ID, + legacy_simulation_id=9003, + ) + baseline_id = uuid4() + assert report.new_report(baseline_id, None).reform_simulation_id is None + + +def test_a_household_comparison_has_two_policies_on_one_household() -> None: + definitions = parse_definitions( + household_report_definition( + reform=household_simulation_definition( + 9004, policy_id=str(US_REFORM_POLICY_ID) + ) + ) + ) + + (report,) = definitions.reports + assert [simulation.policy_id for simulation in report.simulations()] == [ + None, + US_REFORM_POLICY_ID, + ] + + +def test_a_single_simulation_economy_report_repeats_its_simulation() -> None: + # v1 calculates an economy report with one simulation as that policy over + # itself, so the export names the same simulation as baseline and reform. + single = economy_simulation_definition(9001, policy_id=str(US_REFORM_POLICY_ID)) + + definitions = parse_definitions( + economy_report_definition(baseline=single, reform=single) + ) + + (report,) = definitions.reports + assert [simulation.legacy_simulation_id for simulation in report.simulations()] == [ + 9001, + 9001, + ] + + +def test_the_schema_version_is_one() -> None: + assert REPORT_DEFINITIONS_SCHEMA_VERSION == 1 + + with pytest.raises(ValidationError) as error: + ReportDefinitionsFile.model_validate_json( + definitions_json(economy_report_definition(), schema_version=2) + ) + + assert _error_messages(error).startswith("schema_version:") + + +def test_an_unknown_report_kind_is_rejected() -> None: + with pytest.raises(ValidationError) as error: + parse_definitions(economy_report_definition(kind="budget_window")) + + assert "reports.0:" in _error_messages(error) + + +@pytest.mark.parametrize( + ("report_definition", "location"), + [ + (economy_report_definition(reform=None), "reports.0.economy.reform"), + ( + economy_report_definition( + baseline=economy_simulation_definition( + 9001, household_id=str(US_HOUSEHOLD_ID) + ) + ), + "reports.0.economy.baseline.household_id", + ), + ( + household_report_definition( + baseline=household_simulation_definition(9003, region_code="state/ca") + ), + "reports.0.household.baseline.region_code", + ), + ( + economy_report_definition(dynamic_id=str(uuid4())), + "reports.0.economy.dynamic_id", + ), + ], + ids=[ + "economy without a reform", + "economy simulation with a household", + "household simulation with a region", + "a field the format does not have", + ], +) +def test_a_report_must_have_the_fields_of_its_kind( + report_definition: dict[str, Any], location: str +) -> None: + with pytest.raises(ValidationError) as error: + parse_definitions(report_definition) + + assert _error_messages(error).startswith(location + ":") + + +@pytest.mark.parametrize("field_name", ["policy_id", "dataset_name"]) +def test_nullable_simulation_fields_must_still_be_given(field_name: str) -> None: + baseline = economy_simulation_definition(9001) + del baseline[field_name] + + with pytest.raises(ValidationError) as error: + parse_definitions(economy_report_definition(baseline=baseline)) + + assert _error_messages(error) == ( + f"reports.0.economy.baseline.{field_name}: Field required" + ) + + +@pytest.mark.parametrize( + ("overrides", "message"), + [ + ({"country": "USA"}, "country must be a lowercase two-letter code"), + ( + {"baseline": economy_simulation_definition(9001, region_code="state ca")}, + "region_code must be 1 to 255 characters with no whitespace", + ), + ( + {"baseline": economy_simulation_definition(9001, dataset_name="")}, + "dataset_name must be 1 to 255 characters with no whitespace", + ), + ({"year": 1899}, "greater than or equal to 1900"), + ({"year": 2201}, "less than or equal to 2200"), + ({"legacy_report_id": 0}, "greater than 0"), + ( + {"baseline": economy_simulation_definition(-3)}, + "greater than 0", + ), + ({"options": ["target"]}, "Input should be an object"), + ], + ids=[ + "country", + "region code", + "dataset name", + "year too early", + "year too late", + "report ID", + "simulation ID", + "options", + ], +) +def test_malformed_values_are_rejected(overrides: dict[str, Any], message: str) -> None: + with pytest.raises(ValidationError) as error: + parse_definitions(economy_report_definition(**overrides)) + + assert message in _error_messages(error) + + +@pytest.mark.parametrize( + "overrides", + [ + {"legacy_report_id": "4512"}, + {"legacy_report_id": True}, + {"year": "2026"}, + {"baseline": economy_simulation_definition(9001.0)}, + ], + ids=["report ID as text", "report ID as a boolean", "year as text", "float ID"], +) +def test_numbers_are_not_converted(overrides: dict[str, Any]) -> None: + with pytest.raises(ValidationError): + parse_definitions(economy_report_definition(**overrides)) + + +def test_an_economy_report_compares_policies_on_one_region_and_dataset() -> None: + with pytest.raises(ValidationError) as error: + parse_definitions( + economy_report_definition( + reform=economy_simulation_definition( + 9002, policy_id=str(US_REFORM_POLICY_ID), region_code="state/ny" + ) + ) + ) + + assert "the same region_code and dataset_name" in _error_messages(error) + + +def test_a_household_report_compares_policies_on_one_household() -> None: + with pytest.raises(ValidationError) as error: + parse_definitions( + household_report_definition( + reform=household_simulation_definition(9004, household_id=str(uuid4())) + ) + ) + + assert "the same household_id" in _error_messages(error) + + +def test_each_v1_report_appears_once() -> None: + with pytest.raises(ValidationError) as error: + parse_definitions( + economy_report_definition(4512), + household_report_definition(4512), + economy_report_definition(4600), + economy_report_definition(4600), + ) + + assert "v1 reports appear more than once: 4512, 4600" in _error_messages(error) + + +def test_a_simulation_shared_by_reports_is_described_the_same_way() -> None: + shared_baseline = economy_simulation_definition(9001) + + definitions = parse_definitions( + economy_report_definition(4512, baseline=shared_baseline), + economy_report_definition(4514, baseline=shared_baseline), + ) + + assert len(definitions.reports) == 2 + + +def test_a_simulation_described_two_ways_is_rejected() -> None: + with pytest.raises(ValidationError) as error: + parse_definitions( + economy_report_definition(4512), + economy_report_definition( + 4514, + baseline=economy_simulation_definition(9001, dataset_name="cps_2024"), + reform=economy_simulation_definition( + 9002, + policy_id=str(US_REFORM_POLICY_ID), + dataset_name="cps_2024", + ), + ), + ) + + assert ( + "v1 simulations are described differently in different reports: 9001, 9002" + in _error_messages(error) + ) + + +def test_policy_and_household_ids_are_uuids() -> None: + with pytest.raises(ValidationError) as error: + parse_definitions( + economy_report_definition( + reform=economy_simulation_definition(9002, policy_id="17") + ) + ) + + assert _error_messages(error).startswith("reports.0.economy.reform.policy_id:") + + +@pytest.mark.parametrize( + "overrides", + [ + {"legacy_report_id": 2**63}, + {"baseline": economy_simulation_definition(2**63)}, + ], + ids=["report ID", "simulation ID"], +) +def test_ids_beyond_the_database_s_bigint_range_are_rejected( + overrides: dict[str, Any], +) -> None: + with pytest.raises(ValidationError) as error: + parse_definitions(economy_report_definition(**overrides)) + + assert "less than or equal to 9223372036854775807" in _error_messages(error) + + +@pytest.mark.parametrize( + "options", + [ + {"limit": float("nan")}, + {"limits": [1, float("inf")]}, + {"nested": {"floor": float("-inf")}}, + ], + ids=["NaN", "Infinity in a list", "-Infinity in an object"], +) +def test_options_hold_only_finite_numbers(options: dict[str, Any]) -> None: + with pytest.raises(ValidationError) as error: + parse_definitions(economy_report_definition(options=options)) + + assert "options must not hold NaN or Infinity" in _error_messages(error) + + +@pytest.mark.parametrize( + "overrides", + [ + { + "baseline": economy_simulation_definition(9001, region_code="state/\x00ca"), + "reform": economy_simulation_definition( + 9002, policy_id=str(US_REFORM_POLICY_ID), region_code="state/\x00ca" + ), + }, + { + "baseline": economy_simulation_definition(9001, dataset_name="cps\x00"), + "reform": economy_simulation_definition( + 9002, policy_id=str(US_REFORM_POLICY_ID), dataset_name="cps\x00" + ), + }, + {"options": {"note": "a\x00b"}}, + {"options": {"a\x00b": 1}}, + {"options": {"notes": [{"text": "\x00"}]}}, + ], + ids=[ + "region code", + "dataset name", + "option value", + "option key", + "nested option value", + ], +) +def test_text_never_holds_a_nul_character(overrides: dict[str, Any]) -> None: + with pytest.raises(ValidationError) as error: + parse_definitions(economy_report_definition(**overrides)) + + assert "must not contain a NUL character" in _error_messages(error) + + +def test_the_readme_examples_follow_the_formats() -> None: + readme = Path(migration_provisioning.__file__).with_name("README.md").read_text() + definitions_example, mapping_example = re.findall( + r"```json\n(.*?)```", readme, flags=re.DOTALL + ) + + definitions = ReportDefinitionsFile.model_validate_json(definitions_example) + mapping = ReportMappingFile.model_validate_json(mapping_example) + + assert [report.kind for report in definitions.reports] == ["economy", "household"] + assert [mapped.legacy_report_id for mapped in mapping.reports] == [4512] diff --git a/libs/policyengine-simulation-persistence/tests/test_migration_provisioning_models.py b/libs/policyengine-simulation-persistence/tests/test_migration_provisioning_models.py new file mode 100644 index 000000000..c2b7f152f --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_migration_provisioning_models.py @@ -0,0 +1,51 @@ +"""Every Pydantic model of migration provisioning resolves its annotations. + +The repository's runtime-evaluated-models guard imports other packages. This +module covers one half of what it checks for the provisioning models: each +module imports without running anything, and each model it defines builds its +validators. It does not check how Ruff resolves the models' base classes; every +model here derives from ``pydantic.BaseModel`` within its own module, which +Ruff follows. +""" + +from __future__ import annotations + +import importlib +import inspect +import pkgutil + +import pytest +from policyengine_simulation_persistence import migration_provisioning +from pydantic import BaseModel + +MODULE_NAMES = sorted( + module.name + for module in pkgutil.iter_modules( + migration_provisioning.__path__, migration_provisioning.__name__ + "." + ) +) + + +def test_every_module_is_scanned() -> None: + assert [name.rsplit(".", 1)[1] for name in MODULE_NAMES] == [ + "__main__", + "command", + "definitions", + "mapping", + "provisioning", + ] + + +@pytest.mark.parametrize("module_name", MODULE_NAMES) +def test_importing_a_module_runs_nothing_and_its_models_resolve( + module_name: str, +) -> None: + module = importlib.import_module(module_name) + + models = [ + member + for _, member in inspect.getmembers(module, inspect.isclass) + if issubclass(member, BaseModel) and member.__module__ == module_name + ] + for model in models: + model.model_rebuild(force=True, raise_errors=True) diff --git a/libs/policyengine-simulation-persistence/tests/test_migration_script.py b/libs/policyengine-simulation-persistence/tests/test_migration_script.py new file mode 100644 index 000000000..e7b09e217 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_migration_script.py @@ -0,0 +1,54 @@ +"""The deploy job's migration script, run for real against PostgreSQL.""" + +from __future__ import annotations + +import os +import subprocess +from pathlib import Path + +import pytest +from migration_support import ( + MIGRATION_DATABASE_URL_VARIABLE, + REPOSITORY_ROOT, + applied_revision, +) +from policyengine_simulation_persistence import ( + HEAD_REVISION, +) + +pytestmark = pytest.mark.postgres + +SCRIPT = REPOSITORY_ROOT / ".github" / "scripts" / "migrate-simulation-database.sh" +LIBRARY_DIRECTORY = Path(__file__).resolve().parents[1] + + +def _run_script(database, mode: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + ["bash", str(SCRIPT)], + cwd=LIBRARY_DIRECTORY, + env={ + **os.environ, + MIGRATION_DATABASE_URL_VARIABLE: database.migrator_url, + "SIMULATION_MIGRATION_MODE": mode, + }, + capture_output=True, + text=True, + check=False, + ) + + +def test_verify_mode_refuses_a_database_behind_the_chain(disposable_database) -> None: + result = _run_script(disposable_database, "verify") + + assert result.returncode != 0 + assert applied_revision(disposable_database.migrator_url) is None + + +def test_upgrade_mode_migrates_and_then_verify_mode_passes(disposable_database) -> None: + upgraded = _run_script(disposable_database, "upgrade") + assert upgraded.returncode == 0, upgraded.stderr + assert applied_revision(disposable_database.migrator_url) == HEAD_REVISION + + verified = _run_script(disposable_database, "verify") + assert verified.returncode == 0, verified.stderr + assert "simulation-test-password" not in upgraded.stdout + upgraded.stderr diff --git a/libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py b/libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py new file mode 100644 index 000000000..eb66ec159 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py @@ -0,0 +1,178 @@ +"""Simulation schema ownership: one chain, no executor dependency, none in Modal.""" + +from __future__ import annotations + +import pytest +from migration_support import REPOSITORY_ROOT +from ownership_checks import ( + declares_persistence_dependency, + migration_chain_violations, + modal_credential_violations, + modal_source_violations, + persistence_dependency_violations, +) + +FIXTURES = REPOSITORY_ROOT / "rules" / "fixtures" +WORKFLOWS = REPOSITORY_ROOT / ".github" / "workflows" +WORKFLOW_SCRIPTS = REPOSITORY_ROOT / ".github" / "scripts" +MODAL_PROJECTS = ( + REPOSITORY_ROOT / "projects" / "policyengine-simulation-executor", + REPOSITORY_ROOT / "projects" / "policyengine-simulation-gateway", +) + + +def _texts(directory, pattern): + return { + path.name: path.read_text(encoding="utf-8") + for path in sorted(directory.glob(pattern)) + } + + +def test_repository_has_one_simulation_migration_chain() -> None: + assert migration_chain_violations(REPOSITORY_ROOT) == [] + + +def test_rule_proof_one_migration_chain() -> None: + """Proof for rule sim.persistence.one-migration-chain (rules/registry.toml).""" + fixtures = FIXTURES / "sim.persistence.one-migration-chain" + + assert migration_chain_violations(fixtures / "bad_repository") != [] + assert migration_chain_violations(fixtures / "good_repository") == [] + + +def test_modal_projects_never_depend_on_the_persistence_library() -> None: + for project_root in MODAL_PROJECTS: + assert persistence_dependency_violations(project_root) == [] + + +def test_rule_proof_executor_independent() -> None: + """Proof for rule sim.persistence.executor-independent (rules/registry.toml).""" + fixtures = FIXTURES / "sim.persistence.executor-independent" + + assert declares_persistence_dependency( + (fixtures / "bad_pyproject.toml").read_text(encoding="utf-8") + ) + assert not declares_persistence_dependency( + (fixtures / "good_pyproject.toml").read_text(encoding="utf-8") + ) + + +def test_modal_receives_no_database_url_but_the_comparison_paths() -> None: + assert ( + modal_credential_violations( + _texts(WORKFLOWS, "*.yml"), + _texts(WORKFLOW_SCRIPTS, "*.sh"), + ) + == [] + ) + for project_root in MODAL_PROJECTS: + sources = { + str(path): path.read_text(encoding="utf-8") + for path in sorted((project_root / "src").rglob("*.py")) + } + assert modal_source_violations(sources) == [] + + +def test_rule_proof_no_modal_database_credentials() -> None: + """Proof for rule sim.persistence.no-modal-database-credentials.""" + fixtures = FIXTURES / "sim.persistence.no-modal-database-credentials" + no_scripts: dict[str, str] = {} + + assert modal_credential_violations( + {"bad.yml": (fixtures / "bad_workflow.yml").read_text(encoding="utf-8")}, + no_scripts, + ) + assert not modal_credential_violations( + {"good.yml": (fixtures / "good_workflow.yml").read_text(encoding="utf-8")}, + no_scripts, + ) + + +def _modal_secret_workflow(*, step_env: str = "", job_env: str = "") -> str: + return ( + "jobs:\n" + " deploy:\n" + f"{job_env}" + " steps:\n" + " - name: Synchronize Modal secrets\n" + f"{step_env}" + " run: uv run modal secret create runtime --from-dotenv .env\n" + ) + + +@pytest.mark.parametrize( + ("step_env", "job_env"), + [ + (" env:\n REPORTS_DATABASE_URL: x\n", ""), + (" env:\n DATABASE_URL: x\n", ""), + (" env:\n SIMULATION_DATABASE_URL: x\n", ""), + ("", " env:\n SIMULATION_MIGRATION_DATABASE_URL: x\n"), + (" env:\n DSN: postgresql://user:secret@db/postgres\n", ""), + ], + ids=[ + "any-database-url", + "bare-database-url", + "simulation-url", + "job-level-env", + "url-literal", + ], +) +def test_modal_steps_may_carry_only_the_comparison_paths_database_url( + step_env, job_env +) -> None: + workflow = _modal_secret_workflow(step_env=step_env, job_env=job_env) + + assert modal_credential_violations({"deploy.yml": workflow}, {}) != [] + + +def test_the_comparison_paths_database_url_may_reach_modal_until_it_is_removed() -> ( + None +): + workflow = _modal_secret_workflow( + step_env=( + " env:\n" + " STAGE12_DATABASE_URL_SECRET_NAME: stage12-database-url\n" + ) + ) + script = "jq -n '{STAGE12_DATABASE_URL: $url}'\nuv run modal secret create x\n" + + assert modal_credential_violations({"deploy.yml": workflow}, {}) == [] + assert modal_credential_violations({}, {"stage12-sync.sh": script}) == [] + + +def test_a_modal_script_may_not_carry_another_database_url() -> None: + script = ( + "uv run modal secret create x REPORTS_DATABASE_URL=${REPORTS_DATABASE_URL}\n" + ) + + assert modal_credential_violations({}, {"sync.sh": script}) == [ + "sync.sh gives Modal a database credential other than the comparison " + "path's: REPORTS_DATABASE_URL" + ] + + +def test_modal_sources_may_name_only_the_comparison_paths_database_url() -> None: + assert ( + modal_source_violations({"worker.py": "os.environ['STAGE12_DATABASE_URL']"}) + == [] + ) + assert modal_source_violations( + {"worker.py": "os.environ['SIMULATION_DATABASE_URL']"} + ) == ["worker.py names SIMULATION_DATABASE_URL"] + assert modal_source_violations({"worker.py": "os.getenv('DATABASE_URL')"}) == [ + "worker.py names DATABASE_URL" + ] + + +def test_a_temporary_modal_app_may_not_receive_another_database_url() -> None: + workflow = ( + "jobs:\n" + " precompute:\n" + " steps:\n" + " - name: Precompute\n" + " env:\n" + " SIMULATION_DATABASE_URL: x\n" + " run: uv run --frozen modal run src/modal/precompute_app.py\n" + ) + + assert modal_credential_violations({"precompute.yml": workflow}, {}) != [] diff --git a/libs/policyengine-simulation-persistence/tests/test_reports.py b/libs/policyengine-simulation-persistence/tests/test_reports.py new file mode 100644 index 000000000..d1061a8b7 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_reports.py @@ -0,0 +1,229 @@ +"""simulation.reports: which simulations a report compares, its year and options.""" + +from __future__ import annotations + +from uuid import uuid4 + +import pytest +from policyengine_simulation_persistence import ( + ReportKind, + ReportRecord, + create_simulation_engine, +) +from policyengine_simulation_persistence.reports import reports +from public_api_support import superuser_engine +from report_support import ( + economy_pair, + economy_report, + household_baseline, + report_store, +) +from simulation_support import economy_simulation, simulation_store +from sqlalchemy import insert, text +from sqlalchemy.exc import IntegrityError + +pytestmark = pytest.mark.postgres + + +def test_an_economy_report_compares_a_baseline_and_a_reform( + migrated_database, public_api_rows +) -> None: + baseline, reform = economy_pair(migrated_database, public_api_rows) + store = report_store(migrated_database) + + created = store.create( + economy_report(baseline, reform, options={"target": "cliffs"}) + ) + + assert isinstance(created, ReportRecord) + assert created.kind is ReportKind.ECONOMY + assert created.baseline_simulation_id == baseline.id + assert created.reform_simulation_id == reform.id + assert created.year == 2026 + assert created.options == {"target": "cliffs"} + assert created.legacy_report_id is None + assert store.get(created.id) == created + + +def test_a_household_report_may_have_only_a_baseline( + migrated_database, public_api_rows +) -> None: + baseline = household_baseline(migrated_database, public_api_rows) + + created = report_store(migrated_database).create( + economy_report( + baseline, + baseline, + kind=ReportKind.HOUSEHOLD, + reform_simulation_id=None, + ) + ) + + assert created.kind is ReportKind.HOUSEHOLD + assert created.reform_simulation_id is None + assert created.options == {} + + +def test_an_economy_report_needs_a_reform(migrated_database, public_api_rows) -> None: + baseline, reform = economy_pair(migrated_database, public_api_rows) + + with pytest.raises(IntegrityError, match="economy_has_reform"): + report_store(migrated_database).create( + economy_report(baseline, reform, reform_simulation_id=None) + ) + + +def test_simulations_must_have_the_report_country( + migrated_database, public_api_rows +) -> None: + baseline, _ = economy_pair(migrated_database, public_api_rows) + uk_reform = simulation_store(migrated_database).create( + economy_simulation( + public_api_rows, + country="uk", + policy_id=public_api_rows.uk_policy_id, + region_code="country/wales", + ) + ) + + with pytest.raises(IntegrityError, match="fk_reports_reform_simulation"): + report_store(migrated_database).create(economy_report(baseline, uk_reform)) + + +def test_simulations_must_have_the_report_kind( + migrated_database, public_api_rows +) -> None: + baseline, reform = economy_pair(migrated_database, public_api_rows) + + with pytest.raises(IntegrityError, match="fk_reports_baseline_simulation"): + report_store(migrated_database).create( + economy_report( + baseline, + reform, + kind=ReportKind.HOUSEHOLD, + reform_simulation_id=None, + ) + ) + + +def test_an_unknown_simulation_is_rejected(migrated_database, public_api_rows) -> None: + baseline, reform = economy_pair(migrated_database, public_api_rows) + + with pytest.raises(IntegrityError, match="fk_reports_reform_simulation"): + report_store(migrated_database).create( + economy_report(baseline, reform, reform_simulation_id=uuid4()) + ) + + +@pytest.mark.parametrize( + ("column", "value", "constraint"), + [ + ("year", "1899", "ck_reports_year"), + ("options", "'[]'::jsonb", "ck_reports_options"), + ], +) +def test_the_database_checks_the_year_and_options( + migrated_database, public_api_rows, column, value, constraint +) -> None: + """The database checks what NewReport also checks, for any other writer.""" + baseline, reform = economy_pair(migrated_database, public_api_rows) + values = {"year": "2026", "options": "'{}'::jsonb", column: value} + engine = superuser_engine(migrated_database.superuser_url) + try: + with ( + pytest.raises(IntegrityError, match=constraint), + engine.begin() as connection, + ): + connection.execute( + text( + "INSERT INTO simulation.reports (id, kind, country, " # noqa: S608 # fixed test values + "baseline_simulation_id, reform_simulation_id, year, options) " + f"VALUES (:id, 'economy', 'us', :baseline, :reform, " + f"{values['year']}, {values['options']})" + ), + {"id": uuid4(), "baseline": baseline.id, "reform": reform.id}, + ) + finally: + engine.dispose() + + +def test_the_public_api_cannot_delete_a_simulation_a_report_uses( + migrated_database, public_api_rows +) -> None: + baseline, reform = economy_pair(migrated_database, public_api_rows) + report_store(migrated_database).create(economy_report(baseline, reform)) + engine = superuser_engine(migrated_database.superuser_url) + try: + with ( + pytest.raises(IntegrityError, match="violates foreign key constraint"), + engine.begin() as connection, + ): + connection.execute( + text("DELETE FROM simulation.simulations WHERE id = :id"), + {"id": baseline.id}, + ) + finally: + engine.dispose() + + +def test_a_migrated_report_is_created_once(migrated_database, public_api_rows) -> None: + baseline, reform = economy_pair(migrated_database, public_api_rows) + store = report_store(migrated_database) + first = store.create_migrated(economy_report(baseline, reform, legacy_report_id=91)) + + again = store.create_migrated( + economy_report(baseline, reform, legacy_report_id=91, year=2027) + ) + + assert again == first + assert store.get_by_legacy_report_id(91) == first + assert store.get_by_legacy_report_id(92) is None + + +def test_a_migrated_report_needs_its_legacy_id( + migrated_database, public_api_rows +) -> None: + baseline, reform = economy_pair(migrated_database, public_api_rows) + + with pytest.raises(ValueError, match="legacy_report_id"): + report_store(migrated_database).create_migrated( + economy_report(baseline, reform) + ) + + +def test_an_unknown_report_reads_as_none(migrated_database) -> None: + assert report_store(migrated_database).get(uuid4()) is None + + +@pytest.mark.parametrize( + ("column", "value", "constraint"), + [ + ("kind", "microsimulation", "ck_reports_kind"), + ("country", "GB", "ck_reports_country"), + ("legacy_report_id", -4, "ck_reports_legacy_report_id"), + ("reform_simulation_id", None, "ck_reports_economy_has_reform"), + ], +) +def test_the_database_checks_each_report_field( + migrated_database, public_api_rows, column, value, constraint +) -> None: + """The database checks what NewReport also checks, for any other writer.""" + baseline, reform = economy_pair(migrated_database, public_api_rows) + values = { + "id": uuid4(), + "kind": "economy", + "country": "us", + "baseline_simulation_id": baseline.id, + "reform_simulation_id": reform.id, + "year": 2026, + column: value, + } + engine = create_simulation_engine(migrated_database.runtime_url) + try: + with ( + pytest.raises(IntegrityError, match=constraint), + engine.begin() as connection, + ): + connection.execute(insert(reports).values(**values)) + finally: + engine.dispose() diff --git a/libs/policyengine-simulation-persistence/tests/test_schema.py b/libs/policyengine-simulation-persistence/tests/test_schema.py new file mode 100644 index 000000000..8ac5b7b05 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_schema.py @@ -0,0 +1,119 @@ +"""Schema names and connection settings, without a database.""" + +from __future__ import annotations + +import importlib +import pkgutil + +import policyengine_simulation_persistence +import pytest +from alembic import command +from alembic.script import ScriptDirectory +from migration_support import MIGRATION_DATABASE_URL_VARIABLE, alembic_config +from policyengine_simulation_persistence import ( + HEAD_REVISION, + RUNTIME_ROLE, + SIMULATION_SCHEMA, + create_simulation_engine, + normalized_database_url, + simulation_metadata, +) + + +def test_the_schema_and_runtime_role_have_their_deployed_names() -> None: + assert SIMULATION_SCHEMA == "simulation" + assert RUNTIME_ROLE == "simulation_runtime" + + +def test_every_table_is_in_the_simulation_schema() -> None: + assert simulation_metadata.schema == SIMULATION_SCHEMA + for table in simulation_metadata.tables.values(): + assert table.schema == SIMULATION_SCHEMA, table.name + + +def test_database_url_selects_psycopg_without_changing_the_secret_format() -> None: + url = normalized_database_url( + "postgresql://simulation_runtime:secret@db.example:5432/postgres" + ) + + assert url.drivername == "postgresql+psycopg" + assert url.username == "simulation_runtime" + assert url.host == "db.example" + assert url.database == "postgres" + + +@pytest.mark.parametrize( + "database_url", + [ + "", + " postgresql://user:secret@db.example/postgres", + "postgres://user:secret@db.example/postgres", + "postgresql+psycopg://user:secret@db.example/postgres", + "postgresql://db.example/postgres", + "postgresql://user:secret@db.example", + "not a url", + ], +) +def test_malformed_database_urls_are_rejected_without_echoing_them( + database_url, +) -> None: + with pytest.raises(ValueError, match="Simulation database URL") as error: + normalized_database_url(database_url) + + assert "secret" not in str(error.value) + assert error.value.__cause__ is None or "secret" not in str(error.value.__cause__) + + +def test_runtime_engine_never_prepares_server_side_statements(monkeypatch) -> None: + captured_options = {} + + def capture_engine(url, **options): + captured_options.update(options, url=url) + return object() + + monkeypatch.setattr( + "policyengine_simulation_persistence.engine.create_engine", + capture_engine, + ) + + create_simulation_engine("postgresql://runtime:secret@db.example/postgres") + + assert captured_options["url"].drivername == "postgresql+psycopg" + assert captured_options["connect_args"]["prepare_threshold"] is None + assert captured_options["connect_args"]["connect_timeout"] > 0 + assert captured_options["pool_pre_ping"] is True + assert captured_options["pool_size"] >= 1 + + +def test_head_revision_names_the_single_head_of_the_chain() -> None: + script = ScriptDirectory.from_config(alembic_config()) + + assert script.get_heads() == [HEAD_REVISION] + + +def test_migrations_refuse_to_run_without_the_migration_database_url( + monkeypatch, +) -> None: + monkeypatch.delenv(MIGRATION_DATABASE_URL_VARIABLE, raising=False) + + with pytest.raises(RuntimeError, match=MIGRATION_DATABASE_URL_VARIABLE): + command.upgrade(alembic_config(), "head") + + +def test_migrations_refuse_offline_sql_generation(monkeypatch) -> None: + monkeypatch.setenv( + MIGRATION_DATABASE_URL_VARIABLE, + "postgresql://simulation_migrator:secret@db.example/postgres", + ) + + with pytest.raises(RuntimeError, match="online"): + command.upgrade(alembic_config(), "head", sql=True) + + +def test_importing_the_package_registers_every_table() -> None: + package_tables = set(simulation_metadata.tables) + for module in pkgutil.iter_modules(policyengine_simulation_persistence.__path__): + importlib.import_module(f"policyengine_simulation_persistence.{module.name}") + + assert set(simulation_metadata.tables) == package_tables + assert {"simulation.simulations", "simulation.reports"} <= package_tables diff --git a/libs/policyengine-simulation-persistence/tests/test_simulations.py b/libs/policyengine-simulation-persistence/tests/test_simulations.py new file mode 100644 index 000000000..f574b8f26 --- /dev/null +++ b/libs/policyengine-simulation-persistence/tests/test_simulations.py @@ -0,0 +1,247 @@ +"""simulation.simulations: what to run, independent of model versions.""" + +from __future__ import annotations + +from uuid import uuid4 + +import pytest +from policyengine_simulation_persistence import ( + NewSimulation, + SimulationKind, + SimulationRecord, + create_simulation_engine, +) +from policyengine_simulation_persistence.simulations import simulations +from public_api_support import delete_public_api_row +from simulation_support import ( + economy_simulation, + household_simulation, + simulation_store, +) +from sqlalchemy import insert +from sqlalchemy.exc import IntegrityError + +pytestmark = pytest.mark.postgres + + +def test_an_economy_simulation_names_a_region_by_code( + migrated_database, public_api_rows +) -> None: + store = simulation_store(migrated_database) + + created = store.create( + economy_simulation(public_api_rows, dataset_name="populace_us_2024") + ) + + assert isinstance(created, SimulationRecord) + assert created.kind is SimulationKind.ECONOMY + assert created.country == "us" + assert created.policy_id == public_api_rows.us_policy_id + assert created.region_code == "state/ca" + assert created.dataset_name == "populace_us_2024" + assert created.household_id is None + assert created.created_at.tzinfo is not None + assert store.get(created.id) == created + + +def test_a_null_dataset_name_means_the_region_default( + migrated_database, public_api_rows +) -> None: + created = simulation_store(migrated_database).create( + economy_simulation(public_api_rows) + ) + + assert created.dataset_name is None + + +def test_current_law_has_no_policy(migrated_database, public_api_rows) -> None: + created = simulation_store(migrated_database).create( + economy_simulation(public_api_rows, policy_id=None) + ) + + assert created.policy_id is None + + +def test_a_household_simulation_names_its_household_and_dynamic( + migrated_database, public_api_rows +) -> None: + created = simulation_store(migrated_database).create( + household_simulation( + public_api_rows, + policy_id=public_api_rows.us_policy_id, + dynamic_id=public_api_rows.dynamic_id, + ) + ) + + assert created.kind is SimulationKind.HOUSEHOLD + assert created.household_id == public_api_rows.us_household_id + assert created.dynamic_id == public_api_rows.dynamic_id + assert created.region_code is None + + +def _mismatched_population(rows, case: str) -> NewSimulation: + household = {"kind": SimulationKind.HOUSEHOLD, "household_id": rows.us_household_id} + economy = {"kind": SimulationKind.ECONOMY, "region_code": "state/ca"} + values = { + "household-without-household": {**household, "household_id": None}, + "household-with-region": {**household, "region_code": "state/ca"}, + "household-with-dataset": {**household, "dataset_name": "populace_us_2024"}, + "economy-without-region": {**economy, "region_code": None}, + "economy-with-household": {**economy, "household_id": rows.us_household_id}, + }[case] + return NewSimulation(country="us", **values) + + +@pytest.mark.parametrize( + "case", + [ + "household-without-household", + "household-with-region", + "household-with-dataset", + "economy-without-region", + "economy-with-household", + ], +) +def test_the_population_must_match_the_kind( + migrated_database, public_api_rows, case +) -> None: + with pytest.raises(IntegrityError, match="population_matches_kind"): + simulation_store(migrated_database).create( + _mismatched_population(public_api_rows, case) + ) + + +@pytest.mark.parametrize( + ("kind", "reference"), + [ + (SimulationKind.ECONOMY, "policy"), + (SimulationKind.HOUSEHOLD, "household"), + ], +) +def test_a_policy_or_household_must_be_of_the_simulation_country( + migrated_database, public_api_rows, kind, reference +) -> None: + if reference == "policy": + new_simulation = economy_simulation( + public_api_rows, country="uk", region_code="country/wales" + ) + constraint = "fk_simulations_policy_country" + else: + new_simulation = household_simulation(public_api_rows, country="uk") + constraint = "fk_simulations_household_country" + assert new_simulation.kind is kind + + with pytest.raises(IntegrityError, match=constraint): + simulation_store(migrated_database).create(new_simulation) + + +def test_an_unknown_dynamic_is_rejected(migrated_database, public_api_rows) -> None: + with pytest.raises(IntegrityError, match="fk_simulations_dynamic"): + simulation_store(migrated_database).create( + economy_simulation(public_api_rows, dynamic_id=uuid4()) + ) + + +def _simulation_using(rows, table_name: str) -> NewSimulation: + if table_name == "households": + return household_simulation(rows) + if table_name == "dynamics": + return economy_simulation(rows, dynamic_id=rows.dynamic_id) + return economy_simulation(rows) + + +@pytest.mark.parametrize( + ("table_name", "row_attribute"), + [ + ("policies", "us_policy_id"), + ("households", "us_household_id"), + ("dynamics", "dynamic_id"), + ], +) +def test_the_public_api_cannot_delete_a_row_a_simulation_uses( + migrated_database, public_api_rows, table_name, row_attribute +) -> None: + simulation_store(migrated_database).create( + _simulation_using(public_api_rows, table_name) + ) + + with pytest.raises(IntegrityError, match="violates foreign key constraint"): + delete_public_api_row( + migrated_database.superuser_url, + table_name, + getattr(public_api_rows, row_attribute), + ) + + +def test_a_migrated_simulation_is_created_once( + migrated_database, public_api_rows +) -> None: + store = simulation_store(migrated_database) + first = store.create_migrated( + economy_simulation(public_api_rows, legacy_simulation_id=4021) + ) + + again = store.create_migrated( + economy_simulation( + public_api_rows, legacy_simulation_id=4021, region_code="state/ny" + ) + ) + + assert again == first + assert store.get_by_legacy_simulation_id(4021) == first + assert store.get_by_legacy_simulation_id(4022) is None + + +def test_a_migrated_simulation_needs_its_legacy_id( + migrated_database, public_api_rows +) -> None: + with pytest.raises(ValueError, match="legacy_simulation_id"): + simulation_store(migrated_database).create_migrated( + economy_simulation(public_api_rows) + ) + + +def test_two_simulations_cannot_share_a_legacy_id( + migrated_database, public_api_rows +) -> None: + store = simulation_store(migrated_database) + store.create(economy_simulation(public_api_rows, legacy_simulation_id=7)) + + with pytest.raises(IntegrityError, match="legacy_simulation_id"): + store.create(economy_simulation(public_api_rows, legacy_simulation_id=7)) + + +def test_an_unknown_simulation_reads_as_none(migrated_database) -> None: + assert simulation_store(migrated_database).get(uuid4()) is None + + +@pytest.mark.parametrize( + ("column", "value", "constraint"), + [ + ("kind", "microsimulation", "ck_simulations_kind"), + ("country", "USA", "ck_simulations_country"), + ("region_code", "state ca", "ck_simulations_region_code"), + ("dataset_name", "populace us", "ck_simulations_dataset_name"), + ("legacy_simulation_id", 0, "ck_simulations_legacy_simulation_id"), + ], +) +def test_the_database_checks_each_simulation_field( + migrated_database, public_api_rows, column, value, constraint +) -> None: + """The database checks what NewSimulation also checks, for any other writer.""" + values = { + "id": uuid4(), + "kind": "economy", + "country": "us", + "region_code": "state/ca", + column: value, + } + engine = create_simulation_engine(migrated_database.runtime_url) + try: + with ( + pytest.raises(IntegrityError, match=constraint), + engine.begin() as connection, + ): + connection.execute(insert(simulations).values(**values)) + finally: + engine.dispose() diff --git a/libs/policyengine-simulation-persistence/uv.lock b/libs/policyengine-simulation-persistence/uv.lock new file mode 100644 index 000000000..207031bc6 --- /dev/null +++ b/libs/policyengine-simulation-persistence/uv.lock @@ -0,0 +1,375 @@ +version = 1 +revision = 5 +requires-python = "==3.13.*" + +[[package]] +name = "alembic" +version = "1.20.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "mako" }, + { name = "sqlalchemy" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ed/aa/02910bdb8e2f1444f6654d5b296cd827d126f82209050ee7b1000f92ac4b/alembic-1.20.0.tar.gz", hash = "sha256:db505480647bc60386c5369402f4a57a506b7539c9e9ef5e270d45cbbe4939bf", size = 2093272, upload-time = "2026-09-11T19:09:11.126Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3f/27/78a89b55b0904d222183164e079b4ca56208e94eff1d35ad1f1ad5be9b06/alembic-1.20.0-py3-none-any.whl", hash = "sha256:77eb101048d95f982c0353e9233404889dcd7a6fc244c107836c0e2fc9cf7d9d", size = 268719, upload-time = "2026-09-11T19:09:12.88Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "coverage" +version = "7.16.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/2f/55/d1eaf3e73781174340a00dc1ba2aee8a65f82fadb18e2797b192b6b3925b/coverage-7.16.2.tar.gz", hash = "sha256:ca64d9f1f384f151b9511bec01126072acd2f313439f8ed015a22d8790aab6fa", size = 971999, upload-time = "2026-09-27T12:29:01.118Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f0/f6/8eb4f220ef24f84fb27d852d4f9bf83e0c73ec1a4a08dd9a87e3f4529739/coverage-7.16.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:1a37c6e478cf687e1aa30a593d19c92c02fad9d122b51ab73f51b8dc7a0c0fc9", size = 223993, upload-time = "2026-09-27T12:26:40.164Z" }, + { url = "https://files.pythonhosted.org/packages/40/23/d4bbaf0c154e0b0c2b5264890dbf6ef098dcb50ec8f2469be9490d191660/coverage-7.16.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0993d0e90858c03943d3cb152e068a20dd4707924deec84dd2230261baae3b1b", size = 224368, upload-time = "2026-09-27T12:26:41.762Z" }, + { url = "https://files.pythonhosted.org/packages/7f/48/fc1e88fd571ec5cb38150b7f89f7696ca1bdf9920e01432febb69774cc85/coverage-7.16.2-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:bb2fc905bbf4e6b7f40806ea79e31515abf6349594cdf0adf27c4215f0463204", size = 255358, upload-time = "2026-09-27T12:26:43.442Z" }, + { url = "https://files.pythonhosted.org/packages/1d/56/6785397d07c29c8e70fbb9a07e97d062b43c21ffc5f12385917847f09f63/coverage-7.16.2-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:4358b9c8c0125b460407f3017c6cce8156e904b32772c5630d27112f52bdbfe5", size = 257955, upload-time = "2026-09-27T12:26:45.725Z" }, + { url = "https://files.pythonhosted.org/packages/27/3b/c8cdd07721e5f99abd81cea970d971997f99bf158c0b85f51bd284179c8b/coverage-7.16.2-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1f15254427c9b33eedac4f198eaf9e356eb4f6214551afb43da6194a2c088ad7", size = 259183, upload-time = "2026-09-27T12:26:47.208Z" }, + { url = "https://files.pythonhosted.org/packages/9b/11/606b192fe43d32574ec6238549d48de588fdcc18485682a5ec0a8ac357f2/coverage-7.16.2-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9a75a4704ff640e46170042eec1f984385a121227c505d5a16ad8e495f452541", size = 261325, upload-time = "2026-09-27T12:26:49.084Z" }, + { url = "https://files.pythonhosted.org/packages/67/90/eea481f8b0305ceeb33f081a5f47e298391dbd1b589de0c4b3b3aa50d3f2/coverage-7.16.2-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:14253fc7bb15749b849795a06f5d3b6d8bc3fb8a4b5ddc341faf7a89dce205fc", size = 255531, upload-time = "2026-09-27T12:26:50.509Z" }, + { url = "https://files.pythonhosted.org/packages/6b/be/dedbf9aea1457b120c27ac10b8fc2a357f37fa2b54c3e7286d42980a0a2a/coverage-7.16.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:921415102a90637fcc2e3f169f61dad7699ecf690e8639fc21b813acbedc0967", size = 257329, upload-time = "2026-09-27T12:26:52.005Z" }, + { url = "https://files.pythonhosted.org/packages/fa/cb/b25c19d5bb2bd0f2e4e27fe8e2ffcae80c7a91ae181c0dc749ed60e9b1a4/coverage-7.16.2-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:cce2bc991293f15cc4084ca116827b5900c5f34e1a54dfe83f10ab5c43162eb7", size = 255294, upload-time = "2026-09-27T12:26:53.634Z" }, + { url = "https://files.pythonhosted.org/packages/5f/a2/892c5c5f4ad44b7b2ca009aee705191f3f268f15052244f2f9e3539b2e35/coverage-7.16.2-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:e1fa594c887365b69745f25a416806e61085dd07b94c9eae68a6e20730629b23", size = 259444, upload-time = "2026-09-27T12:26:55.243Z" }, + { url = "https://files.pythonhosted.org/packages/ed/99/a562537deba0a3e370182ae71c149be796c39d8087365f17a09188f27145/coverage-7.16.2-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:11e597173af1dc33d5f8a7332ada544199269a223af1ee1770ddd5e245ad0fe8", size = 255111, upload-time = "2026-09-27T12:26:56.851Z" }, + { url = "https://files.pythonhosted.org/packages/2d/20/854ec68641a9b3362ff068a32dfa41637299761617ef253791dbade6fc76/coverage-7.16.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3e7f99698ba3a7d13988bdd984b7ebf13af4dbe2166dc8502eef90d77603b0a4", size = 256882, upload-time = "2026-09-27T12:26:58.41Z" }, + { url = "https://files.pythonhosted.org/packages/db/0d/748e4518b0ac0f9ff2687c248a6e5f8c0737306e709372632a2556f84443/coverage-7.16.2-cp313-cp313-win32.whl", hash = "sha256:f80bd9f9633eafc73d0a913ba2645c96ba58bba1befc30590f7c0fbfde59d865", size = 226045, upload-time = "2026-09-27T12:26:59.983Z" }, + { url = "https://files.pythonhosted.org/packages/31/fa/6e46edba66a183fe4d99d4bb52c173287e9b8dddabe0888d24cb8210e580/coverage-7.16.2-cp313-cp313-win_amd64.whl", hash = "sha256:8be099e979fc42559328a21828281b4578304191ae46ed4e80a407048a82eee6", size = 226581, upload-time = "2026-09-27T12:27:01.494Z" }, + { url = "https://files.pythonhosted.org/packages/1b/d9/9ef6845367600b336ff75d000444a0d32497d6972c833141bd39356abf68/coverage-7.16.2-cp313-cp313-win_arm64.whl", hash = "sha256:28ff850182a67d117990fa2ce5ea1032836d8c9630dae867e8bdd3bff4533b79", size = 226409, upload-time = "2026-09-27T12:27:03.116Z" }, + { url = "https://files.pythonhosted.org/packages/3f/0c/7a64e1ac90541a8edf50daef0914848011fb057a5bf55284a4811e21939a/coverage-7.16.2-py3-none-any.whl", hash = "sha256:11d28e9123a9156cb405d8d27b44256c9a58fb5decc2073a8f17862057e3aa0f", size = 215754, upload-time = "2026-09-27T12:28:59.075Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "mako" +version = "1.4.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/5a/09/e07c4b5579a79f4b16f8d4f29f6c54514ac787c4ad506b8c4f28a0e6b0bf/mako-1.4.3.tar.gz", hash = "sha256:cd6537fe88d5fec315c55c2f8529bc4ce7a9a352ad7db3eeaa6a66e2dd4ec37a", size = 412799, upload-time = "2026-09-22T20:54:31.509Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/a0/053d6af3e8f871e0073b4a36732d9e65be77a72e5434c31b94f6af78a6bb/mako-1.4.3-py3-none-any.whl", hash = "sha256:723296007c870bfd6b3f0c3230dba7198096e5269297ebf5e4eff9e7ffa39d4f", size = 80164, upload-time = "2026-09-22T20:54:33.128Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/9b/e422a865e1d5d57d0e509b4e0bf1c1a70a7f6382c29a5aa428df994c8bc8/markupsafe-3.0.4.tar.gz", hash = "sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6", size = 153777, upload-time = "2026-10-02T23:07:22.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6d/18/4bc5ba32499e87bb2b0ef5b3a9bb9c00a131fa961ddf0be548cb550f548b/markupsafe-3.0.4-cp313-cp313-android_24_arm64_v8a.whl", hash = "sha256:de8b364c423ef0a4bad9069657d617f9a5d2b2062457a89b1fa16ee199c399c1", size = 12786, upload-time = "2026-10-02T23:05:08.709Z" }, + { url = "https://files.pythonhosted.org/packages/4e/6f/17f0c099bf25f3e31e63cc19244d9f6af861a9a4ab778c203997903cfdd0/markupsafe-3.0.4-cp313-cp313-android_24_x86_64.whl", hash = "sha256:34bdde374c5932765d7dc685c4a1d191a3207852d67e8e0a9eb6ea85156181f1", size = 12510, upload-time = "2026-10-02T23:05:09.93Z" }, + { url = "https://files.pythonhosted.org/packages/11/af/1a141081b905036ee904ec4bd945e1f70b4e1b32d33c4e59e8cf1d58b247/markupsafe-3.0.4-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:6bd9e1788e15bfcf6a9082de42e30387e7b85d211ab21e57a939bb8cfaaf8d96", size = 11583, upload-time = "2026-10-02T23:05:10.884Z" }, + { url = "https://files.pythonhosted.org/packages/e7/0a/a89385ae590232622a03e091805cff12f24fabe6c11e0e8bae096cece81c/markupsafe-3.0.4-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:5066b244f576f91afc8ee3ba029a89f99d39c79b1853fe9d39bea9f0afbec148", size = 11801, upload-time = "2026-10-02T23:05:11.913Z" }, + { url = "https://files.pythonhosted.org/packages/ed/85/ea548dc013962eb73653124bc595635fbf9e0fa41d1f181a967ccb784dfb/markupsafe-3.0.4-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:7a83aa6e4805df46fed18e989d3d16f86ef60cb50bbc8d9ce3a6be89165fbf6e", size = 11443, upload-time = "2026-10-02T23:05:12.887Z" }, + { url = "https://files.pythonhosted.org/packages/cc/72/15f2e5ec9cf2eb00d5cdfe968d94e4156a7bd7303832c3f3b2c403a36839/markupsafe-3.0.4-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:2d1b7d9308288661f56672b1b157d75fc536714d3638487bbea17b6318a78248", size = 11665, upload-time = "2026-10-02T23:05:13.829Z" }, + { url = "https://files.pythonhosted.org/packages/ca/e0/4030bea613677e333c8a2c901fd405055f657f9d06acba5b7357984b6ef7/markupsafe-3.0.4-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:73e77980c7207854f00fc4e71fb1626868d5740ab4012623d55c7a99ad122a72", size = 12061, upload-time = "2026-10-02T23:05:14.807Z" }, + { url = "https://files.pythonhosted.org/packages/f3/a5/28b76a7449eb702966b88bef599e2360b411fbb3afeee8fe560939be06ec/markupsafe-3.0.4-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7018d4af1cd272e847aa5917983ab5e83e4f6579f9dbfecd4a79c0ca80b144c2", size = 24395, upload-time = "2026-10-02T23:05:15.909Z" }, + { url = "https://files.pythonhosted.org/packages/07/6c/21232811afc3a063b5e934b1ae2efda52f46154ec382f585149c020e61fe/markupsafe-3.0.4-cp313-cp313-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:c90d5b3d4e944e065a301d741b3c1d784f6bd1f503aa68b4967e32b2ba313d85", size = 27183, upload-time = "2026-10-02T23:05:16.976Z" }, + { url = "https://files.pythonhosted.org/packages/14/38/6ccdfa5b59049cb36fb80cbc80aee9cf1fc9bb77d1335ad435f2070b08cf/markupsafe-3.0.4-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:18a801868a884f216e784d7d14db2a4077143ce7610440aee2ce8f734e7cfcde", size = 24772, upload-time = "2026-10-02T23:05:18.209Z" }, + { url = "https://files.pythonhosted.org/packages/63/e0/cec6865dfe88cb48fedd4b20aed6af5158e41092adcbf3e028bcc6ec2108/markupsafe-3.0.4-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:434139499bb20b502ed3baa1f169e618f924a97e7a777fea1a49446d80106cf6", size = 23017, upload-time = "2026-10-02T23:05:19.286Z" }, + { url = "https://files.pythonhosted.org/packages/ee/76/6ed4940bb7648a9aac457c14f870cfdd5105f139a0fb1f29cd61fafa47d1/markupsafe-3.0.4-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:9e227f3dbe6bde7491cf0a9965d00b88c6b1a4a95d11480ddf88bb96d397c19f", size = 22036, upload-time = "2026-10-02T23:05:20.352Z" }, + { url = "https://files.pythonhosted.org/packages/a1/4f/ed476226d4fe46a09090a36025bf319296810028df55eb12f1253b540f3a/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b8cd1f918b26fd7b1832ece557cc18f2d8747309ff8b3f0ef9d4250c5ad67a39", size = 23786, upload-time = "2026-10-02T23:05:21.576Z" }, + { url = "https://files.pythonhosted.org/packages/9a/35/66ff30450e35ef5fba9ebc930c9411747e537fd9447b65e44f5007e2b84d/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:a5fcffb37e602b0b3c1638a97746b9b96125caa9bcf6fa41d337a9261de231ee", size = 22672, upload-time = "2026-10-02T23:05:22.922Z" }, + { url = "https://files.pythonhosted.org/packages/32/0b/72f45ce4b4efcbca4b80cf1b06703eff0be8d37e82abb78f66c85a7ead1e/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:5989cb26b2e1efc6a42216a9f6b5ee495ce5ace2e5b352a9af489976b32d1ee2", size = 24107, upload-time = "2026-10-02T23:05:24.175Z" }, + { url = "https://files.pythonhosted.org/packages/d2/03/71776e5fdcba04614b384cc102e8a4198208579d896fd1394cb7cb9aa900/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:add96447a86d205ab616665d53b2950ee81083757f56e6ea833c8b2917646b46", size = 21615, upload-time = "2026-10-02T23:05:25.215Z" }, + { url = "https://files.pythonhosted.org/packages/ab/5f/801ce02a02e7aee0f784b1ec7843026178f6adeb9c93ac67eb1992a9a84d/markupsafe-3.0.4-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:2628d3a8cb648ecebb3c5d6b0a1052d400e4d8b7ac0fb786be8d285b50040d17", size = 23076, upload-time = "2026-10-02T23:05:26.423Z" }, + { url = "https://files.pythonhosted.org/packages/4a/85/c43776625428f3bb4a61e8633940400e3efe6409e3c6f5bff26de5e45618/markupsafe-3.0.4-cp313-cp313-win32.whl", hash = "sha256:672d207103e6b16ca098611b0f9efad6bc00afd47c03d6ef62186495ca677dc0", size = 14118, upload-time = "2026-10-02T23:05:27.716Z" }, + { url = "https://files.pythonhosted.org/packages/6f/36/163da64de88a13db79214ef75fa041be7fa13bdb42261cf5b7484de14bfb/markupsafe-3.0.4-cp313-cp313-win_amd64.whl", hash = "sha256:1f1f9477e174582b0a1b583d60b66e1f2cf5d3fe12cee985e4aedf44766600e5", size = 14313, upload-time = "2026-10-02T23:05:28.749Z" }, + { url = "https://files.pythonhosted.org/packages/9f/a8/9b662783ffaa1149221432a923cee562f78b9cbbb8baa3df9b3753e63e1e/markupsafe-3.0.4-cp313-cp313-win_arm64.whl", hash = "sha256:06de8ef6331f6e822c28d577dc8bf43fe398800477c49498f38fc38b67ff33fc", size = 14189, upload-time = "2026-10-02T23:05:29.917Z" }, +] + +[[package]] +name = "nodeenv" +version = "1.11.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/105de02c1322cfada6d9710d9146ef8026419d433c9d08359a2d35805811/nodeenv-1.11.0.tar.gz", hash = "sha256:3ce8fe5b71d16e8af7039ca65257354100bc772965d6bc549070649e53b1b146", size = 99301, upload-time = "2026-09-26T11:29:21.367Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/54/c8/12811c9b48fde162bb72b6f2e78fada9a247a09d7bb5be2050a5d099c77b/nodeenv-1.11.0-py2.py3-none-any.whl", hash = "sha256:edaa16e6c14d7cf395d75d4bbd5a26390f4dc06501a33b4e76282b02cc688a25", size = 34277, upload-time = "2026-09-26T11:29:19.933Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "policyengine-simulation-persistence" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "psycopg", extra = ["binary"] }, + { name = "sqlalchemy" }, +] + +[package.optional-dependencies] +build = [ + { name = "pyright" }, +] +migrations = [ + { name = "alembic" }, +] +provisioning = [ + { name = "pydantic" }, +] +test = [ + { name = "alembic" }, + { name = "pydantic" }, + { name = "pytest" }, + { name = "pytest-cov" }, + { name = "pyyaml" }, +] + +[package.metadata] +requires-dist = [ + { name = "alembic", marker = "extra == 'migrations'", specifier = ">=1.17.1,<2" }, + { name = "alembic", marker = "extra == 'test'", specifier = ">=1.17.1,<2" }, + { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, + { name = "pydantic", marker = "extra == 'provisioning'", specifier = ">=2.11,<3" }, + { name = "pydantic", marker = "extra == 'test'", specifier = ">=2.11,<3" }, + { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, + { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, + { name = "pyyaml", marker = "extra == 'test'", specifier = ">=6.0.2" }, + { name = "sqlalchemy", specifier = ">=2,<3" }, +] +provides-extras = ["migrations", "provisioning", "test", "build"] + +[[package]] +name = "psycopg" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/76/26/3ea4ca5eaea1c0debcdf7ee7c1613fbe721dc27a03c461c0817ffd8a0601/psycopg-3.3.6.tar.gz", hash = "sha256:c081f2250df751a943036e42db6df4571c66cd0aabe8291a7a506512b12007d2", size = 168171, upload-time = "2026-09-18T13:22:55.152Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/de/748bd7609c71cae5d737f0ba9192f19329f70180ecda8fff3cac02c5abe3/psycopg-3.3.6-py3-none-any.whl", hash = "sha256:a1db9f7148b06a28606767efaca51fa6f9398c5c0a3810519be69d7000bdb631", size = 215490, upload-time = "2026-09-18T13:15:29.374Z" }, +] + +[package.optional-dependencies] +binary = [ + { name = "psycopg-binary", marker = "implementation_name != 'pypy'" }, +] + +[[package]] +name = "psycopg-binary" +version = "3.3.6" +source = { registry = "https://pypi.org/simple" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/c3/c072584b69ad44a747b448cfc9766fecb8aae56e372a017e2ef668790057/psycopg_binary-3.3.6-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5ad8f35e67cc16d1fad1fa8c88972dc9b3a3141ea67897399904edab96a301b6", size = 4712284, upload-time = "2026-09-18T13:19:13.451Z" }, + { url = "https://files.pythonhosted.org/packages/0a/b9/4283b785339e8e2318d03048994b093d650ea6289fabaa806b765dc0d449/psycopg_binary-3.3.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:373704aea331d3f3e3402c125a1543f5875e2986ebb54f97d1647942161f803f", size = 4772031, upload-time = "2026-09-18T13:19:18.524Z" }, + { url = "https://files.pythonhosted.org/packages/6f/72/7a1321d359246769fff1affffbd0132785a28f7f63c18524c15a502398f4/psycopg_binary-3.3.6-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:b82491019b884d62318b5f30706c3d7e6d4e5a6cb7eabcb3edc0c1b0fdaceae9", size = 5556392, upload-time = "2026-09-18T13:19:24.418Z" }, + { url = "https://files.pythonhosted.org/packages/de/b0/c6f8a0585a5dacbea74e130bcfc66629390e8f5bbc79d2a8e806e8952150/psycopg_binary-3.3.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:cec5ea900390897d0b46130f60bc2883bf19c314f9044235217c8be88b0ef269", size = 5237855, upload-time = "2026-09-18T13:19:31.257Z" }, + { url = "https://files.pythonhosted.org/packages/e2/fc/c3a7a8bbef7e945ec584ac61d460a612363ea398511cd0e220242b1d69f1/psycopg_binary-3.3.6-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:98c02090d88f2ebc0ec1e8da538f77d225ce0fffecf372aa39262e62a1b054ef", size = 6833856, upload-time = "2026-09-18T13:19:43.622Z" }, + { url = "https://files.pythonhosted.org/packages/a9/f2/8e80b921db728ebb68fc105bd7c4277f908210ad755bd6481d5ea7add740/psycopg_binary-3.3.6-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ee2c4728c691245e24501fcd7a97b5b381236b9985bc445bba88cdce7d1b5784", size = 5070730, upload-time = "2026-09-18T13:19:49.968Z" }, + { url = "https://files.pythonhosted.org/packages/54/6a/5b313e0c5348244f0e973aff3258bf86766656256d5ece8d541a53e35b4a/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f19cc87343eaa55255e76b31259a570072ac95d6ae82c92dd34b97691f5e49dc", size = 4598089, upload-time = "2026-09-18T13:19:56.426Z" }, + { url = "https://files.pythonhosted.org/packages/32/e9/db7f76ec24bf6699e92bf604e5c4bae10664a681a8999ef42aa0faf0f2c6/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:fdccb3a0e184b03e9baa673b15a809cf36c339c85dbda0ebc25a698846dfbee8", size = 4278481, upload-time = "2026-09-18T13:20:04.681Z" }, + { url = "https://files.pythonhosted.org/packages/61/83/72c67013656f4d6b547caabffb193e91d57e63f90eefdcc6d045c400e97d/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:9892188bb15e5803beb51afe8a25add6b56be391a53058e8bca03b74e1e6bf22", size = 4009229, upload-time = "2026-09-18T13:20:11.905Z" }, + { url = "https://files.pythonhosted.org/packages/82/35/5e4500df2c999eb0faed8b184e6958b834172128274f06167a5deef4c19c/psycopg_binary-3.3.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:3af90f92769d8cc10f94515ee7a0aef36ea85ca733a0ce22858f6e0953f41138", size = 4321467, upload-time = "2026-09-18T13:20:17.949Z" }, + { url = "https://files.pythonhosted.org/packages/55/7f/e350e1cf498ba2565c3f87b12f429d2012eb86b76c2b3845a19ee5fbb4d6/psycopg_binary-3.3.6-cp313-cp313-win_amd64.whl", hash = "sha256:0ebfad5d131de9f892ae9e70cc7616207768b6714b66a52d4612b8ceaf78b372", size = 3658179, upload-time = "2026-09-18T13:20:22.691Z" }, +] + +[[package]] +name = "pydantic" +version = "2.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7c/0b/8e10b2e693af8ec54346a14caf36221334775975a747c9ceaa3f8371d96d/pydantic-2.14.1.tar.gz", hash = "sha256:94f478203dd03404682a1ada216965651dd74b1d2d5ffd62e00e0837caab5c26", size = 900362, upload-time = "2026-10-11T18:37:55.396Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/ea/a56b9fe5066f3537b7882f77e9c5dfb26d8c2eefdaed9b5fc73d57b4dc22/pydantic-2.14.1-py3-none-any.whl", hash = "sha256:9195d967ec791692a04438115466764fb8b9a27b31f14a760437694f40d6b454", size = 488106, upload-time = "2026-10-11T18:37:53.437Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.50.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/24/af4ec4be49fbc810f35b0bdcacc3d433b56bbfa469fd3bfd4ae116cd9bf1/pydantic_core-2.50.1.tar.gz", hash = "sha256:e50d7b94baac6c7d09927fa5ca5800a0c7ee5015c7fcff65beb3a1931b5a6e09", size = 513515, upload-time = "2026-10-11T18:35:44.82Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ae/57/0e237d7091d2cd44a35d243b7344227f90440166860469cd036afc23a04d/pydantic_core-2.50.1-cp313-cp313-macosx_10_12_x86_64.whl", hash = "sha256:d5e062c01286d861fd6a1c4ff6e063547b3e713067f2df033c0ff97ac2ca006b", size = 2222774, upload-time = "2026-10-11T18:32:43.103Z" }, + { url = "https://files.pythonhosted.org/packages/f7/b9/c720e56858d4e1539503297ed37063e0c08e0f3541c41b777f6a800f75fa/pydantic_core-2.50.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0c003c3b7f49debb893d2d85ae099ac5959c9839e2f330fadb1fcdf7a6594482", size = 2024738, upload-time = "2026-10-11T18:32:44.587Z" }, + { url = "https://files.pythonhosted.org/packages/4d/b8/fbfc25875219cc060e613170ff10e850c6d8924beb4910da57c2ee3ba1d2/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:409e0ea40ec30d9158f33574fd758e689f6045a0f2596701828c27816ca9687d", size = 2047049, upload-time = "2026-10-11T18:32:46.164Z" }, + { url = "https://files.pythonhosted.org/packages/d7/43/34210124d504c553688f2f04b48500b131237528ac545b445e0d6d30e0d5/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:131059670f1d2444269b8585cb888963994871932447c08b39ac6a51fcfef658", size = 2173415, upload-time = "2026-10-11T18:32:47.722Z" }, + { url = "https://files.pythonhosted.org/packages/65/cf/6e178e8fdc11da5965bef980983bf46326a42f436871dd51ba0a57f39df1/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6dbcbee53bf17196a7f745aa9bf5a9603953a1e365b1f020be3207c676a3e7c4", size = 2361719, upload-time = "2026-10-11T18:32:49.217Z" }, + { url = "https://files.pythonhosted.org/packages/11/14/bd5169d356aa91bf777e28ba0b281c192d286d03c2812c9c9db023a2f00e/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:325c23f3e35cfbf0fe3486fa5f7260d1e45885173002d30a28ca019994124255", size = 2439803, upload-time = "2026-10-11T18:32:51.025Z" }, + { url = "https://files.pythonhosted.org/packages/1c/bc/d79d000e5203ebef39af839f2ce77a777fcad6e08ad26af9a2fcd114ffc8/pydantic_core-2.50.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17e722e156d0444ecaefbe640bdb60928752bf2013e2b7a11cdb099aaae19bec", size = 2158402, upload-time = "2026-10-11T18:32:52.714Z" }, + { url = "https://files.pythonhosted.org/packages/1b/a5/4902cb5fd599422c130bd3124ab31ee8b771199bd56662702eed07d3fec7/pydantic_core-2.50.1-cp313-cp313-manylinux_2_31_riscv64.whl", hash = "sha256:aa8224f10880d9bf1b5993988ba153d42a8b4f3f4f511f93b1f09c93ff613c72", size = 2229755, upload-time = "2026-10-11T18:32:54.126Z" }, + { url = "https://files.pythonhosted.org/packages/1a/f5/c1481f8669f6060d89110c9b1374173fc8767ca276b84f4870bd8acd5e3f/pydantic_core-2.50.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:41bc8237121bd8dc8d888dfd6279fc166ffc88c1f1bf3a8bf00869680533ca4c", size = 2242303, upload-time = "2026-10-11T18:32:55.641Z" }, + { url = "https://files.pythonhosted.org/packages/04/f9/77fc3c7653ba9b6e42049e17e96b25388187d4a7c274ab1cb07f58ac8419/pydantic_core-2.50.1-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:45c6266d071c241f2a168d45bf8c54344f0effce35e7e6b73afdec11f3687568", size = 2278345, upload-time = "2026-10-11T18:32:57.38Z" }, + { url = "https://files.pythonhosted.org/packages/95/9b/0579c5d12e7f2b16b27e6782427987341fcce07b0725e02ddb0b74add0c4/pydantic_core-2.50.1-cp313-cp313-musllinux_1_1_armv7l.whl", hash = "sha256:1deeacb112d14d3f4fcb16b165f7dbaf76c70ba6e82f37ba042bdab51970a0b8", size = 2449947, upload-time = "2026-10-11T18:32:58.896Z" }, + { url = "https://files.pythonhosted.org/packages/a8/ac/1b677db91eba54cc5922f4de6edc46ba45c2bd712dfdc0130382d158e214/pydantic_core-2.50.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:1c96fd793b73d1b92e65570132505498fe7b21eaef73cdf74e67e5dfba7ac9e4", size = 2476889, upload-time = "2026-10-11T18:33:00.665Z" }, + { url = "https://files.pythonhosted.org/packages/4d/2a/3a9f6624ee3ea9ccba5249dde11418bb4c35780a7a92608f8768bd3fea39/pydantic_core-2.50.1-cp313-cp313-win32.whl", hash = "sha256:06ead20d39ffd6f2f6f2a8f8a6de67ff8bb1b4f14a8a30e058502514ee2ac685", size = 2101118, upload-time = "2026-10-11T18:33:02.329Z" }, + { url = "https://files.pythonhosted.org/packages/2d/1f/323f78ddd9d9938aac420c1abb4e8ba799fc8bdab0acc67c0593b837c979/pydantic_core-2.50.1-cp313-cp313-win_amd64.whl", hash = "sha256:7816e98acc08119dc0f340ab167048ecc54126316330c1f0caf7c6756c88e28f", size = 2170840, upload-time = "2026-10-11T18:33:03.919Z" }, + { url = "https://files.pythonhosted.org/packages/cb/09/8497c52a739ae425c3ac2f7f56414cbc711c67d374346174f40fe2062644/pydantic_core-2.50.1-cp313-cp313-win_arm64.whl", hash = "sha256:c17799a62c142d61b8a3c51752a7cbc87fe2ad4ccfab10e628a77b405075c662", size = 2161633, upload-time = "2026-10-11T18:33:05.518Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pyright" +version = "1.1.414" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "nodeenv" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e1/1b/244c7b710031ada80f27e579ec20d28a2285dfc318fed0339866b1047f12/pyright-1.1.414.tar.gz", hash = "sha256:523c0a97c60da6333234955c277730c9cf4f5bd6d5399e7b7d2b0fc5d3599524", size = 4154638, upload-time = "2026-09-10T12:26:53.181Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/ba/18b6e682ead424ad24bcc134339ae5d1b931cd9ae260540592a058a91279/pyright-1.1.414-py3-none-any.whl", hash = "sha256:2a6b4b3298c9eec174c5ed83bd338de6eee82df2992f3e1930e6199d381be36f", size = 6225049, upload-time = "2026-09-10T12:26:51.427Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pytest-cov" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "coverage" }, + { name = "pluggy" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/51/a849f96e117386044471c8ec2bd6cfebacda285da9525c9106aeb28da671/pytest_cov-7.1.0.tar.gz", hash = "sha256:30674f2b5f6351aa09702a9c8c364f6a01c27aae0c1366ae8016160d1efc56b2", size = 55592, upload-time = "2026-03-21T20:11:16.284Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/7a/d968e294073affff457b041c2be9868a40c1c71f4a35fcc1e45e5493067b/pytest_cov-7.1.0-py3-none-any.whl", hash = "sha256:a0461110b7865f9a271aa1b51e516c9a95de9d696734a2f71e3e78f46e1d4678", size = 22876, upload-time = "2026-03-21T20:11:14.438Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/11/0fd08f8192109f7169db964b5707a2f1e8b745d4e239b784a5a1dd80d1db/pyyaml-6.0.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:8da9669d359f02c0b91ccc01cac4a67f16afec0dac22c2ad09f46bee0697eba8", size = 181669, upload-time = "2025-09-25T21:32:23.673Z" }, + { url = "https://files.pythonhosted.org/packages/b1/16/95309993f1d3748cd644e02e38b75d50cbc0d9561d21f390a76242ce073f/pyyaml-6.0.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2283a07e2c21a2aa78d9c4442724ec1eb15f5e42a723b99cb3d822d48f5f7ad1", size = 173252, upload-time = "2025-09-25T21:32:25.149Z" }, + { url = "https://files.pythonhosted.org/packages/50/31/b20f376d3f810b9b2371e72ef5adb33879b25edb7a6d072cb7ca0c486398/pyyaml-6.0.3-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ee2922902c45ae8ccada2c5b501ab86c36525b883eff4255313a253a3160861c", size = 767081, upload-time = "2025-09-25T21:32:26.575Z" }, + { url = "https://files.pythonhosted.org/packages/49/1e/a55ca81e949270d5d4432fbbd19dfea5321eda7c41a849d443dc92fd1ff7/pyyaml-6.0.3-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:a33284e20b78bd4a18c8c2282d549d10bc8408a2a7ff57653c0cf0b9be0afce5", size = 841159, upload-time = "2025-09-25T21:32:27.727Z" }, + { url = "https://files.pythonhosted.org/packages/74/27/e5b8f34d02d9995b80abcef563ea1f8b56d20134d8f4e5e81733b1feceb2/pyyaml-6.0.3-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0f29edc409a6392443abf94b9cf89ce99889a1dd5376d94316ae5145dfedd5d6", size = 801626, upload-time = "2025-09-25T21:32:28.878Z" }, + { url = "https://files.pythonhosted.org/packages/f9/11/ba845c23988798f40e52ba45f34849aa8a1f2d4af4b798588010792ebad6/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:f7057c9a337546edc7973c0d3ba84ddcdf0daa14533c2065749c9075001090e6", size = 753613, upload-time = "2025-09-25T21:32:30.178Z" }, + { url = "https://files.pythonhosted.org/packages/3d/e0/7966e1a7bfc0a45bf0a7fb6b98ea03fc9b8d84fa7f2229e9659680b69ee3/pyyaml-6.0.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:eda16858a3cab07b80edaf74336ece1f986ba330fdb8ee0d6c0d68fe82bc96be", size = 794115, upload-time = "2025-09-25T21:32:31.353Z" }, + { url = "https://files.pythonhosted.org/packages/de/94/980b50a6531b3019e45ddeada0626d45fa85cbe22300844a7983285bed3b/pyyaml-6.0.3-cp313-cp313-win32.whl", hash = "sha256:d0eae10f8159e8fdad514efdc92d74fd8d682c933a6dd088030f3834bc8e6b26", size = 137427, upload-time = "2025-09-25T21:32:32.58Z" }, + { url = "https://files.pythonhosted.org/packages/97/c9/39d5b874e8b28845e4ec2202b5da735d0199dbe5b8fb85f91398814a9a46/pyyaml-6.0.3-cp313-cp313-win_amd64.whl", hash = "sha256:79005a0d97d5ddabfeeea4cf676af11e647e41d81c9a7722a193022accdb6b7c", size = 154090, upload-time = "2025-09-25T21:32:33.659Z" }, + { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, +] + +[[package]] +name = "sqlalchemy" +version = "2.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/36/dc/9addbf490110b5727380059ee542ad8353bdcee45a37ba6a5e0976966ae0/sqlalchemy-2.1.6.tar.gz", hash = "sha256:499eae5c9051efe96cb9a73ec2fbde8ffa4186f102b9094ebab8fcc9d3b99f52", size = 10589700, upload-time = "2026-10-21T16:25:06.497Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/db/ecf1cf1f5c8cb7661081dcb0561633c174213d7af5fccc17ab48cbe9812a/sqlalchemy-2.1.6-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:0534d3f41295a74d9fe2e0838e06ce6329a0855b2965df5f5d1c4d3f6f1406d0", size = 2468626, upload-time = "2026-10-21T18:49:33.602Z" }, + { url = "https://files.pythonhosted.org/packages/9b/98/1cccefb4fc94f4efd621604063acbbea4e3d530c27151fd7249d68adabf2/sqlalchemy-2.1.6-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:650f97996a4c529f4930969f3ba11bcdb57ff160ed1cb87ea0174393bf4f5f73", size = 4602545, upload-time = "2026-10-21T18:54:18.766Z" }, + { url = "https://files.pythonhosted.org/packages/d3/b0/1817d7406ed92c8647a1d1db965a1bc2dd5a3f80d18e4d28ad62daf9bd72/sqlalchemy-2.1.6-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0bada6bd29ae58a0581bf6977c0a43dfd67388fd09129f1475c37eca370cccd8", size = 4655491, upload-time = "2026-10-21T19:05:20.434Z" }, + { url = "https://files.pythonhosted.org/packages/41/5c/0bdaa1676fb182b93e3c87c09cafd08a734ef29820aac6312b87a88f8537/sqlalchemy-2.1.6-cp313-cp313-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:96f5fcdfb7e89b6d895e2ddc165f67b3126a585dfdf2a95b0c1f3fcf3380d7d4", size = 4319667, upload-time = "2026-10-21T19:23:16.637Z" }, + { url = "https://files.pythonhosted.org/packages/c1/d7/319e5a049b6f5235d1229858972abb05b8eae2fdd6639dc7bdbbc9cdb68d/sqlalchemy-2.1.6-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:015bd7b0687b5feaecb9988bc2f36ecb39daca83bd806b094b43b19ea503b0a9", size = 4525975, upload-time = "2026-10-21T18:54:21.553Z" }, + { url = "https://files.pythonhosted.org/packages/19/3b/6e8e5c14a3bcfae59a081a27f33e4f225df8140daab8e90fc3c42afd0d0e/sqlalchemy-2.1.6-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:bf883ae772971bc6155cb64cdfb9e31c50a2099f3b7c8b51b0401bf608a9fe8e", size = 4320900, upload-time = "2026-10-21T19:23:18.835Z" }, + { url = "https://files.pythonhosted.org/packages/b8/a7/7ad220d5ea6308a9a32395785f4ce8beff4d00b0ae56ab9f2dafcca28743/sqlalchemy-2.1.6-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:1ff897d13de613768123dee68474956a51fe641c42bba7af6cb7776716a8dbcb", size = 4616756, upload-time = "2026-10-21T19:05:22.265Z" }, + { url = "https://files.pythonhosted.org/packages/08/81/53a198f5a29f247adc68c0c6da69bd2c7bdd89d3ffc58f9e626ee3160f18/sqlalchemy-2.1.6-cp313-cp313-win32.whl", hash = "sha256:9db8bacb2a69b4b5388bc1439463170af28f4cc710933085b81d1658d931d2c2", size = 2382042, upload-time = "2026-10-21T17:11:33.968Z" }, + { url = "https://files.pythonhosted.org/packages/b4/87/69e86a23cd9ccef0d654c09dc69dfea6d62b1311d2f13ca3daab1bcd922e/sqlalchemy-2.1.6-cp313-cp313-win_amd64.whl", hash = "sha256:b9f0b92b4a88b630f1e67df816b43cb017bb583479917dd100541d4f490a9517", size = 2432597, upload-time = "2026-10-21T17:11:35.514Z" }, + { url = "https://files.pythonhosted.org/packages/98/90/a7f81395135cc5da724403ece4340f7e04c1ce752da15d0ad337591ec9c9/sqlalchemy-2.1.6-cp313-cp313-win_arm64.whl", hash = "sha256:bc495d00f235c3df6f376187a4ef48a56e162fdcc0a113ee34518332aa1aa2df", size = 2393224, upload-time = "2026-10-21T18:54:18.405Z" }, + { url = "https://files.pythonhosted.org/packages/c1/69/c75534e336b06b3fd3f76de73d89736510800e60c77c58d4e4ce18389018/sqlalchemy-2.1.6-py3-none-any.whl", hash = "sha256:106878cf3795cd3d357e2e858609f9e44f20f19b2d8e48a3532bef3f90b03fbe", size = 2063055, upload-time = "2026-10-21T18:42:10.234Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/68/f1b440335057bfce71b6e50a9d09445aa2ecbd08359a337976627b8409e7/tzdata-2026.5.tar.gz", hash = "sha256:8cc73c0a0bfca7dbfa59235d60b2eff82231dee33f53d206db1acd9173cfc0a7", size = 200404, upload-time = "2026-10-03T09:23:14.143Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/21/1e5995a1c920cce14e4bffae20c665ec10e7ed03ab25e006cd741092b718/tzdata-2026.5-py2.py3-none-any.whl", hash = "sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac", size = 347996, upload-time = "2026-10-03T09:23:12.535Z" }, +] diff --git a/migrations/alembic.ini b/migrations/alembic.ini new file mode 100644 index 000000000..66aaf1189 --- /dev/null +++ b/migrations/alembic.ini @@ -0,0 +1,44 @@ +# The Simulation Entrypoint's only Alembic chain: the `simulation` schema. +# Run it from the persistence library's environment, for example: +# uv run --frozen --project libs/policyengine-simulation-persistence --extra migrations \ +# alembic -c migrations/alembic.ini upgrade head +# The database URL comes only from SIMULATION_MIGRATION_DATABASE_URL. +# Ownership rules: libs/policyengine-simulation-persistence/README.md + +[alembic] +script_location = %(here)s +path_separator = os +file_template = %%(rev)s_%%(slug)s + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 000000000..b024e1d89 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,113 @@ +"""Alembic environment for the Simulation Entrypoint's ``simulation`` schema.""" + +from __future__ import annotations + +import os +from logging.config import fileConfig +from typing import TYPE_CHECKING + +from alembic import context +from policyengine_simulation_persistence import ( + ALEMBIC_VERSION_TABLE, + SIMULATION_SCHEMA, + normalized_database_url, + simulation_metadata, +) +from sqlalchemy import create_engine, inspect, pool, text +from sqlalchemy.schema import CreateSchema + +if TYPE_CHECKING: + from sqlalchemy import Connection + +DATABASE_URL_VARIABLE = "SIMULATION_MIGRATION_DATABASE_URL" +CONNECT_TIMEOUT_SECONDS = 10 +# How long a migration statement waits for a lock before the migration fails. +# Adding a foreign key locks the Public API table it refers to, and the Public +# API's writes queue behind a migration that waits, so it must not wait long. +LOCK_TIMEOUT = "10s" + +config = context.config +if config.config_file_name is not None: + fileConfig(config.config_file_name, disable_existing_loggers=False) + + +def _include_name(name: str | None, type_: str, _parent_names: object) -> bool: + """Compare only the simulation schema, never other services' schemas.""" + if type_ == "schema": + return name == SIMULATION_SCHEMA + return True + + +def _include_object( + _object: object, + name: str | None, + type_: str, + _reflected: object, + _compare_to: object, +) -> bool: + """Leave Alembic's own version table out of the schema comparison.""" + return not (type_ == "table" and name == ALEMBIC_VERSION_TABLE) + + +def _ensure_simulation_schema(connection: Connection) -> None: + """Create the simulation schema, owned by the migrator, if it is missing. + + PostgreSQL checks CREATE on the database before it checks whether a schema + exists, even for CREATE SCHEMA IF NOT EXISTS. Creating the schema only when + it is missing lets an operator create it in advance, owned by the migrator, + instead of granting the migrator CREATE on the database. + """ + if not inspect(connection).has_schema(SIMULATION_SCHEMA): + connection.execute(CreateSchema(SIMULATION_SCHEMA)) + + +def _run(connection: Connection) -> None: + connection.execute(text(f"SET LOCAL lock_timeout = '{LOCK_TIMEOUT}'")) + # The version table lives in the simulation schema, so the schema must + # exist before Alembic looks for it. + _ensure_simulation_schema(connection) + context.configure( + connection=connection, + target_metadata=simulation_metadata, + version_table=ALEMBIC_VERSION_TABLE, + version_table_schema=SIMULATION_SCHEMA, + include_schemas=True, + include_name=_include_name, + include_object=_include_object, + compare_type=True, + compare_server_default=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + """Apply migrations to the database named by the migration URL.""" + database_url = os.environ.get(DATABASE_URL_VARIABLE, "") + if not database_url: + msg = f"{DATABASE_URL_VARIABLE} must name the simulation database" + raise RuntimeError(msg) + engine = create_engine( + normalized_database_url(database_url), + poolclass=pool.NullPool, + connect_args={ + "connect_timeout": CONNECT_TIMEOUT_SECONDS, + "prepare_threshold": None, + }, + ) + try: + # One transaction covers the schema creation and every revision, so a + # failed migration leaves nothing half-applied. + with engine.begin() as connection: + _run(connection) + finally: + engine.dispose() + + +if context.is_offline_mode(): + msg = ( + "Simulation migrations run only online, against the database they " + "change; offline SQL generation is not supported" + ) + raise RuntimeError(msg) +run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 000000000..a56f67cc9 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +${imports if imports else ""} +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} +branch_labels = ${repr(branch_labels)} +depends_on = ${repr(depends_on)} + + +def upgrade() -> None: + """Apply this revision.""" + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + """Revert this revision.""" + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/0001_initial_simulation_schema.py b/migrations/versions/0001_initial_simulation_schema.py new file mode 100644 index 000000000..038498dd3 --- /dev/null +++ b/migrations/versions/0001_initial_simulation_schema.py @@ -0,0 +1,52 @@ +"""Grant the Simulation Entrypoint runtime role access to the simulation schema. + +The environment creates the ``simulation`` schema, owned by the migrator, and +Alembic creates ``simulation.alembic_version`` before this revision runs. This +revision lets the runtime role use the schema, read the applied revision for +the readiness check, and read and write every table the migrator creates in +later revisions. The runtime role never owns or changes schema objects. + +Revision ID: 0001 +Revises: +Create Date: 2026-10-22 +""" + +from __future__ import annotations + +from alembic import op + +revision = "0001" +down_revision = None +branch_labels = None +depends_on = None + +# The statements name the schema and role literally: a migration is a fixed +# historical record, so later renames must not change what it did. +UPGRADE_STATEMENTS = ( + "GRANT USAGE ON SCHEMA simulation TO simulation_runtime", + "GRANT SELECT ON simulation.alembic_version TO simulation_runtime", + "ALTER DEFAULT PRIVILEGES IN SCHEMA simulation " + "GRANT SELECT, INSERT, UPDATE ON TABLES TO simulation_runtime", + "ALTER DEFAULT PRIVILEGES IN SCHEMA simulation " + "GRANT USAGE, SELECT ON SEQUENCES TO simulation_runtime", +) +DOWNGRADE_STATEMENTS = ( + "ALTER DEFAULT PRIVILEGES IN SCHEMA simulation " + "REVOKE USAGE, SELECT ON SEQUENCES FROM simulation_runtime", + "ALTER DEFAULT PRIVILEGES IN SCHEMA simulation " + "REVOKE SELECT, INSERT, UPDATE ON TABLES FROM simulation_runtime", + "REVOKE SELECT ON simulation.alembic_version FROM simulation_runtime", + "REVOKE USAGE ON SCHEMA simulation FROM simulation_runtime", +) + + +def upgrade() -> None: + """Grant schema usage, revision reads and default table privileges.""" + for statement in UPGRADE_STATEMENTS: + op.execute(statement) + + +def downgrade() -> None: + """Revoke everything ``upgrade`` granted.""" + for statement in DOWNGRADE_STATEMENTS: + op.execute(statement) diff --git a/migrations/versions/0002_simulations_and_reports.py b/migrations/versions/0002_simulations_and_reports.py new file mode 100644 index 000000000..b3fd790de --- /dev/null +++ b/migrations/versions/0002_simulations_and_reports.py @@ -0,0 +1,196 @@ +"""Add the simulation and report definition tables. + +``simulations`` records what each simulation runs, independent of model +versions: its kind, country, policy (null for current law), dynamic, and either +a household or a region code with an optional dataset name. Foreign keys into +the Public API's policies, households and dynamics in the same database are +``ON DELETE RESTRICT`` and need ``REFERENCES`` on those tables for the migrator +role. ``reports`` records which simulations each report compares, of the +report's own country and kind, with its year and options. Definitions never +change, so the runtime role may read and insert them but not update them. +Nothing writes the tables yet. + +Revision ID: 0002 +Revises: 0001 +Create Date: 2026-10-22 +""" + +from __future__ import annotations + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +revision = "0002" +down_revision = "0001" +branch_labels = None +depends_on = None + +SCHEMA = "simulation" +PUBLIC_API_SCHEMA = "public" + + +def _public_api(table_and_column: str) -> str: + return f"{PUBLIC_API_SCHEMA}.{table_and_column}" + + +def _create_simulations() -> None: + op.create_table( + "simulations", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("kind", sa.Text(), nullable=False), + sa.Column("country", sa.Text(), nullable=False), + sa.Column("policy_id", sa.Uuid(), nullable=True), + sa.Column("dynamic_id", sa.Uuid(), nullable=True), + sa.Column("household_id", sa.Uuid(), nullable=True), + sa.Column("region_code", sa.String(length=255), nullable=True), + sa.Column("dataset_name", sa.String(length=255), nullable=True), + sa.Column("legacy_simulation_id", sa.BigInteger(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.CheckConstraint( + "kind IN ('economy', 'household')", + name=op.f("ck_simulations_kind"), + ), + sa.CheckConstraint( + "country ~ '^[a-z]{2}$'", name=op.f("ck_simulations_country") + ), + sa.CheckConstraint( + "(kind = 'household' AND household_id IS NOT NULL " + "AND region_code IS NULL AND dataset_name IS NULL) " + "OR (kind = 'economy' AND household_id IS NULL " + "AND region_code IS NOT NULL)", + name=op.f("ck_simulations_population_matches_kind"), + ), + sa.CheckConstraint( + "region_code IS NULL OR region_code ~ '^\\S+$'", + name=op.f("ck_simulations_region_code"), + ), + sa.CheckConstraint( + "dataset_name IS NULL OR dataset_name ~ '^\\S+$'", + name=op.f("ck_simulations_dataset_name"), + ), + sa.CheckConstraint( + "legacy_simulation_id IS NULL OR legacy_simulation_id > 0", + name=op.f("ck_simulations_legacy_simulation_id"), + ), + sa.ForeignKeyConstraint( + ["policy_id", "country"], + [_public_api("policies.id"), _public_api("policies.country_id")], + name="fk_simulations_policy_country", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["dynamic_id"], + [_public_api("dynamics.id")], + name="fk_simulations_dynamic", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["household_id", "country"], + [_public_api("households.id"), _public_api("households.country_id")], + name="fk_simulations_household_country", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name="pk_simulations"), + sa.UniqueConstraint( + "legacy_simulation_id", name="uq_simulations_legacy_simulation_id" + ), + sa.UniqueConstraint( + "id", "country", "kind", name="uq_simulations_id_country_kind" + ), + schema=SCHEMA, + ) + for column in ("policy_id", "household_id", "dynamic_id"): + op.create_index( + f"ix_simulations_{column}", "simulations", [column], schema=SCHEMA + ) + + +def _create_reports() -> None: + simulation_key = [ + f"{SCHEMA}.simulations.id", + f"{SCHEMA}.simulations.country", + f"{SCHEMA}.simulations.kind", + ] + op.create_table( + "reports", + sa.Column("id", sa.Uuid(), nullable=False), + sa.Column("kind", sa.Text(), nullable=False), + sa.Column("country", sa.Text(), nullable=False), + sa.Column("baseline_simulation_id", sa.Uuid(), nullable=False), + sa.Column("reform_simulation_id", sa.Uuid(), nullable=True), + sa.Column("year", sa.Integer(), nullable=False), + sa.Column( + "options", + postgresql.JSONB(), + server_default=sa.text("'{}'::jsonb"), + nullable=False, + ), + sa.Column("legacy_report_id", sa.BigInteger(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.CheckConstraint( + "kind IN ('economy', 'household')", name=op.f("ck_reports_kind") + ), + sa.CheckConstraint("country ~ '^[a-z]{2}$'", name=op.f("ck_reports_country")), + sa.CheckConstraint("year BETWEEN 1900 AND 2200", name=op.f("ck_reports_year")), + sa.CheckConstraint( + "jsonb_typeof(options) = 'object'", name=op.f("ck_reports_options") + ), + sa.CheckConstraint( + "kind = 'household' OR reform_simulation_id IS NOT NULL", + name=op.f("ck_reports_economy_has_reform"), + ), + sa.CheckConstraint( + "legacy_report_id IS NULL OR legacy_report_id > 0", + name=op.f("ck_reports_legacy_report_id"), + ), + sa.ForeignKeyConstraint( + ["baseline_simulation_id", "country", "kind"], + simulation_key, + name="fk_reports_baseline_simulation", + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id", "country", "kind"], + simulation_key, + name="fk_reports_reform_simulation", + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name="pk_reports"), + sa.UniqueConstraint("legacy_report_id", name="uq_reports_legacy_report_id"), + schema=SCHEMA, + ) + for column in ("baseline_simulation_id", "reform_simulation_id"): + op.create_index(f"ix_reports_{column}", "reports", [column], schema=SCHEMA) + + +def upgrade() -> None: + """Create both tables; the runtime role may not update either.""" + _create_simulations() + _create_reports() + op.execute( + "REVOKE UPDATE ON simulation.simulations, simulation.reports " + "FROM simulation_runtime" + ) + + +def downgrade() -> None: + """Drop the tables ``upgrade`` created, with their indexes and privileges.""" + for column in ("reform_simulation_id", "baseline_simulation_id"): + op.drop_index(f"ix_reports_{column}", table_name="reports", schema=SCHEMA) + op.drop_table("reports", schema=SCHEMA) + for column in ("dynamic_id", "household_id", "policy_id"): + op.drop_index( + f"ix_simulations_{column}", table_name="simulations", schema=SCHEMA + ) + op.drop_table("simulations", schema=SCHEMA) diff --git a/projects/policyengine-simulation-entry/DEPLOYMENT.md b/projects/policyengine-simulation-entry/DEPLOYMENT.md index 9783ef1cc..87ddd1f5e 100644 --- a/projects/policyengine-simulation-entry/DEPLOYMENT.md +++ b/projects/policyengine-simulation-entry/DEPLOYMENT.md @@ -22,6 +22,40 @@ Credential values belong in the cloud secret store. They must not be committed to the repository or stored as plain GitHub variables. The deployment workflow receives only the configuration needed to reference those protected resources. +## Simulation database + +The Simulation Entrypoint owns the `simulation` schema in the Postgres database +that holds the Public API's v2 tables. Before the entrypoint candidate deploys +in each environment, the `migrate_simulation_database` job applies the +schema's Alembic chain (`migrations/`) with the migrator role. It is the only +job that receives `SIMULATION_MIGRATION_DATABASE_URL`, a GitHub environment +secret in `beta` and `prod`, passed in that step's `env:` alone. If the +migration fails, the entrypoint is not deployed. Only a deployment from `main` +applies revisions; one dispatched from another branch (the Stage 12 deploy can +be) only requires the database to be at that branch's head, so an unmerged +revision never reaches a shared database. + +The deploy service account must be able to list the versions of the runtime +URL's secret (`gcloud secrets versions list`), as for the other runtime +secrets. + +The entrypoint itself connects as the runtime role. The deploy step resolves +the Secret Manager secret named by the `SIMULATION_DATABASE_URL_SECRET_NAME` +environment variable to its newest enabled version and mounts that version as +`SIMULATION_DATABASE_URL`. Only the entrypoint's runtime service account may +read it. Modal never receives either URL. + +`GET /ready` reports the database's applied revision next to the revision the +image expects, under `simulation_database`. The database does not change the +readiness status, because no old-gateway or Stage 12 route uses it. The +candidate smoke check requires the two revisions to match, which shows that the +runtime secret reached the revision and the migration ran. While a cold +candidate has not reached its database yet, it asks again, up to six requests +in all; the stable and +rollback checks do not, because an older image may run against a newer schema. +The ownership rules and roles are described in +`libs/policyengine-simulation-persistence/README.md`. + ## Candidate qualification For each eligible revision, the workflow first deploys `beta` diff --git a/projects/policyengine-simulation-entry/Dockerfile.dockerignore b/projects/policyengine-simulation-entry/Dockerfile.dockerignore index 9379db178..a54f9060f 100644 --- a/projects/policyengine-simulation-entry/Dockerfile.dockerignore +++ b/projects/policyengine-simulation-entry/Dockerfile.dockerignore @@ -13,6 +13,8 @@ libs/* !libs/policyengine-simulation-contract/** !libs/policyengine-simulation-observability/ !libs/policyengine-simulation-observability/** +!libs/policyengine-simulation-persistence/ +!libs/policyengine-simulation-persistence/** !libs/policyengine-stage12-persistence/ !libs/policyengine-stage12-persistence/** libs/**/tests diff --git a/projects/policyengine-simulation-entry/pyproject.toml b/projects/policyengine-simulation-entry/pyproject.toml index 1b43b4f1d..11f569523 100644 --- a/projects/policyengine-simulation-entry/pyproject.toml +++ b/projects/policyengine-simulation-entry/pyproject.toml @@ -19,6 +19,7 @@ dependencies = [ "policyengine-observability[fastapi,google,httpx,otlp-grpc]>=3.0.4,<4", "policyengine-simulation-contract", "policyengine-simulation-observability", + "policyengine-simulation-persistence", "policyengine-stage12-persistence", "pyjwt>=2.10.1,<3", "uvicorn>=0.35.0,<1", @@ -30,6 +31,7 @@ packages = ["src/policyengine_simulation_entry"] [tool.uv.sources] policyengine-simulation-contract = { path = "../../libs/policyengine-simulation-contract", editable = true } policyengine-simulation-observability = { path = "../../libs/policyengine-simulation-observability", editable = true } +policyengine-simulation-persistence = { path = "../../libs/policyengine-simulation-persistence", editable = true } policyengine-stage12-persistence = { path = "../../libs/policyengine-stage12-persistence", editable = true } [project.optional-dependencies] diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py index be3a699ce..d862ebcd1 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/app.py @@ -25,7 +25,6 @@ JobSubmitResponse, PingRequest, PingResponse, - ReadinessResponse, SimulationErrorResponse, SimulationRequest, VersionMap, @@ -42,6 +41,7 @@ from policyengine_simulation_observability.observability import ( init_simulation_observability, ) +from policyengine_simulation_persistence import HEAD_REVISION from starlette.datastructures import MutableHeaders from policyengine_simulation_entry.auth import ( @@ -59,15 +59,19 @@ ComparisonBackend, EntryRuntime, ResponseIdentifier, + SimulationDatabaseConnection, model_json, request_identifiers, route_template, set_observability_header, ) from policyengine_simulation_entry.schemas import ( + SimulationDatabaseReadiness, + SimulationEntrypointReadinessResponse, TemporaryStage12ReportResponse, TemporaryStage12SubmissionResponse, ) +from policyengine_simulation_entry.simulation_database import SimulationDatabase from policyengine_simulation_entry.stage12_backend import ( Stage12ComparisonBackend, TemporaryStage12DispatchFailed, @@ -84,11 +88,13 @@ "STAGE12_MODAL_SUBMISSION_TIMEOUT_SECONDS", "ComparisonBackend", "ResponseIdentifier", + "SimulationDatabaseConnection", "app", "create_app", ] logger = logging.getLogger(__name__) +SIMULATION_DATABASE_READINESS_TIMEOUT_SECONDS = 3.0 def _temporary_submission_payload( @@ -456,6 +462,34 @@ async def get_budget_window_job( ) +async def _simulation_database_readiness( + database: SimulationDatabaseConnection, +) -> SimulationDatabaseReadiness: + """Report the database's applied revision next to the image's.""" + try: + revision = await asyncio.wait_for( + database.read_revision(), + timeout=SIMULATION_DATABASE_READINESS_TIMEOUT_SECONDS, + ) + # Any failure to reach or read the database is reported, never raised: + # the old-gateway routes this check covers do not use it. + except Exception as error: # noqa: BLE001 # reported as unreachable + logger.warning( + "simulation_database_unreachable", + extra={"error_type": type(error).__name__}, + ) + return SimulationDatabaseReadiness( + reachable=False, + revision=None, + expected_revision=HEAD_REVISION, + ) + return SimulationDatabaseReadiness( + reachable=True, + revision=revision, + expected_revision=HEAD_REVISION, + ) + + def _register_service_routes(app: FastAPI, entry: EntryRuntime) -> None: """Add the public version, health, readiness and ping routes.""" @@ -488,14 +522,23 @@ async def get_country_versions(kind: str, request: Request) -> Response: async def health() -> HealthResponse: return HealthResponse() - @app.get("/ready", response_model=ReadinessResponse) + @app.get("/ready", response_model=SimulationEntrypointReadinessResponse) async def ready() -> Response: - if await entry.backend.ready(): - return JSONResponse(ReadinessResponse(status="ready").model_dump()) - return JSONResponse( - ReadinessResponse(status="not_ready").model_dump(), - status_code=503, + backend_ready = await entry.backend.ready() + status = "ready" if backend_ready else "not_ready" + status_code = 200 if backend_ready else 503 + if entry.database is None: + # Without a configured database the response keeps its old shape. + readiness = SimulationEntrypointReadinessResponse(status=status) + return JSONResponse( + readiness.model_dump(mode="json", exclude={"simulation_database"}), + status_code=status_code, + ) + readiness = SimulationEntrypointReadinessResponse( + status=status, + simulation_database=await _simulation_database_readiness(entry.database), ) + return JSONResponse(readiness.model_dump(mode="json"), status_code=status_code) @app.post( "/ping", @@ -515,9 +558,15 @@ def create_app( backend: SimulationBackend | None = None, auth_dependency: AuthenticationDependency | None = None, comparison_backend: ComparisonBackend | None = None, + simulation_database: SimulationDatabaseConnection | None = None, ) -> FastAPI: """Build the app with injectable auth/backend seams for hermetic tests.""" runtime_settings = settings or Settings.from_env() + runtime_database = simulation_database + if runtime_database is None and runtime_settings.simulation_database_url: + runtime_database = SimulationDatabase.from_url( + runtime_settings.simulation_database_url + ) @asynccontextmanager async def lifespan(_: FastAPI) -> AsyncIterator[None]: @@ -527,6 +576,8 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: yield finally: await runtime_backend.close() + if runtime_database is not None: + runtime_database.dispose() runtime.shutdown() app = FastAPI( @@ -565,6 +616,7 @@ async def lifespan(_: FastAPI) -> AsyncIterator[None]: runtime=runtime, backend=runtime_backend, comparison=runtime_comparison, + database=runtime_database, ) _install_request_context(app, runtime_settings) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py index 8385097e9..5d23e8c54 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/config.py @@ -92,6 +92,10 @@ class Settings: stage12_database_url: str = "" stage12_artifact_bucket: str = "" stage12_retention_days: int = REQUIRED_STAGE12_RETENTION_DAYS + # The runtime role's URL for the Simulation Entrypoint's own schema. Empty + # outside deployed environments; nothing on the old-gateway or Stage 12 + # paths reads it. + simulation_database_url: str = "" @classmethod def from_env(cls) -> Settings: @@ -142,6 +146,7 @@ def from_env(cls) -> Settings: stage12_database_url=os.getenv("STAGE12_DATABASE_URL", ""), stage12_artifact_bucket=os.getenv("STAGE12_ARTIFACT_BUCKET", ""), stage12_retention_days=int(os.getenv("STAGE12_RETENTION_DAYS", "30")), + simulation_database_url=os.getenv("SIMULATION_DATABASE_URL", ""), ) @property @@ -170,6 +175,7 @@ def validate(self) -> None: self._validate_caller_auth() self._validate_old_gateway() self._validate_stage12() + self._validate_simulation_database() def _validate_caller_auth(self) -> None: """Require caller authentication in production, with an HTTPS issuer.""" @@ -267,3 +273,11 @@ def _validate_stage12(self) -> None: ): msg = "STAGE12_DATABASE_URL must use the canonical postgresql:// scheme." raise ConfigurationError(msg) + + def _validate_simulation_database(self) -> None: + """Require the canonical scheme for the schema's runtime URL, when set.""" + if self.simulation_database_url and not self.simulation_database_url.startswith( + "postgresql://" + ): + msg = "SIMULATION_DATABASE_URL must use the canonical postgresql:// scheme." + raise ConfigurationError(msg) diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/entry_runtime.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/entry_runtime.py index 279700705..4898f758f 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/entry_runtime.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/entry_runtime.py @@ -210,6 +210,21 @@ def _add_response_telemetry( attributes[response_identifier_key] = response_identifier +class SimulationDatabaseConnection(Protocol): + """The Simulation Entrypoint's own database, as the routes use it. + + The real implementation is ``SimulationDatabase``; tests inject fakes. + """ + + async def read_revision(self) -> str | None: + """Return the applied migration revision, or None before the first one.""" + ... + + def dispose(self) -> None: + """Close every pooled connection.""" + ... + + class EntryRuntime: """The settings, telemetry runtime and backends the entry routes share.""" @@ -220,12 +235,14 @@ def __init__( runtime: ObservabilityRuntime, backend: SimulationBackend, comparison: ComparisonBackend | None, + database: SimulationDatabaseConnection | None, ) -> None: """Keep the dependencies that ``create_app`` resolved.""" self.settings = settings self.runtime = runtime self.backend = backend self.comparison = comparison + self.database = database def bind_observability_id(self, request: Request, value: object) -> str | None: """Bind a normalized observability ID to the request and telemetry context.""" diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/schemas.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/schemas.py index d387526df..74aa0f74e 100644 --- a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/schemas.py +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/schemas.py @@ -115,3 +115,28 @@ class TemporaryStage12ReportResponse(BaseModel): report: ComparisonReportRecord simulations: tuple[ComparisonSimulationRecord, ...] + + +class SimulationDatabaseReadiness(BaseModel): + """The Simulation Entrypoint's own database, as the readiness check found it. + + ``revision`` is the migration revision the database has applied, or null + when it was unreachable or has no migrations yet; ``expected_revision`` is + the newest revision this image knows. Neither changes ``status``: no + old-gateway or Stage 12 route uses this database. + """ + + model_config = ConfigDict(extra="forbid", frozen=True) + + reachable: bool + revision: str | None + expected_revision: str + + +class SimulationEntrypointReadinessResponse(BaseModel): + """Old-gateway readiness, plus the database revision when one is configured.""" + + model_config = ConfigDict(extra="forbid", frozen=True) + + status: Literal["ready", "not_ready"] + simulation_database: SimulationDatabaseReadiness | None = None diff --git a/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/simulation_database.py b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/simulation_database.py new file mode 100644 index 000000000..e0ca4f1bd --- /dev/null +++ b/projects/policyengine-simulation-entry/src/policyengine_simulation_entry/simulation_database.py @@ -0,0 +1,42 @@ +"""The Simulation Entrypoint's connection to its own ``simulation`` schema.""" + +from __future__ import annotations + +import asyncio +from typing import TYPE_CHECKING + +from policyengine_simulation_persistence import ( + create_simulation_engine, + read_schema_revision, +) + +if TYPE_CHECKING: + from sqlalchemy import Engine + + +class SimulationDatabase: + """Run short, blocking database operations off the event loop.""" + + def __init__(self, engine: Engine) -> None: + """Use ``engine``, which connects as the runtime role.""" + self.engine = engine + + @classmethod + def from_url(cls, database_url: str) -> SimulationDatabase: + """Build the runtime engine; no connection opens until first use. + + Raise ValueError, without echoing the URL, when it is malformed. + """ + return cls(create_simulation_engine(database_url)) + + def _read_revision(self) -> str | None: + with self.engine.connect() as connection: + return read_schema_revision(connection) + + async def read_revision(self) -> str | None: + """Return the applied migration revision, or None before the first one.""" + return await asyncio.to_thread(self._read_revision) + + def dispose(self) -> None: + """Close every pooled connection.""" + self.engine.dispose() diff --git a/projects/policyengine-simulation-entry/tests/deployment_rule_checks.py b/projects/policyengine-simulation-entry/tests/deployment_rule_checks.py new file mode 100644 index 000000000..fa06bed25 --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/deployment_rule_checks.py @@ -0,0 +1,55 @@ +"""Checks behind two deployment rules, runnable on the repository or a fixture. + +- ``combined_secret_resolutions`` (sim.deploy.workflow-shell-safety): Bash + gives an array assignment the exit status of its last command substitution + only, so an assignment that resolves several secrets lets an earlier + failure pass unnoticed. +- ``libraries_missing_from_context`` (sim.deploy.container-context-minimal): + every local library the entrypoint installs must be in its build context. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import tomllib + +ARRAY_ASSIGNMENT = re.compile(r"^\s*\w+\+?=\(", re.MULTILINE) +# The line that closes a multi-line array assignment: only its parenthesis. +ARRAY_END = re.compile(r"\n[ \t]*\)") +SECRET_RESOLUTION = "$(resolve_secret_reference" + + +def combined_secret_resolutions(script_text: str) -> list[str]: + """Return array assignments that resolve more than one secret.""" + violations = [] + for match in ARRAY_ASSIGNMENT.finditer(script_text): + line_end = script_text.find("\n", match.end()) + first_line = script_text[match.start() : line_end if line_end >= 0 else None] + if ")" in first_line: + assignment = first_line + else: + closing = ARRAY_END.search(script_text, match.end()) + assignment = script_text[ + match.start() : closing.start() if closing is not None else None + ] + if assignment.count(SECRET_RESOLUTION) > 1: + violations.append(assignment.strip().splitlines()[0]) + return violations + + +def libraries_missing_from_context(pyproject_text: str, ignore_text: str) -> list[str]: + """Return the local libraries the project installs but its context leaves out.""" + rules = ignore_text.splitlines() + sources = ( + tomllib.loads(pyproject_text).get("tool", {}).get("uv", {}).get("sources", {}) + ) + libraries = [ + Path(source["path"]).name for source in sources.values() if "path" in source + ] + return [ + library + for library in libraries + if f"!libs/{library}/" not in rules or f"!libs/{library}/**" not in rules + ] diff --git a/projects/policyengine-simulation-entry/tests/migration_wiring_checks.py b/projects/policyengine-simulation-entry/tests/migration_wiring_checks.py new file mode 100644 index 000000000..8b899767d --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/migration_wiring_checks.py @@ -0,0 +1,67 @@ +"""The check behind sim.persistence.migrate-before-entrypoint's workflow part. + +It takes a deploy workflow's jobs, so the rule test runs it on the reusable +deploy workflow and the proof test on the rule's fixtures. +""" + +from __future__ import annotations + +import json + +MIGRATION_JOB = "migrate_simulation_database" +MIGRATION_STEP = "Apply simulation schema migrations" +MIGRATION_SECRET = "SIMULATION_MIGRATION_DATABASE_URL" +ENTRYPOINT_JOB = "deploy_entrypoint" +MIGRATION_MODE = "${{ github.ref == 'refs/heads/main' && 'upgrade' || 'verify' }}" + + +def _needs(job: dict) -> list[str]: + needs = job.get("needs", []) + return [needs] if isinstance(needs, str) else list(needs) + + +def secret_carriers( + jobs: dict, *, workflow_env: dict | None = None +) -> list[tuple[str, str]]: + """Return each job, job env or step that holds the migrator URL. + + A workflow-level ``env:`` reaches every job, so it counts as a carrier + for all of them. + """ + carriers = [] + if MIGRATION_SECRET in json.dumps(workflow_env or {}): + carriers.append(("workflow", "workflow env")) + for job_name, job in jobs.items(): + if MIGRATION_SECRET in json.dumps(job.get("env", {})): + carriers.append((job_name, "job env")) + carriers.extend( + (job_name, str(step.get("name"))) + for step in job.get("steps", []) + if MIGRATION_SECRET in json.dumps(step) + ) + return carriers + + +def migration_wiring_violations( + jobs: dict, *, workflow_env: dict | None = None +) -> list[str]: + """Return how a deploy workflow's jobs depart from the migration wiring.""" + migration = jobs.get(MIGRATION_JOB) + if migration is None: + return [f"no {MIGRATION_JOB} job"] + violations = [] + entrypoint = jobs.get(ENTRYPOINT_JOB, {}) + if MIGRATION_JOB not in _needs(entrypoint) or ( + f"needs.{MIGRATION_JOB}.result == 'success'" not in entrypoint.get("if", "") + ): + violations.append( + f"{ENTRYPOINT_JOB} does not wait for a successful {MIGRATION_JOB}" + ) + carriers = secret_carriers(jobs, workflow_env=workflow_env) + if carriers != [(MIGRATION_JOB, MIGRATION_STEP)]: + violations.append(f"{MIGRATION_SECRET} reaches {carriers}") + steps = {step.get("name"): step for step in migration.get("steps", [])} + mode = steps.get(MIGRATION_STEP, {}).get("env", {}).get("SIMULATION_MIGRATION_MODE") + if mode != MIGRATION_MODE: + violations.append(f"{MIGRATION_STEP} applies revisions from any branch") + return violations diff --git a/projects/policyengine-simulation-entry/tests/test_config.py b/projects/policyengine-simulation-entry/tests/test_config.py index 8dbf6809f..23ed45320 100644 --- a/projects/policyengine-simulation-entry/tests/test_config.py +++ b/projects/policyengine-simulation-entry/tests/test_config.py @@ -297,3 +297,41 @@ def test_validation_reports_the_first_failed_check(overrides, message): ) def test_consistent_configuration_validates(overrides): make_settings(**overrides).validate() + + +def test_the_simulation_database_is_optional(monkeypatch): + from policyengine_simulation_entry.config import Settings + + monkeypatch.delenv("SIMULATION_DATABASE_URL", raising=False) + settings = Settings.from_env() + + assert settings.simulation_database_url == "" + make_settings(simulation_database_url="").validate() + + +def test_the_simulation_database_url_is_read_from_the_environment(monkeypatch): + from policyengine_simulation_entry.config import Settings + + monkeypatch.setenv( + "SIMULATION_DATABASE_URL", + "postgresql://simulation_runtime:secret@db.example:5432/postgres", + ) + + assert Settings.from_env().simulation_database_url == ( + "postgresql://simulation_runtime:secret@db.example:5432/postgres" + ) + + +@pytest.mark.parametrize( + "database_url", + [ + "postgres://simulation_runtime:secret@db.example/postgres", + "postgresql+psycopg://simulation_runtime:secret@db.example/postgres", + "mysql://simulation_runtime:secret@db.example/postgres", + ], +) +def test_the_simulation_database_url_must_be_canonical(database_url): + with pytest.raises(ConfigurationError, match="SIMULATION_DATABASE_URL") as error: + make_settings(simulation_database_url=database_url).validate() + + assert "secret" not in str(error.value) diff --git a/projects/policyengine-simulation-entry/tests/test_database_access_boundaries.py b/projects/policyengine-simulation-entry/tests/test_database_access_boundaries.py index 81d7ad590..ad60f091d 100644 --- a/projects/policyengine-simulation-entry/tests/test_database_access_boundaries.py +++ b/projects/policyengine-simulation-entry/tests/test_database_access_boundaries.py @@ -23,8 +23,9 @@ / "sim.boundaries.sqlalchemy-and-alembic-only" ) # Every directory that holds non-test source. The executor's src/modal is -# under projects/*/src. -SOURCE_ROOT_PATTERNS = ("projects/*/src", "libs/*/src", ".github/scripts") +# under projects/*/src. The simulation schema's Alembic chain may import +# alembic and change the schema, but still connects only through SQLAlchemy. +SOURCE_ROOT_PATTERNS = ("projects/*/src", "libs/*/src", ".github/scripts", "migrations") # Directories the rule names, each of which must contribute scanned files. REQUIRED_SOURCE_DIRECTORIES = ( "projects/policyengine-simulation-entry/src", @@ -34,8 +35,10 @@ "libs/policyengine-fastapi/src", "libs/policyengine-simulation-contract/src", "libs/policyengine-simulation-observability/src", + "libs/policyengine-simulation-persistence/src", "libs/policyengine-stage12-persistence/src", ".github/scripts", + "migrations", ) diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py index a47600cbc..804fae52b 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets.py @@ -210,8 +210,9 @@ def _assert_beta_then_prod_release( def _assert_deployment_environment_settings(reusable_workflow: str) -> None: assert ( - # Sentinel, restore and final routing acknowledgement use secrets too. - reusable_workflow.count("environment: ${{ inputs.release_environment }}") == 15 + # Sentinel, restore, final routing acknowledgement and the simulation + # database migration use secrets too. + reusable_workflow.count("environment: ${{ inputs.release_environment }}") == 16 ) assert "APP_ENVIRONMENT: ${{ inputs.deployment_environment }}" in reusable_workflow assert "STAGE12_ENABLED_VALUE: ${{ vars.STAGE12_ENABLED }}" in reusable_workflow diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py index 1052bfab5..a6d870841 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets_cloud_run.py @@ -10,10 +10,12 @@ from pathlib import Path import pytest +from deployment_rule_checks import SECRET_RESOLUTION, combined_secret_resolutions from test_deployment_assets_support import ( CLOUD_RUN_DEPLOY_SCRIPT, DEPLOY_WORKFLOW, DOCKERFILE, + REPOSITORY_ROOT, REUSABLE_DEPLOY_WORKFLOW, STAGE12_DEPLOY_WORKFLOW, ) @@ -39,7 +41,7 @@ def test_cloud_run_deployment_does_not_interpolate_github_values_in_shell(): assert ":latest" not in deployment_script -def _cloud_run_deploy_environment(tmp_path: Path) -> dict[str, str]: +def cloud_run_deploy_environment(tmp_path: Path) -> dict[str, str]: """Return a complete deployment environment backed by a fake gcloud.""" fake_bin = tmp_path / "bin" fake_bin.mkdir() @@ -114,11 +116,14 @@ def _cloud_run_deploy_environment(tmp_path: Path) -> dict[str, str]: "STAGE12_DATABASE_URL_SECRET_NAME": "stage12-staging-database-url", "MODAL_TOKEN_ID_SECRET_NAME": "modal-token-id", "MODAL_TOKEN_SECRET_SECRET_NAME": "modal-token-secret", + "SIMULATION_DATABASE_URL_SECRET_NAME": ( + "simulation-staging-runtime-database-url" + ), } def test_cloud_run_deployment_escapes_environment_and_pins_secret_versions(tmp_path): - command_environment = _cloud_run_deploy_environment(tmp_path) + command_environment = cloud_run_deploy_environment(tmp_path) arguments_capture = Path(command_environment["GCLOUD_ARGUMENTS_CAPTURE"]) env_capture = Path(command_environment["GCLOUD_ENV_FILE_CAPTURE"]) @@ -152,6 +157,7 @@ def test_cloud_run_deployment_escapes_environment_and_pins_secret_versions(tmp_p secrets_argument = arguments[arguments.index("--set-secrets") + 1] assert secrets_argument == ( "OLD_GATEWAY_AUTH_CLIENT_SECRET=old-client-secret:3," + "SIMULATION_DATABASE_URL=simulation-staging-runtime-database-url:3," "MODAL_TOKEN_ID=modal-token-id:3," "MODAL_TOKEN_SECRET=modal-token-secret:3," "STAGE12_DATABASE_URL=stage12-staging-database-url:3" @@ -159,6 +165,75 @@ def test_cloud_run_deployment_escapes_environment_and_pins_secret_versions(tmp_p assert ":latest" not in secrets_argument +@pytest.mark.parametrize( + "secret_name_variable", + [ + "OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME", + "SIMULATION_DATABASE_URL_SECRET_NAME", + "MODAL_TOKEN_ID_SECRET_NAME", + "MODAL_TOKEN_SECRET_SECRET_NAME", + "STAGE12_DATABASE_URL_SECRET_NAME", + ], +) +def test_cloud_run_deployment_stops_when_a_runtime_secret_cannot_be_resolved( + tmp_path, secret_name_variable +): + command_environment = cloud_run_deploy_environment(tmp_path) | { + secret_name_variable: "not a secret name" + } + + result = subprocess.run( + [CLOUD_RUN_DEPLOY_SCRIPT], + check=False, + capture_output=True, + text=True, + env=command_environment, + ) + + assert result.returncode != 0 + assert "invalid Secret Manager reference" in result.stderr + assert not Path(command_environment["GCLOUD_ARGUMENTS_CAPTURE"]).exists() + + +def test_cloud_run_deployment_resolves_each_secret_in_its_own_assignment(): + script = Path(CLOUD_RUN_DEPLOY_SCRIPT).read_text(encoding="utf-8") + + assert SECRET_RESOLUTION in script + assert combined_secret_resolutions(script) == [] + + +def test_rule_proof_workflow_shell_safety(): + """Proof for rule sim.deploy.workflow-shell-safety (rules/registry.toml).""" + fixtures = ( + REPOSITORY_ROOT / "rules" / "fixtures" / ("sim.deploy.workflow-shell-safety") + ) + + assert combined_secret_resolutions( + (fixtures / "bad_deploy_script.sh").read_text(encoding="utf-8") + ) == ["runtime_secrets=("] + assert ( + combined_secret_resolutions( + (fixtures / "good_deploy_script.sh").read_text(encoding="utf-8") + ) + == [] + ) + + +def test_stage12_secrets_are_resolved_by_the_stage12_block_alone(): + """The comparison path shares no deploy helper with the main path.""" + script = Path(CLOUD_RUN_DEPLOY_SCRIPT).read_text(encoding="utf-8") + stage12_block = script.split('if [[ "${DEPLOY_STAGE12_V2}" == "true" ]]; then', 1)[ + 1 + ].split("\nfi\n", 1)[0] + main_path = script.replace(stage12_block, "") + + assert "add_stage12_runtime_secret STAGE12_DATABASE_URL" in stage12_block + assert "add_runtime_secret " not in stage12_block + assert "add_stage12_runtime_secret " not in main_path.replace( + "add_stage12_runtime_secret() {", "" + ) + + @pytest.mark.parametrize( ("configured", "expected"), [(None, "0"), ("", "0"), ("0", "0"), ("1", "1")], @@ -169,7 +244,7 @@ def test_cloud_run_deployment_defaults_permission_enforcement_to_log_only( configured, expected, ): - command_environment = _cloud_run_deploy_environment(tmp_path) + command_environment = cloud_run_deploy_environment(tmp_path) if configured is not None: command_environment["SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE"] = ( configured @@ -192,7 +267,7 @@ def test_cloud_run_deployment_rejects_invalid_permission_enforcement( tmp_path, configured, ): - command_environment = _cloud_run_deploy_environment(tmp_path) + command_environment = cloud_run_deploy_environment(tmp_path) command_environment["SIMULATION_ENTRYPOINT_ENFORCE_PERMISSIONS_VALUE"] = configured result = subprocess.run( diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets_support.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets_support.py index ae004c799..00ab839cf 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets_support.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets_support.py @@ -128,7 +128,22 @@ def write_fake_smoke_curl(tmp_path: Path, revisions: list[str]) -> tuple[Path, P printf '{"status":"healthy"}' > "${body_file}" ;; */ready) - printf '{"status":"ready"}' + ready_body='{"status":"ready"}' + if [ -n "${FAKE_CURL_READY_BODIES:-}" ]; then + # One body per line, served in order; the last one repeats. + ready_count=0 + if [ -f "${FAKE_CURL_READY_BODIES}.count" ]; then + ready_count="$(cat "${FAKE_CURL_READY_BODIES}.count")" + fi + ready_count=$((ready_count + 1)) + printf '%s' "${ready_count}" > "${FAKE_CURL_READY_BODIES}.count" + line_count="$(wc -l < "${FAKE_CURL_READY_BODIES}")" + if [ "${ready_count}" -gt "${line_count}" ]; then + ready_count="${line_count}" + fi + ready_body="$(sed -n "${ready_count}p" "${FAKE_CURL_READY_BODIES}")" + fi + printf '%s' "${FAKE_CURL_READY:-${ready_body}}" ;; */versions) versions='{}' diff --git a/projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py b/projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py index cda498290..df2e4e669 100644 --- a/projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py +++ b/projects/policyengine-simulation-entry/tests/test_deployment_assets_traffic.py @@ -6,6 +6,7 @@ import textwrap import tomllib +from deployment_rule_checks import libraries_missing_from_context from test_deployment_assets_support import ( DOCKERIGNORE, GITIGNORE, @@ -172,6 +173,40 @@ def test_container_context_excludes_local_environments_and_unrelated_projects(): ) +def test_container_context_includes_every_local_library_the_entrypoint_installs(): + entry_project = REPOSITORY_ROOT / "projects" / "policyengine-simulation-entry" + pyproject = (entry_project / "pyproject.toml").read_text(encoding="utf-8") + + assert "path = " in pyproject + assert ( + libraries_missing_from_context( + pyproject, DOCKERIGNORE.read_text(encoding="utf-8") + ) + == [] + ) + + +def test_rule_proof_container_context_minimal(): + """Proof for rule sim.deploy.container-context-minimal (rules/registry.toml).""" + fixtures = ( + REPOSITORY_ROOT + / "rules" + / "fixtures" + / ("sim.deploy.container-context-minimal") + ) + pyproject = (fixtures / "pyproject.toml").read_text(encoding="utf-8") + + assert libraries_missing_from_context( + pyproject, (fixtures / "bad.dockerignore").read_text(encoding="utf-8") + ) == ["fixture-persistence"] + assert ( + libraries_missing_from_context( + pyproject, (fixtures / "good.dockerignore").read_text(encoding="utf-8") + ) + == [] + ) + + def test_production_lock_contains_the_direct_stage12_database_runtime(): lockfile = tomllib.loads( ( diff --git a/projects/policyengine-simulation-entry/tests/test_openapi.py b/projects/policyengine-simulation-entry/tests/test_openapi.py index 060b6bb8c..65e79cf45 100644 --- a/projects/policyengine-simulation-entry/tests/test_openapi.py +++ b/projects/policyengine-simulation-entry/tests/test_openapi.py @@ -122,6 +122,22 @@ def test_local_area_breakdowns_are_public(): ]["properties"]["result"]["anyOf"] +def test_readiness_documents_the_simulation_database_revision(): + spec = create_app().openapi() + readiness = spec["paths"]["/ready"]["get"]["responses"]["200"] + schema = readiness["content"]["application/json"]["schema"] + database = spec["components"]["schemas"]["SimulationDatabaseReadiness"] + + assert schema == { + "$ref": "#/components/schemas/SimulationEntrypointReadinessResponse" + } + assert set(database["properties"]) == { + "reachable", + "revision", + "expected_revision", + } + + def test_route_table_is_frozen(): assert methods(cast("OpenAPIDocument", create_app().openapi())) == EXPECTED_ROUTES @@ -161,7 +177,9 @@ def test_normalized_contract_matches_old_gateway(): cloud_run_spec ) == normalized_compatibility_paths(gateway_spec) cloud_run_schemas = deepcopy(cloud_run_spec["components"]["schemas"]) - cloud_run_schemas.pop("ReadinessResponse") + # The entrypoint's readiness response adds its own database's revision. + cloud_run_schemas.pop("SimulationEntrypointReadinessResponse") + cloud_run_schemas.pop("SimulationDatabaseReadiness") cloud_run_schemas.pop("SimulationErrorResponse") assert cloud_run_schemas == gateway_spec["components"]["schemas"] gateway_operation_ids = operation_ids(gateway_spec) diff --git a/projects/policyengine-simulation-entry/tests/test_simulation_database.py b/projects/policyengine-simulation-entry/tests/test_simulation_database.py new file mode 100644 index 000000000..95b1defae --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_simulation_database.py @@ -0,0 +1,184 @@ +"""The Simulation Entrypoint's own database: readiness reporting and lifetime.""" + +from __future__ import annotations + +import asyncio + +import pytest +from conftest import FakeBackend, make_settings +from fastapi.testclient import TestClient +from policyengine_simulation_entry import app as app_module +from policyengine_simulation_entry import simulation_database as database_module +from policyengine_simulation_entry.app import create_app +from policyengine_simulation_entry.simulation_database import SimulationDatabase +from policyengine_simulation_persistence import HEAD_REVISION +from sqlalchemy.exc import OperationalError + +DATABASE_URL = "postgresql://simulation_runtime:secret@db.example:5432/postgres" + + +class FakeSimulationDatabase: + """Report a fixed revision, or fail the way an unreachable database does.""" + + def __init__(self, *, revision=HEAD_REVISION, failure=None, delay_seconds=0): + self.revision = revision + self.failure = failure + self.delay_seconds = delay_seconds + self.disposed = False + + async def read_revision(self): + if self.delay_seconds: + await asyncio.sleep(self.delay_seconds) + if self.failure is not None: + raise self.failure + return self.revision + + def dispose(self): + self.disposed = True + + +def _client(backend, database): + return TestClient( + create_app( + settings=make_settings(simulation_database_url=DATABASE_URL), + backend=backend, + auth_dependency=lambda: None, + simulation_database=database, + ) + ) + + +def test_readiness_reports_the_database_revision_and_the_expected_one(): + backend = FakeBackend() + with _client(backend, FakeSimulationDatabase(revision="0001")) as client: + result = client.get("/ready") + + assert result.status_code == 200 + assert result.json() == { + "status": "ready", + "simulation_database": { + "reachable": True, + "revision": "0001", + "expected_revision": HEAD_REVISION, + }, + } + + +def test_readiness_reports_a_database_that_has_no_migrations_yet(): + with _client(FakeBackend(), FakeSimulationDatabase(revision=None)) as client: + database = client.get("/ready").json()["simulation_database"] + + assert database == { + "reachable": True, + "revision": None, + "expected_revision": HEAD_REVISION, + } + + +def test_an_unreachable_database_does_not_change_old_gateway_readiness(caplog): + failure = OperationalError("SELECT 1", {}, Exception("password=secret")) + with _client(FakeBackend(), FakeSimulationDatabase(failure=failure)) as client: + result = client.get("/ready") + + assert result.status_code == 200 + assert result.json() == { + "status": "ready", + "simulation_database": { + "reachable": False, + "revision": None, + "expected_revision": HEAD_REVISION, + }, + } + assert "secret" not in caplog.text + + +def test_readiness_waits_only_briefly_for_the_database(monkeypatch): + monkeypatch.setattr( + app_module, "SIMULATION_DATABASE_READINESS_TIMEOUT_SECONDS", 0.01 + ) + slow_database = FakeSimulationDatabase(delay_seconds=1) + with _client(FakeBackend(), slow_database) as client: + database = client.get("/ready").json()["simulation_database"] + + assert database["reachable"] is False + + +def test_readiness_still_follows_the_old_gateway(): + backend = FakeBackend() + backend.is_ready = False + with _client(backend, FakeSimulationDatabase()) as client: + result = client.get("/ready") + + assert result.status_code == 503 + assert result.json()["status"] == "not_ready" + assert result.json()["simulation_database"]["reachable"] is True + + +def test_the_database_is_released_at_shutdown(): + database = FakeSimulationDatabase() + with _client(FakeBackend(), database): + assert database.disposed is False + + assert database.disposed is True + + +def test_settings_build_the_database_only_when_it_is_configured(monkeypatch): + built_urls = [] + + def build(database_url): + built_urls.append(database_url) + return FakeSimulationDatabase() + + monkeypatch.setattr(SimulationDatabase, "from_url", staticmethod(build)) + + create_app(settings=make_settings(), backend=FakeBackend()) + assert built_urls == [] + + create_app( + settings=make_settings(simulation_database_url=DATABASE_URL), + backend=FakeBackend(), + ) + assert built_urls == [DATABASE_URL] + + +def test_the_database_reads_the_revision_without_blocking_the_event_loop( + monkeypatch, +): + connections = [] + + class FakeConnection: + def __enter__(self): + connections.append(self) + return self + + def __exit__(self, *_): + return False + + class FakeEngine: + disposed = False + + def connect(self): + return FakeConnection() + + def dispose(self): + self.disposed = True + + monkeypatch.setattr( + database_module, + "read_schema_revision", + lambda connection: "0001" if connection in connections else None, + ) + engine = FakeEngine() + database = SimulationDatabase(engine) + + assert asyncio.run(database.read_revision()) == "0001" + database.dispose() + assert engine.disposed is True + + +@pytest.mark.parametrize("database_url", ["", "postgres://user:secret@db/postgres"]) +def test_a_malformed_database_url_never_builds_an_engine(database_url): + with pytest.raises(ValueError, match="Simulation database URL") as error: + SimulationDatabase.from_url(database_url) + + assert "secret" not in str(error.value) diff --git a/projects/policyengine-simulation-entry/tests/test_simulation_database_deployment.py b/projects/policyengine-simulation-entry/tests/test_simulation_database_deployment.py new file mode 100644 index 000000000..ee6486c0e --- /dev/null +++ b/projects/policyengine-simulation-entry/tests/test_simulation_database_deployment.py @@ -0,0 +1,442 @@ +"""Deployment wiring for the Simulation Entrypoint's own database. + +The migration job applies the chain before the entrypoint candidate deploys, +with the migrator URL in that job alone; the entrypoint receives the runtime +URL as a pinned Cloud Run secret; and the candidate smoke requires the +database to be at the image's revision. +""" + +from __future__ import annotations + +import json +import subprocess +from pathlib import Path + +import pytest +import yaml +from migration_wiring_checks import migration_wiring_violations, secret_carriers +from test_deployment_assets_cloud_run import cloud_run_deploy_environment +from test_deployment_assets_support import ( + CLOUD_RUN_DEPLOY_SCRIPT, + REUSABLE_DEPLOY_WORKFLOW, + SMOKE_SCRIPT, + evaluate_github_expression, + smoke_test_env, + workflow_jobs, + write_fake_smoke_curl, +) +from workflow_script_support import ( + REPOSITORY_ROOT, + WORKFLOWS_DIR, + FakeCommands, + run_script, +) + +MIGRATION_SCRIPT = "migrate-simulation-database.sh" +MIGRATION_JOB = "migrate_simulation_database" +MIGRATION_SECRET = "SIMULATION_MIGRATION_DATABASE_URL" +FAKE_MIGRATION_URL = "postgresql://simulation_migrator:secret@db.example/postgres" + + +@pytest.fixture +def fakes(tmp_path: Path) -> FakeCommands: + return FakeCommands.create(tmp_path) + + +# --- The migration job ---------------------------------------------------- + + +def test_the_migration_job_runs_before_the_entrypoint_deploys(): + jobs = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW) + migration = jobs[MIGRATION_JOB] + deploy_entrypoint = jobs["deploy_entrypoint"] + + assert migration["needs"] == "prepare" + assert migration["environment"] == "${{ inputs.release_environment }}" + assert MIGRATION_JOB in deploy_entrypoint["needs"] + context = { + "inputs": {"deploy_stage12_v2": True}, + "needs": { + "prepare": {"result": "success"}, + "configure_stage12_v2": {"result": "success"}, + MIGRATION_JOB: {"result": "success"}, + }, + } + assert evaluate_github_expression(deploy_entrypoint["if"], context, job_if=True) + for failed_result in ("failure", "skipped", "cancelled"): + context["needs"][MIGRATION_JOB]["result"] = failed_result + assert not evaluate_github_expression( + deploy_entrypoint["if"], context, job_if=True + ) + + +def test_the_migration_job_applies_the_chain_with_the_migrator_url_alone(): + migration = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)[MIGRATION_JOB] + steps = {step.get("name"): step for step in migration["steps"]} + install = steps["Install migration dependencies"] + apply_step = steps["Apply simulation schema migrations"] + + assert install["working-directory"] == "libs/policyengine-simulation-persistence" + assert install["run"] == "uv sync --frozen --extra migrations" + assert apply_step["working-directory"] == "libs/policyengine-simulation-persistence" + assert apply_step["run"] == f"../../.github/scripts/{MIGRATION_SCRIPT}" + assert apply_step["env"] == { + MIGRATION_SECRET: "${{ secrets.SIMULATION_MIGRATION_DATABASE_URL }}", + "SIMULATION_MIGRATION_MODE": ( + "${{ github.ref == 'refs/heads/main' && 'upgrade' || 'verify' }}" + ), + } + assert "env" not in migration + assert "permissions" not in migration + + +def _workflow(path: Path) -> dict: + return yaml.safe_load(path.read_text(encoding="utf-8")) + + +def test_only_the_migration_step_receives_the_migrator_url(): + carriers = [ + (path.name, *carrier) + for path in sorted(WORKFLOWS_DIR.glob("*.yml")) + for carrier in secret_carriers( + _workflow(path)["jobs"], workflow_env=_workflow(path).get("env") + ) + ] + + assert carriers == [ + ( + "simulation-deploy.reusable.yml", + MIGRATION_JOB, + "Apply simulation schema migrations", + ) + ] + + +def test_the_deploy_workflow_follows_the_migration_wiring(): + workflow = _workflow(REUSABLE_DEPLOY_WORKFLOW) + + assert ( + migration_wiring_violations(workflow["jobs"], workflow_env=workflow.get("env")) + == [] + ) + + +def test_the_migrator_url_may_not_reach_every_job_through_workflow_env(): + workflow = yaml.safe_load( + ( + REPOSITORY_ROOT + / "rules" + / "fixtures" + / "sim.persistence.migrate-before-entrypoint" + / "good_workflow.yml" + ).read_text(encoding="utf-8") + ) + workflow["env"] = { + MIGRATION_SECRET: "${{ secrets.SIMULATION_MIGRATION_DATABASE_URL }}" + } + + assert migration_wiring_violations(workflow["jobs"], workflow_env=workflow["env"]) + + +def test_rule_proof_migrate_before_entrypoint(): + """Proof for rule sim.persistence.migrate-before-entrypoint.""" + fixtures = ( + REPOSITORY_ROOT + / "rules" + / "fixtures" + / ("sim.persistence.migrate-before-entrypoint") + ) + + bad = migration_wiring_violations( + yaml.safe_load((fixtures / "bad_workflow.yml").read_text(encoding="utf-8"))[ + "jobs" + ] + ) + good = migration_wiring_violations( + yaml.safe_load((fixtures / "good_workflow.yml").read_text(encoding="utf-8"))[ + "jobs" + ] + ) + + assert len(bad) == 3 + assert good == [] + + +UV_ALEMBIC = ["uv", "run", "--frozen", "--extra", "migrations", "alembic", "-c", + "../../migrations/alembic.ini"] # fmt: skip + + +def _run_migration_script(fakes, **environment): + values = { + "SIMULATION_MIGRATION_DATABASE_URL": FAKE_MIGRATION_URL, + "SIMULATION_MIGRATION_MODE": "upgrade", + **environment, + } + return run_script(MIGRATION_SCRIPT, env=fakes.env(**values)) + + +def test_the_migration_script_upgrades_and_then_requires_the_head(fakes): + fakes.add("uv") + + result = _run_migration_script(fakes) + + assert result.returncode == 0, result.stderr + assert fakes.calls("uv") == [ + [*UV_ALEMBIC, "upgrade", "head"], + [*UV_ALEMBIC, "current", "--check-heads"], + ] + assert "secret" not in result.stdout + result.stderr + + +def test_verify_mode_only_requires_the_database_at_the_head(fakes): + fakes.add("uv") + + result = _run_migration_script(fakes, SIMULATION_MIGRATION_MODE="verify") + + assert result.returncode == 0, result.stderr + assert fakes.calls("uv") == [[*UV_ALEMBIC, "current", "--check-heads"]] + + +@pytest.mark.parametrize("mode", ["", "downgrade", "UPGRADE"]) +def test_the_migration_script_refuses_an_unknown_mode(fakes, mode): + fakes.add("uv") + + result = _run_migration_script(fakes, SIMULATION_MIGRATION_MODE=mode) + + assert result.returncode != 0 + assert "SIMULATION_MIGRATION_MODE" in result.stderr + assert fakes.calls("uv") == [] + + +def test_the_migration_script_refuses_to_run_without_the_migrator_url(fakes): + fakes.add("uv") + + result = _run_migration_script(fakes, SIMULATION_MIGRATION_DATABASE_URL="") + + assert result.returncode != 0 + assert MIGRATION_SECRET in result.stderr + assert fakes.calls("uv") == [] + + +def test_a_failed_upgrade_fails_the_job_before_the_head_check(fakes): + fakes.add("uv", 'if [[ " $* " == *" upgrade "* ]]; then exit 3; fi\n') + + result = _run_migration_script(fakes) + + assert result.returncode == 3 + assert len(fakes.calls("uv")) == 1 + + +def test_only_the_default_branch_applies_migrations(): + migration = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)[MIGRATION_JOB] + steps = {step.get("name"): step for step in migration["steps"]} + mode = steps["Apply simulation schema migrations"]["env"][ + "SIMULATION_MIGRATION_MODE" + ] + + # A deployment dispatched from another branch must not apply that branch's + # revisions; it only requires the database to be at its chain's head. + assert ( + evaluate_github_expression(mode, {"github": {"ref": "refs/heads/main"}}) + == "upgrade" + ) + assert ( + evaluate_github_expression(mode, {"github": {"ref": "refs/heads/feature"}}) + == "verify" + ) + + +# --- The entrypoint's runtime URL ----------------------------------------- + + +def _run_deploy(tmp_path, **overrides): + environment = cloud_run_deploy_environment(tmp_path) | overrides + result = subprocess.run( + [CLOUD_RUN_DEPLOY_SCRIPT], + check=False, + capture_output=True, + text=True, + env=environment, + ) + return result, environment + + +def _secrets_argument(environment) -> str: + arguments = ( + Path(environment["GCLOUD_ARGUMENTS_CAPTURE"]) + .read_text(encoding="utf-8") + .splitlines() + ) + return arguments[arguments.index("--set-secrets") + 1] + + +@pytest.mark.parametrize("deploy_stage12", ["true", "false"]) +def test_the_entrypoint_receives_the_runtime_url_as_a_pinned_secret( + tmp_path, deploy_stage12 +): + result, environment = _run_deploy(tmp_path, DEPLOY_STAGE12_V2=deploy_stage12) + + assert result.returncode == 0, result.stderr + assert ( + "SIMULATION_DATABASE_URL=simulation-staging-runtime-database-url:3" + in _secrets_argument(environment).split(",") + ) + runtime_environment = json.loads( + Path(environment["GCLOUD_ENV_FILE_CAPTURE"]).read_text(encoding="utf-8") + ) + assert "SIMULATION_DATABASE_URL" not in runtime_environment + assert MIGRATION_SECRET not in runtime_environment + + +@pytest.mark.parametrize( + "secret_name", + ["", "not a secret name", "projects/x/secrets/name/versions/1"], +) +def test_the_entrypoint_deploy_requires_a_valid_runtime_url_secret( + tmp_path, secret_name +): + result, _ = _run_deploy(tmp_path, SIMULATION_DATABASE_URL_SECRET_NAME=secret_name) + + assert result.returncode != 0 + assert "SIMULATION_DATABASE_URL_SECRET_NAME" in result.stderr or ( + "Secret Manager reference" in result.stderr + ) + + +def test_the_deploy_step_passes_the_runtime_secret_name(): + deploy_entrypoint = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["deploy_entrypoint"] + deploy_step = next( + step + for step in deploy_entrypoint["steps"] + if step.get("name") == "Deploy Cloud Run entrypoint candidate" + ) + + assert deploy_step["env"]["SIMULATION_DATABASE_URL_SECRET_NAME"] == ( + "${{ vars.SIMULATION_DATABASE_URL_SECRET_NAME }}" + ) + + +# --- The candidate smoke -------------------------------------------------- + + +def test_the_candidate_smoke_requires_the_current_database_revision(): + integration = workflow_jobs(REUSABLE_DEPLOY_WORKFLOW)["integration"] + smoke = next( + step + for step in integration["steps"] + if step.get("name") == "Verify the complete deployed request path" + ) + + assert smoke["env"]["REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION"] == "true" + + +def _ready_body(revision, expected_revision="0001"): + return json.dumps( + { + "status": "ready", + "simulation_database": { + "reachable": revision is not None, + "revision": revision, + "expected_revision": expected_revision, + }, + } + ) + + +def _run_smoke(tmp_path, ready_body, *, require): + state_file, revisions_file = write_fake_smoke_curl(tmp_path, ["revision-1"]) + environment = smoke_test_env(tmp_path, state_file, revisions_file) | { + "FAKE_CURL_READY": ready_body, + } + if require is not None: + environment["REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION"] = require + return subprocess.run( + ["bash", SMOKE_SCRIPT, "https://candidate.example.test", "revision-1"], + check=False, + capture_output=True, + text=True, + env=environment, + ) + + +def test_the_smoke_accepts_a_database_at_the_expected_revision(tmp_path): + result = _run_smoke(tmp_path, _ready_body("0001"), require="true") + + assert result.returncode == 0, result.stderr + + +@pytest.mark.parametrize( + "ready_body", + [ + _ready_body(None), + _ready_body("0001", expected_revision="0002"), + json.dumps({"status": "ready"}), + ], + ids=["unreachable", "behind", "unconfigured"], +) +def test_the_smoke_rejects_a_database_not_at_the_expected_revision( + tmp_path, ready_body +): + result = _run_smoke(tmp_path, ready_body, require="true") + + assert result.returncode == 1 + assert "simulation database" in result.stderr + + +def _run_smoke_with_ready_bodies(tmp_path, bodies): + bodies_file = tmp_path / "ready-bodies" + bodies_file.write_text("\n".join(bodies) + "\n", encoding="utf-8") + state_file, revisions_file = write_fake_smoke_curl(tmp_path, ["revision-1"]) + environment = smoke_test_env(tmp_path, state_file, revisions_file) | { + "FAKE_CURL_READY_BODIES": str(bodies_file), + "REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION": "true", + "SIMULATION_DATABASE_READY_MAX_ATTEMPTS": "3", + } + result = subprocess.run( + ["bash", SMOKE_SCRIPT, "https://candidate.example.test", "revision-1"], + check=False, + capture_output=True, + text=True, + env=environment, + ) + ready_requests = int(Path(f"{bodies_file}.count").read_text(encoding="utf-8")) + return result, ready_requests + + +def test_the_smoke_waits_for_a_database_a_cold_instance_has_not_reached(tmp_path): + result, ready_requests = _run_smoke_with_ready_bodies( + tmp_path, [_ready_body(None), _ready_body(None), _ready_body("0001")] + ) + + assert result.returncode == 0, result.stderr + assert ready_requests == 3 + + +def test_the_smoke_stops_waiting_for_an_unreachable_database(tmp_path): + result, ready_requests = _run_smoke_with_ready_bodies(tmp_path, [_ready_body(None)]) + + assert result.returncode == 1 + assert ready_requests == 3 + assert "simulation database" in result.stderr + + +def test_the_smoke_does_not_wait_for_a_database_at_another_revision(tmp_path): + result, ready_requests = _run_smoke_with_ready_bodies( + tmp_path, [_ready_body("0001", expected_revision="0002"), _ready_body("0002")] + ) + + assert result.returncode == 1 + assert ready_requests == 1 + + +def test_stable_and_rollback_smokes_ignore_the_database_revision(tmp_path): + result = _run_smoke(tmp_path, _ready_body(None), require=None) + + assert result.returncode == 0, result.stderr + + +def test_the_smoke_refuses_an_unknown_database_requirement(tmp_path): + result = _run_smoke(tmp_path, _ready_body("0001"), require="yes") + + assert result.returncode == 2 + assert "REQUIRE_CURRENT_SIMULATION_DATABASE_REVISION" in result.stderr diff --git a/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py b/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py index daf0a01c1..25e967219 100644 --- a/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py +++ b/projects/policyengine-simulation-entry/tests/test_stage12_schema_authority.py @@ -2,7 +2,9 @@ DDL_PATTERN is shared with sim.boundaries.sqlalchemy-and-alembic-only, which matches it against the string literals of every source directory; see -database_access_support.py. +database_access_support.py. The Simulation Entrypoint's own schema has its own +ownership rules, checked in +libs/policyengine-simulation-persistence/tests/test_ownership_boundaries.py. """ from __future__ import annotations @@ -41,12 +43,6 @@ def test_rule_proof_stage12_no_schema_ddl() -> None: assert DDL_PATTERN.search(good) is None -def test_simulation_repository_has_no_schema_migration_chain() -> None: - assert not list(REPOSITORY_ROOT.glob("alembic*.ini")) - assert not list(REPOSITORY_ROOT.rglob("alembic/versions")) - assert not list(REPOSITORY_ROOT.rglob("migrations/versions")) - - def test_stage12_runtime_cannot_create_production_user_associations() -> None: forbidden = { "user_simulation", diff --git a/projects/policyengine-simulation-entry/uv.lock b/projects/policyengine-simulation-entry/uv.lock index c3bfa7d53..f967a66c4 100644 --- a/projects/policyengine-simulation-entry/uv.lock +++ b/projects/policyengine-simulation-entry/uv.lock @@ -901,6 +901,7 @@ dependencies = [ { name = "policyengine-observability", extra = ["fastapi", "google", "httpx", "otlp-grpc"] }, { name = "policyengine-simulation-contract" }, { name = "policyengine-simulation-observability" }, + { name = "policyengine-simulation-persistence" }, { name = "policyengine-stage12-persistence" }, { name = "pyjwt" }, { name = "uvicorn" }, @@ -928,6 +929,7 @@ requires-dist = [ { name = "policyengine-observability", extras = ["fastapi", "google", "httpx", "otlp-grpc"], specifier = ">=3.0.4,<4" }, { name = "policyengine-simulation-contract", editable = "../../libs/policyengine-simulation-contract" }, { name = "policyengine-simulation-observability", editable = "../../libs/policyengine-simulation-observability" }, + { name = "policyengine-simulation-persistence", editable = "../../libs/policyengine-simulation-persistence" }, { name = "policyengine-stage12-persistence", editable = "../../libs/policyengine-stage12-persistence" }, { name = "pyjwt", specifier = ">=2.10.1,<3" }, { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, @@ -963,6 +965,30 @@ requires-dist = [ ] provides-extras = ["test", "build"] +[[package]] +name = "policyengine-simulation-persistence" +version = "0.1.0" +source = { editable = "../../libs/policyengine-simulation-persistence" } +dependencies = [ + { name = "psycopg", extra = ["binary"] }, + { name = "sqlalchemy" }, +] + +[package.metadata] +requires-dist = [ + { name = "alembic", marker = "extra == 'migrations'", specifier = ">=1.17.1,<2" }, + { name = "alembic", marker = "extra == 'test'", specifier = ">=1.17.1,<2" }, + { name = "psycopg", extras = ["binary"], specifier = ">=3.2,<4" }, + { name = "pydantic", marker = "extra == 'provisioning'", specifier = ">=2.11,<3" }, + { name = "pydantic", marker = "extra == 'test'", specifier = ">=2.11,<3" }, + { name = "pyright", marker = "extra == 'build'", specifier = ">=1.1.401" }, + { name = "pytest", marker = "extra == 'test'", specifier = ">=8.3.4" }, + { name = "pytest-cov", marker = "extra == 'test'", specifier = ">=6.1.1" }, + { name = "pyyaml", marker = "extra == 'test'", specifier = ">=6.0.2" }, + { name = "sqlalchemy", specifier = ">=2,<3" }, +] +provides-extras = ["migrations", "provisioning", "test", "build"] + [[package]] name = "policyengine-stage12-persistence" version = "0.1.0" diff --git a/ruff.toml b/ruff.toml index 38c26d845..6cc798b35 100644 --- a/ruff.toml +++ b/ruff.toml @@ -9,7 +9,8 @@ per-file-target-version = { "scripts/**" = "py311", "projects/policyengine-simulation-executor/**" = "py313" } # Directories that are not regular packages on purpose, so they have no # __init__.py: standalone scripts, the implicit namespace package that holds -# scripts.rules, and proof fixtures. Ruff resolves modules as if each listed +# scripts.rules, proof fixtures, and the Alembic chain, whose environment and +# revisions Alembic loads by path. Ruff resolves modules as if each listed # directory, and every directory under it, had an __init__.py. This is a # top-level setting: ruff rejects it in [lint]. The executor's src/modal/utils # scripts are not listed, because ruff would then treat them as part of a @@ -17,6 +18,7 @@ per-file-target-version = { "scripts/**" = "py311", "projects/policyengine-simul # an inline INP001 suppression instead. namespace-packages = [ ".github/scripts", + "migrations", "projects/policyengine-simulation-executor/scripts", "rules/fixtures", "scripts", diff --git a/rules/fixtures/sim.deploy.container-context-minimal/bad.dockerignore b/rules/fixtures/sim.deploy.container-context-minimal/bad.dockerignore new file mode 100644 index 000000000..d8ed8cfbb --- /dev/null +++ b/rules/fixtures/sim.deploy.container-context-minimal/bad.dockerignore @@ -0,0 +1,4 @@ +# Proof fixture: the context leaves out one library the project installs. +libs/* +!libs/fixture-contract/ +!libs/fixture-contract/** diff --git a/rules/fixtures/sim.deploy.container-context-minimal/good.dockerignore b/rules/fixtures/sim.deploy.container-context-minimal/good.dockerignore new file mode 100644 index 000000000..d80779a02 --- /dev/null +++ b/rules/fixtures/sim.deploy.container-context-minimal/good.dockerignore @@ -0,0 +1,6 @@ +# Proof fixture: the context includes every library the project installs. +libs/* +!libs/fixture-contract/ +!libs/fixture-contract/** +!libs/fixture-persistence/ +!libs/fixture-persistence/** diff --git a/rules/fixtures/sim.deploy.container-context-minimal/pyproject.toml b/rules/fixtures/sim.deploy.container-context-minimal/pyproject.toml new file mode 100644 index 000000000..aa2c53cc7 --- /dev/null +++ b/rules/fixtures/sim.deploy.container-context-minimal/pyproject.toml @@ -0,0 +1,9 @@ +# Proof fixture: a project that installs two local libraries by path. +[project] +name = "fixture-entrypoint" +version = "0.1.0" +dependencies = ["fixture-contract", "fixture-persistence"] + +[tool.uv.sources] +fixture-contract = { path = "../../libs/fixture-contract", editable = true } +fixture-persistence = { path = "../../libs/fixture-persistence", editable = true } diff --git a/rules/fixtures/sim.deploy.workflow-shell-safety/bad_deploy_script.sh b/rules/fixtures/sim.deploy.workflow-shell-safety/bad_deploy_script.sh new file mode 100644 index 000000000..73d926095 --- /dev/null +++ b/rules/fixtures/sim.deploy.workflow-shell-safety/bad_deploy_script.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +# Proof fixture: one array assignment resolves two secrets, so a failure to +# resolve the first is ignored and the deploy goes on with an empty reference. +set -euo pipefail + +runtime_secrets=( + "OLD_GATEWAY_AUTH_CLIENT_SECRET=$(resolve_secret_reference "${OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME}")" + "SIMULATION_DATABASE_URL=$(resolve_secret_reference "${SIMULATION_DATABASE_URL_SECRET_NAME}")" +) diff --git a/rules/fixtures/sim.deploy.workflow-shell-safety/good_deploy_script.sh b/rules/fixtures/sim.deploy.workflow-shell-safety/good_deploy_script.sh new file mode 100644 index 000000000..6edd73590 --- /dev/null +++ b/rules/fixtures/sim.deploy.workflow-shell-safety/good_deploy_script.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Proof fixture: each secret is resolved in its own assignment, so set -e stops +# the script at the first failure. +set -euo pipefail + +runtime_secrets=() + +add_runtime_secret() { + local variable_name="$1" + local resolved_reference + resolved_reference="$(resolve_secret_reference "$2")" + runtime_secrets+=("${variable_name}=${resolved_reference}") +} + +add_runtime_secret OLD_GATEWAY_AUTH_CLIENT_SECRET "${OLD_GATEWAY_AUTH_CLIENT_SECRET_SECRET_NAME}" +add_runtime_secret SIMULATION_DATABASE_URL "${SIMULATION_DATABASE_URL_SECRET_NAME}" diff --git a/rules/fixtures/sim.persistence.executor-independent/bad_pyproject.toml b/rules/fixtures/sim.persistence.executor-independent/bad_pyproject.toml new file mode 100644 index 000000000..cb1ae0f5a --- /dev/null +++ b/rules/fixtures/sim.persistence.executor-independent/bad_pyproject.toml @@ -0,0 +1,7 @@ +# Proof fixture: a Modal project that depends on the persistence library. +[project] +name = "policyengine-simulation-executor" +dependencies = [ + "policyengine-simulation-contract", + "policyengine-simulation-persistence", +] diff --git a/rules/fixtures/sim.persistence.executor-independent/good_pyproject.toml b/rules/fixtures/sim.persistence.executor-independent/good_pyproject.toml new file mode 100644 index 000000000..b75b2ff09 --- /dev/null +++ b/rules/fixtures/sim.persistence.executor-independent/good_pyproject.toml @@ -0,0 +1,6 @@ +# Proof fixture: a Modal project that depends only on the contract library. +[project] +name = "policyengine-simulation-executor" +dependencies = [ + "policyengine-simulation-contract", +] diff --git a/rules/fixtures/sim.persistence.migrate-before-entrypoint/bad_workflow.yml b/rules/fixtures/sim.persistence.migrate-before-entrypoint/bad_workflow.yml new file mode 100644 index 000000000..f6f66bd65 --- /dev/null +++ b/rules/fixtures/sim.persistence.migrate-before-entrypoint/bad_workflow.yml @@ -0,0 +1,19 @@ +# Proof fixture: the entrypoint deploys without waiting for the migration, +# the migrator URL reaches every step of the job, and revisions are applied +# from whatever branch dispatched the deployment. +name: Bad deployment +on: workflow_dispatch +jobs: + migrate_simulation_database: + runs-on: ubuntu-latest + env: + SIMULATION_MIGRATION_DATABASE_URL: ${{ secrets.SIMULATION_MIGRATION_DATABASE_URL }} + steps: + - name: Apply simulation schema migrations + env: + SIMULATION_MIGRATION_MODE: upgrade + run: ../../.github/scripts/migrate-simulation-database.sh + deploy_entrypoint: + runs-on: ubuntu-latest + steps: + - run: .github/scripts/deploy-cloud-run-simulation-entry.sh diff --git a/rules/fixtures/sim.persistence.migrate-before-entrypoint/good_workflow.yml b/rules/fixtures/sim.persistence.migrate-before-entrypoint/good_workflow.yml new file mode 100644 index 000000000..c45ddcaf2 --- /dev/null +++ b/rules/fixtures/sim.persistence.migrate-before-entrypoint/good_workflow.yml @@ -0,0 +1,20 @@ +# Proof fixture: the migration job alone receives the migrator URL, in its +# migration step; it applies revisions only from main; and the entrypoint +# deploys only after it succeeded. +name: Good deployment +on: workflow_dispatch +jobs: + migrate_simulation_database: + runs-on: ubuntu-latest + steps: + - name: Apply simulation schema migrations + env: + SIMULATION_MIGRATION_DATABASE_URL: ${{ secrets.SIMULATION_MIGRATION_DATABASE_URL }} + SIMULATION_MIGRATION_MODE: ${{ github.ref == 'refs/heads/main' && 'upgrade' || 'verify' }} + run: ../../.github/scripts/migrate-simulation-database.sh + deploy_entrypoint: + if: ${{ always() && needs.migrate_simulation_database.result == 'success' }} + needs: [migrate_simulation_database] + runs-on: ubuntu-latest + steps: + - run: .github/scripts/deploy-cloud-run-simulation-entry.sh diff --git a/rules/fixtures/sim.persistence.migration-cycle/bad_chain/env.py b/rules/fixtures/sim.persistence.migration-cycle/bad_chain/env.py new file mode 100644 index 000000000..c9aa31716 --- /dev/null +++ b/rules/fixtures/sim.persistence.migration-cycle/bad_chain/env.py @@ -0,0 +1,20 @@ +"""Proof fixture environment: applies this chain to the migration database.""" + +import os + +from alembic import context +from sqlalchemy import create_engine + +url = os.environ["SIMULATION_MIGRATION_DATABASE_URL"].replace( + "postgresql://", "postgresql+psycopg://", 1 +) +engine = create_engine(url) +with engine.begin() as connection: + context.configure( + connection=connection, + target_metadata=None, + version_table="proof_bad_chain_version", + ) + with context.begin_transaction(): + context.run_migrations() +engine.dispose() diff --git a/rules/fixtures/sim.persistence.migration-cycle/bad_chain/versions/0001_proof_table.py b/rules/fixtures/sim.persistence.migration-cycle/bad_chain/versions/0001_proof_table.py new file mode 100644 index 000000000..7b0dde780 --- /dev/null +++ b/rules/fixtures/sim.persistence.migration-cycle/bad_chain/versions/0001_proof_table.py @@ -0,0 +1,21 @@ +"""Proof fixture: a revision whose downgrade leaves its table behind. + +Upgrading again after the downgrade fails, because the table still exists. +""" + +import sqlalchemy as sa +from alembic import op + +revision = "0001" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Create the proof table.""" + op.create_table("proof_bad_table", sa.Column("id", sa.Integer(), primary_key=True)) + + +def downgrade() -> None: + """Leave the proof table behind, which the cycle must catch.""" diff --git a/rules/fixtures/sim.persistence.migration-cycle/good_chain/env.py b/rules/fixtures/sim.persistence.migration-cycle/good_chain/env.py new file mode 100644 index 000000000..f2a7f3950 --- /dev/null +++ b/rules/fixtures/sim.persistence.migration-cycle/good_chain/env.py @@ -0,0 +1,20 @@ +"""Proof fixture environment: applies this chain to the migration database.""" + +import os + +from alembic import context +from sqlalchemy import create_engine + +url = os.environ["SIMULATION_MIGRATION_DATABASE_URL"].replace( + "postgresql://", "postgresql+psycopg://", 1 +) +engine = create_engine(url) +with engine.begin() as connection: + context.configure( + connection=connection, + target_metadata=None, + version_table="proof_good_chain_version", + ) + with context.begin_transaction(): + context.run_migrations() +engine.dispose() diff --git a/rules/fixtures/sim.persistence.migration-cycle/good_chain/versions/0001_proof_table.py b/rules/fixtures/sim.persistence.migration-cycle/good_chain/versions/0001_proof_table.py new file mode 100644 index 000000000..06869afa1 --- /dev/null +++ b/rules/fixtures/sim.persistence.migration-cycle/good_chain/versions/0001_proof_table.py @@ -0,0 +1,19 @@ +"""Proof fixture: a revision whose downgrade undoes its upgrade.""" + +import sqlalchemy as sa +from alembic import op + +revision = "0001" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + """Create the proof table.""" + op.create_table("proof_good_table", sa.Column("id", sa.Integer(), primary_key=True)) + + +def downgrade() -> None: + """Drop the proof table.""" + op.drop_table("proof_good_table") diff --git a/rules/fixtures/sim.persistence.no-modal-database-credentials/bad_workflow.yml b/rules/fixtures/sim.persistence.no-modal-database-credentials/bad_workflow.yml new file mode 100644 index 000000000..739dbc28b --- /dev/null +++ b/rules/fixtures/sim.persistence.no-modal-database-credentials/bad_workflow.yml @@ -0,0 +1,11 @@ +# Proof fixture: a step that copies the simulation database URL into a Modal secret. +name: Bad deployment +on: workflow_dispatch +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Synchronize Modal runtime secrets + env: + SIMULATION_DATABASE_URL: ${{ secrets.SIMULATION_DATABASE_URL }} + run: uv run modal secret create simulation-database "SIMULATION_DATABASE_URL=${SIMULATION_DATABASE_URL}" --force diff --git a/rules/fixtures/sim.persistence.no-modal-database-credentials/good_workflow.yml b/rules/fixtures/sim.persistence.no-modal-database-credentials/good_workflow.yml new file mode 100644 index 000000000..9a2318ce2 --- /dev/null +++ b/rules/fixtures/sim.persistence.no-modal-database-credentials/good_workflow.yml @@ -0,0 +1,16 @@ +# Proof fixture: Modal receives only the Stage 12 secret name; the simulation +# migration URL reaches the migration step alone. +name: Good deployment +on: workflow_dispatch +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - name: Synchronize the separate Stage 12 runtime secret + env: + STAGE12_DATABASE_URL_SECRET_NAME: ${{ vars.STAGE12_DATABASE_URL_SECRET_NAME }} + run: uv run modal secret create stage12-runtime "STAGE12_DATABASE_URL_SECRET_NAME=${STAGE12_DATABASE_URL_SECRET_NAME}" --force + - name: Apply simulation schema migrations + env: + SIMULATION_MIGRATION_DATABASE_URL: ${{ secrets.SIMULATION_MIGRATION_DATABASE_URL }} + run: .github/scripts/simulation-database-migrate.sh diff --git a/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/alembic.ini b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/alembic.ini new file mode 100644 index 000000000..6cb09e697 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/alembic.ini @@ -0,0 +1,2 @@ +[alembic] +script_location = %(here)s diff --git a/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/env.py b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/env.py new file mode 100644 index 000000000..ab2608882 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/env.py @@ -0,0 +1,7 @@ +"""Proof fixture: an Alembic environment that versions the simulation schema.""" + +from alembic import context + +SIMULATION_SCHEMA = "simulation" + +context.configure(version_table_schema=SIMULATION_SCHEMA) diff --git a/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/versions/0001_initial.py b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/versions/0001_initial.py new file mode 100644 index 000000000..5d6301f03 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/migrations/versions/0001_initial.py @@ -0,0 +1,4 @@ +"""Proof fixture: the chain's first revision.""" + +revision = "0001" +down_revision = None diff --git a/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/alembic.ini b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/alembic.ini new file mode 100644 index 000000000..6f4ed3585 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/alembic.ini @@ -0,0 +1,2 @@ +[alembic] +script_location = %(here)s/migrations diff --git a/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/migrations/versions/0001_service.py b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/migrations/versions/0001_service.py new file mode 100644 index 000000000..3a3f03b05 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/bad_repository/projects/service/migrations/versions/0001_service.py @@ -0,0 +1,4 @@ +"""Proof fixture: a revision in a second, forbidden chain.""" + +revision = "0001" +down_revision = None diff --git a/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/alembic.ini b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/alembic.ini new file mode 100644 index 000000000..6cb09e697 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/alembic.ini @@ -0,0 +1,2 @@ +[alembic] +script_location = %(here)s diff --git a/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/env.py b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/env.py new file mode 100644 index 000000000..ab2608882 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/env.py @@ -0,0 +1,7 @@ +"""Proof fixture: an Alembic environment that versions the simulation schema.""" + +from alembic import context + +SIMULATION_SCHEMA = "simulation" + +context.configure(version_table_schema=SIMULATION_SCHEMA) diff --git a/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/versions/0001_initial.py b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/versions/0001_initial.py new file mode 100644 index 000000000..5d6301f03 --- /dev/null +++ b/rules/fixtures/sim.persistence.one-migration-chain/good_repository/migrations/versions/0001_initial.py @@ -0,0 +1,4 @@ +"""Proof fixture: the chain's first revision.""" + +revision = "0001" +down_revision = None diff --git a/rules/registry.toml b/rules/registry.toml index f298eb988..ea16e6a58 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -21,6 +21,10 @@ title = "Service and package boundaries" id = "stage12" title = "Stage 12 persistence" +[[category]] +id = "persistence" +title = "Simulation persistence" + [[category]] id = "deployment" title = "Deployment pipeline" @@ -112,7 +116,7 @@ id = "sim.boundaries.sqlalchemy-and-alembic-only" category = "boundaries" statement = "Code connects to databases only through SQLAlchemy engines and connections, never by importing a database driver directly, and runtime code never changes the schema, which changes only through Alembic migrations." rationale = "docs/engineering/skills/database-access.md#rules" -scope = ["projects/*/src/**/*.py", "libs/*/src/**/*.py", ".github/scripts/**/*.py"] +scope = ["projects/*/src/**/*.py", "libs/*/src/**/*.py", ".github/scripts/**/*.py", "migrations/**/*.py"] tier = "mechanical" status = "enforced" autofix = false @@ -133,48 +137,165 @@ good_fixture = "rules/fixtures/sim.boundaries.sqlalchemy-and-alembic-only/good_d project = "projects/policyengine-simulation-entry" test = "tests/test_database_access_boundaries.py::test_rule_proof_sqlalchemy_and_alembic_only" -# --- Stage 12 persistence -------------------------------------------------- +# --- Simulation persistence ----------------------------------------------- [[rule]] -id = "sim.stage12.no-schema-ddl" -category = "stage12" -statement = "Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema." -rationale = "libs/policyengine-stage12-persistence/README.md" -scope = ["libs/policyengine-stage12-persistence/src/**/*.py", "projects/policyengine-simulation-entry/src/**/*.py", "projects/policyengine-simulation-executor/src/**/*.py"] +id = "sim.persistence.one-migration-chain" +category = "persistence" +statement = "The simulation schema changes only through the one Alembic chain at migrations/, configured by migrations/alembic.ini and versioned in simulation.alembic_version; no other Alembic configuration, environment or revision directory exists." +rationale = "libs/policyengine-simulation-persistence/README.md#ownership-rules" +scope = ["migrations/env.py", "migrations/alembic.ini", "migrations/versions/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "libs/policyengine-simulation-persistence" +nodes = ["tests/test_ownership_boundaries.py::test_repository_has_one_simulation_migration_chain"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run --frozen pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.persistence.one-migration-chain/bad_repository" +good_fixture = "rules/fixtures/sim.persistence.one-migration-chain/good_repository" +project = "libs/policyengine-simulation-persistence" +test = "tests/test_ownership_boundaries.py::test_rule_proof_one_migration_chain" + +[[rule]] +id = "sim.persistence.migration-cycle" +category = "persistence" +statement = "On a fresh PostgreSQL, the simulation chain upgrades to its head, passes alembic check, downgrades to base and upgrades again, run by a non-superuser migrator, and the runtime role can read the revision but cannot change the schema." +rationale = "libs/policyengine-simulation-persistence/README.md#tests" +scope = ["migrations/versions/*.py", "libs/policyengine-simulation-persistence/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "libs/policyengine-simulation-persistence" +nodes = [ + "tests/test_migration_cycle.py::test_upgrade_check_downgrade_upgrade_cycle", + "tests/test_migration_cycle.py::test_the_version_table_lives_only_in_the_simulation_schema", + "tests/test_migration_cycle.py::test_the_runtime_role_reads_the_revision_but_cannot_change_the_schema", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "simulation-database" +invocation = "uv run --frozen pytest tests/ -v -m postgres" +[rule.proof] +bad_fixture = "rules/fixtures/sim.persistence.migration-cycle/bad_chain/versions/0001_proof_table.py" +good_fixture = "rules/fixtures/sim.persistence.migration-cycle/good_chain/versions/0001_proof_table.py" +project = "libs/policyengine-simulation-persistence" +test = "tests/test_migration_cycle.py::test_rule_proof_migration_cycle" + +[[rule]] +id = "sim.persistence.executor-independent" +category = "persistence" +statement = "The Modal executor and gateway never declare, lock or import policyengine-simulation-persistence; only the Simulation Entrypoint and the migration job install it." +rationale = "libs/policyengine-simulation-persistence/README.md#ownership-rules" +scope = ["projects/policyengine-simulation-executor/pyproject.toml", "projects/policyengine-simulation-gateway/pyproject.toml", "projects/policyengine-simulation-executor/src/**/*.py", "projects/policyengine-simulation-gateway/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "libs/policyengine-simulation-persistence" +nodes = ["tests/test_ownership_boundaries.py::test_modal_projects_never_depend_on_the_persistence_library"] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run --frozen pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.persistence.executor-independent/bad_pyproject.toml" +good_fixture = "rules/fixtures/sim.persistence.executor-independent/good_pyproject.toml" +project = "libs/policyengine-simulation-persistence" +test = "tests/test_ownership_boundaries.py::test_rule_proof_executor_independent" + +[[rule]] +id = "sim.persistence.no-modal-database-credentials" +category = "persistence" +statement = "No Modal secret, Modal deployment or Modal application receives a database URL other than the comparison path's own STAGE12_DATABASE_URL, which Modal may hold until the comparison path is removed; in particular never SIMULATION_DATABASE_URL or SIMULATION_MIGRATION_DATABASE_URL." +rationale = "libs/policyengine-simulation-persistence/README.md#ownership-rules" +scope = [".github/workflows/*.yml", ".github/scripts/*.sh", "projects/policyengine-simulation-executor/src/**/*.py", "projects/policyengine-simulation-gateway/src/**/*.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +project = "libs/policyengine-simulation-persistence" +nodes = [ + "tests/test_ownership_boundaries.py::test_modal_receives_no_database_url_but_the_comparison_paths", + "tests/test_ownership_boundaries.py::test_modal_steps_may_carry_only_the_comparison_paths_database_url", + "tests/test_ownership_boundaries.py::test_the_comparison_paths_database_url_may_reach_modal_until_it_is_removed", + "tests/test_ownership_boundaries.py::test_a_modal_script_may_not_carry_another_database_url", + "tests/test_ownership_boundaries.py::test_modal_sources_may_name_only_the_comparison_paths_database_url", + "tests/test_ownership_boundaries.py::test_a_temporary_modal_app_may_not_receive_another_database_url", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "uv run --frozen pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.persistence.no-modal-database-credentials/bad_workflow.yml" +good_fixture = "rules/fixtures/sim.persistence.no-modal-database-credentials/good_workflow.yml" +project = "libs/policyengine-simulation-persistence" +test = "tests/test_ownership_boundaries.py::test_rule_proof_no_modal_database_credentials" + +[[rule]] +id = "sim.persistence.migrate-before-entrypoint" +category = "persistence" +statement = "Each deployment migrates the simulation schema in its own job, the only one that receives SIMULATION_MIGRATION_DATABASE_URL, before the entrypoint candidate deploys, applying revisions only from main and otherwise requiring the database at the chain's head; the entrypoint receives SIMULATION_DATABASE_URL as a pinned Secret Manager version; and the candidate smoke requires the database at the image's revision." +rationale = "libs/policyengine-simulation-persistence/README.md#deployment" +scope = [".github/workflows/simulation-deploy.reusable.yml", ".github/scripts/migrate-simulation-database.sh", ".github/scripts/deploy-cloud-run-simulation-entry.sh", ".github/scripts/cloud-run-simulation-entry-smoke.sh"] tier = "mechanical" status = "enforced" autofix = false [rule.enforcer] kind = "pytest" project = "projects/policyengine-simulation-entry" -nodes = ["tests/test_stage12_schema_authority.py::test_simulation_runtimes_contain_no_schema_ddl"] +nodes = [ + "tests/test_simulation_database_deployment.py::test_the_deploy_workflow_follows_the_migration_wiring", + "tests/test_simulation_database_deployment.py::test_the_migration_job_runs_before_the_entrypoint_deploys", + "tests/test_simulation_database_deployment.py::test_only_the_migration_step_receives_the_migrator_url", + "tests/test_simulation_database_deployment.py::test_only_the_default_branch_applies_migrations", + "tests/test_simulation_database_deployment.py::test_the_entrypoint_receives_the_runtime_url_as_a_pinned_secret", + "tests/test_simulation_database_deployment.py::test_the_candidate_smoke_requires_the_current_database_revision", +] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" invocation = "uv run --frozen pytest tests/" [rule.proof] -bad_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py" -good_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py" +bad_fixture = "rules/fixtures/sim.persistence.migrate-before-entrypoint/bad_workflow.yml" +good_fixture = "rules/fixtures/sim.persistence.migrate-before-entrypoint/good_workflow.yml" project = "projects/policyengine-simulation-entry" -test = "tests/test_stage12_schema_authority.py::test_rule_proof_stage12_no_schema_ddl" +test = "tests/test_simulation_database_deployment.py::test_rule_proof_migrate_before_entrypoint" + +# --- Stage 12 persistence -------------------------------------------------- [[rule]] -id = "sim.stage12.no-migration-chain" +id = "sim.stage12.no-schema-ddl" category = "stage12" -statement = "This repository has no schema migration chain (no Alembic); Stage 12 tables are migrated by policyengine-api." +statement = "Simulation runtimes and the Stage 12 persistence library contain no schema DDL (CREATE, ALTER, DROP or TRUNCATE of tables, types, schemas or databases, or create_all); policyengine-api owns the schema." rationale = "libs/policyengine-stage12-persistence/README.md" -scope = ["projects/*/src/**/*.py", "libs/*/src/**/*.py"] +scope = ["libs/policyengine-stage12-persistence/src/**/*.py", "projects/policyengine-simulation-entry/src/**/*.py", "projects/policyengine-simulation-executor/src/**/*.py"] tier = "mechanical" status = "enforced" autofix = false [rule.enforcer] kind = "pytest" project = "projects/policyengine-simulation-entry" -nodes = ["tests/test_stage12_schema_authority.py::test_simulation_repository_has_no_schema_migration_chain"] +nodes = ["tests/test_stage12_schema_authority.py::test_simulation_runtimes_contain_no_schema_ddl"] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" invocation = "uv run --frozen pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/bad_stage12_runtime.py" +good_fixture = "rules/fixtures/sim.stage12.no-schema-ddl/good_stage12_runtime.py" +project = "projects/policyengine-simulation-entry" +test = "tests/test_stage12_schema_authority.py::test_rule_proof_stage12_no_schema_ddl" [[rule]] id = "sim.stage12.no-production-user-associations" @@ -317,9 +438,9 @@ invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.deploy.workflow-shell-safety" category = "deployment" -statement = "Cloud Run deployment never interpolates GitHub expressions into shell, escapes environment values, and pins secret versions." +statement = "Cloud Run deployment never interpolates GitHub expressions into shell, escapes environment values, pins secret versions, stops before deploying when any runtime secret cannot be resolved, and resolves the Stage 12 secrets with the Stage 12 block's own helper." rationale = "docs/operations" -scope = [".github/workflows/*.yml"] +scope = [".github/workflows/*.yml", ".github/scripts/deploy-cloud-run-simulation-entry.sh"] tier = "mechanical" status = "enforced" autofix = false @@ -329,11 +450,19 @@ project = "projects/policyengine-simulation-entry" nodes = [ "tests/test_deployment_assets_cloud_run.py::test_cloud_run_deployment_does_not_interpolate_github_values_in_shell", "tests/test_deployment_assets_cloud_run.py::test_cloud_run_deployment_escapes_environment_and_pins_secret_versions", + "tests/test_deployment_assets_cloud_run.py::test_cloud_run_deployment_stops_when_a_runtime_secret_cannot_be_resolved", + "tests/test_deployment_assets_cloud_run.py::test_cloud_run_deployment_resolves_each_secret_in_its_own_assignment", + "tests/test_deployment_assets_cloud_run.py::test_stage12_secrets_are_resolved_by_the_stage12_block_alone", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" invocation = "uv run --frozen pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.deploy.workflow-shell-safety/bad_deploy_script.sh" +good_fixture = "rules/fixtures/sim.deploy.workflow-shell-safety/good_deploy_script.sh" +project = "projects/policyengine-simulation-entry" +test = "tests/test_deployment_assets_cloud_run.py::test_rule_proof_workflow_shell_safety" [[rule]] id = "sim.deploy.revision-tags-valid" @@ -533,9 +662,9 @@ invocation = "uv run --frozen pytest tests/" [[rule]] id = "sim.deploy.container-context-minimal" category = "deployment" -statement = "Container builds exclude local environments and unrelated projects, and the entrypoint container builds for the requested target platform." +statement = "Container builds exclude local environments and unrelated projects but include every local library the entrypoint installs, and the entrypoint container builds for the requested target platform." rationale = "docs/operations" -scope = ["projects/policyengine-simulation-entry/Dockerfile"] +scope = ["projects/policyengine-simulation-entry/Dockerfile", "projects/policyengine-simulation-entry/Dockerfile.dockerignore"] tier = "mechanical" status = "enforced" autofix = false @@ -544,12 +673,18 @@ kind = "pytest" project = "projects/policyengine-simulation-entry" nodes = [ "tests/test_deployment_assets_traffic.py::test_container_context_excludes_local_environments_and_unrelated_projects", + "tests/test_deployment_assets_traffic.py::test_container_context_includes_every_local_library_the_entrypoint_installs", "tests/test_deployment_assets_cloud_run.py::test_entrypoint_container_uses_requested_target_platform", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" job = "test" invocation = "uv run --frozen pytest tests/" +[rule.proof] +bad_fixture = "rules/fixtures/sim.deploy.container-context-minimal/bad.dockerignore" +good_fixture = "rules/fixtures/sim.deploy.container-context-minimal/good.dockerignore" +project = "projects/policyengine-simulation-entry" +test = "tests/test_deployment_assets_traffic.py::test_rule_proof_container_context_minimal" [[rule]] id = "sim.deploy.docker-images-build" @@ -1067,11 +1202,12 @@ invocation = "unittest discover -s scripts/rules/tests" [[rule]] id = "sim.quality.pyright-libraries-and-entry" category = "quality" -statement = "The contract, observability, Stage 12 persistence and fastapi libraries and the simulation entry have no Pyright errors, checked with the pinned Pyright version in each package's own locked environment." +statement = "The contract, observability, simulation persistence, Stage 12 persistence and fastapi libraries and the simulation entry have no Pyright errors, checked with the pinned Pyright version in each package's own locked environment." rationale = "docs/engineering/skills/type-checking.md#rules" scope = [ "libs/policyengine-simulation-contract/src/**/*.py", "libs/policyengine-simulation-observability/src/**/*.py", + "libs/policyengine-simulation-persistence/src/**/*.py", "libs/policyengine-stage12-persistence/src/**/*.py", "libs/policyengine-fastapi/src/**/*.py", "projects/policyengine-simulation-entry/src/**/*.py", @@ -1084,6 +1220,7 @@ kind = "pyright" projects = [ "libs/policyengine-simulation-contract", "libs/policyengine-simulation-observability", + "libs/policyengine-simulation-persistence", "libs/policyengine-stage12-persistence", "libs/policyengine-fastapi", "projects/policyengine-simulation-entry",