diff --git a/.github/workflows/v2-integration-check.yml b/.github/workflows/v2-integration-check.yml index 3f402ae80..79d264eee 100644 --- a/.github/workflows/v2-integration-check.yml +++ b/.github/workflows/v2-integration-check.yml @@ -77,8 +77,8 @@ jobs: run: uv run coverage run --branch -m pytest -q tests/integration/test_v2_catalog_installed.py env: RUN_V2_CATALOG_COMPATIBILITY: "1" - - name: Test v2 metadata publication and resource routes - run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_metadata_routes.py + - name: Test v2 metadata publication, resource routes and the countries list + run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_catalog_publication.py tests/integration/test_v2_metadata_routes.py tests/integration/test_v2_country_routes.py - name: Test v2 policy persistence and immediate v1 mirroring run: uv run coverage run -a --branch -m pytest -q tests/integration/test_v2_policy_persistence.py tests/integration/test_v1_policy_dual_write.py tests/integration/test_mysql_policy_dual_write.py tests/integration/test_v2_user_policy_mirroring.py tests/integration/test_v1_user_policy_dual_write.py - name: Test signed-in identity provisioning and user associations diff --git a/changelog.d/v2-countries-route.added.md b/changelog.d/v2-countries-route.added.md new file mode 100644 index 000000000..66b8d8e46 --- /dev/null +++ b/changelog.d/v2-countries-route.added.md @@ -0,0 +1 @@ +Added `GET /v2/countries`, which lists every row of the countries table: each served country's ID and whether it supports household and society-wide calculations. diff --git a/changelog.d/v2-countries-startup-check.added.md b/changelog.d/v2-countries-startup-check.added.md new file mode 100644 index 000000000..8e30ae192 --- /dev/null +++ b/changelog.d/v2-countries-startup-check.added.md @@ -0,0 +1 @@ +The Public API refuses to start when the v2 `countries` table and its country list in code name different countries, or when the table or the privilege to read it is missing. The check runs on Cloud Run and wherever a v2 runtime database is configured; when it cannot query the table, it logs why and startup continues. diff --git a/changelog.d/v2-countries-table.added.md b/changelog.d/v2-countries-table.added.md new file mode 100644 index 000000000..5c7e480ac --- /dev/null +++ b/changelog.d/v2-countries-table.added.md @@ -0,0 +1 @@ +Added a v2 `countries` table that records, for each country the Public API serves, whether it supports household and society-wide calculations. Its revision lists the US and the UK, both with both. Users' primary country and the country of every v2 household, policy and user association now reference this table instead of a fixed US-and-UK check. diff --git a/docs/engineering/migration-contracts.md b/docs/engineering/migration-contracts.md index f39a5e369..f7975a813 100644 --- a/docs/engineering/migration-contracts.md +++ b/docs/engineering/migration-contracts.md @@ -7,8 +7,8 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | Metric | Count | | --- | ---: | | route group count | 9 | -| workflow count | 16 | -| request count | 89 | +| workflow count | 17 | +| request count | 90 | | db entity count | 6 | | sim flow count | 3 | @@ -18,7 +18,7 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | --- | --- | --- | --- | | `health` | `health`, `simulation-gateway-check`, `liveness-check`, `readiness-check` | `none` | `none` | | `specification` | `specification` | `none` | `none` | -| `metadata` | `metadata`, `datasets`, `economy-options`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | +| `metadata` | `metadata`, `countries`, `datasets`, `economy-options`, `parameter-values`, `parameters`, `regions`, `tax-benefit-model-versions`, `tax-benefit-models`, `variables` | `metadata` | `none` | | `policy` | `policy`, `policies`, `user-policy`, `user-policies` | `policy` | `none` | | `household` | `household`, `households`, `user-household`, `user-households`, `calculate`, `calculate-full` | `household` | `household` | | `economy` | `economy` | `simulation` | `economy` | @@ -190,6 +190,15 @@ Generated from `policyengine_api/migration_registry.py` and `tests/contract/regi | `GET` | `/us/metadata` | 200 | `metadata` | none | `status`, `result.current_law_id`, `result.economy_options.region`, `result.economy_options.time_period` | | `GET` | `/uk/metadata` | 200 | `metadata` | none | `status`, `result.current_law_id`, `result.economy_options.region`, `result.economy_options.time_period` | +### `country_selection_v2` + +- Current contract: `typed_v2_resources` +- Future owner: Countries table and one country list in code + +| Method | Path | Status | Route group | Required headers | Stable response fields | +| --- | --- | ---: | --- | --- | --- | +| `GET` | `/v2/countries` | 200 | `metadata` | none | `status`, `message`, `result.items` | + ### `metadata_resources_v2_preview` - Current contract: `typed_v2_resources` diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index d4a043d91..0b5c165b4 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -294,3 +294,33 @@ last dependent table is dropped. Its only post-generation correction drops the nine generated `v2_*` enum types at the end of the baseline downgrade. The v2 Postgres lifecycle test covers empty upgrade, downgrade to base, and re-upgrade so stale enum types cannot make the generated baseline non-reversible. + +### API v2 countries rows + +Revision `fb1a3e40d3d6` was autogenerated after adding the `countries` table +and replacing the US-and-UK checks on `users.primary_country` and on the +`country_id` of households, policies and the four user association tables +with foreign keys to it. It contains one hand-written operation: it inserts +the `uk` and `us` rows, both household and society-wide capable, between the +generated table creation and the generated foreign keys, which otherwise fail +on any database whose rows already name either country. The rows are exactly +the values the replaced checks allowed. The generated downgrade removes them +with the table, and its restored checks fail if any record names another +country. + +Inserting rows is a data operation, which this skill otherwise forbids. The +maintainers approved this insert as a one-off exception, because the new +foreign keys need the rows in the same revision. Unit tests pin the +operation, and the v2 Postgres lifecycle test covers an upgrade that keeps +existing rows, the downgrade, a downgrade refused by another country's +records, and the re-upgrade. + +The table must list exactly the countries in +`policyengine_api/supported_countries.py`: the Public API refuses to start +when they differ (`policyengine_api/startup_country_check.py`). Listing +another country later needs a revision that inserts its row, and the same +human migration decision. The deploy runs that revision before it starts +the new release, so instances of the previous release that start afterwards, +and a rollback to it, refuse to start until the revision is downgraded. The +module docstring of `supported_countries.py` lists the other checks a new +country must widen. diff --git a/docs/generated/migration_contracts.json b/docs/generated/migration_contracts.json index 7a0f1b67d..9ee0774f5 100644 --- a/docs/generated/migration_contracts.json +++ b/docs/generated/migration_contracts.json @@ -1,10 +1,10 @@ { "metadata": { "db_entity_count": 6, - "request_count": 89, + "request_count": 90, "route_group_count": 9, "sim_flow_count": 3, - "workflow_count": 16 + "workflow_count": 17 }, "route_groups": [ { @@ -31,6 +31,7 @@ "name": "metadata", "path_segments": [ "metadata", + "countries", "datasets", "economy-options", "parameter-values", @@ -1348,6 +1349,25 @@ } ] }, + { + "current_contract": "typed_v2_resources", + "future_owner_pr": "Countries table and one country list in code", + "name": "country_selection_v2", + "requests": [ + { + "expected_status": 200, + "headers": [], + "method": "GET", + "path": "/v2/countries", + "route_group": "metadata", + "stable_response_fields": [ + "status", + "message", + "result.items" + ] + } + ] + }, { "current_contract": "typed_v2_resources", "future_owner_pr": "Later metadata read cutover and v2 route-prefix removal", diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 8c3cc5f5d..e7f4bfedb 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -164,7 +164,7 @@ flowchart LR | --- | --- | --- | --- | --- | --- | --- | --- | | `api.catalog.initializer-not-at-startup` | Application startup modules never import the metadata catalog initializer. | mechanical | enforced | yes | pytest `tests/unit/v2/test_catalog_initialization.py` | `pr.yml:test` | **unproven** | | `api.startup.fail-closed-configuration` | Default Cloud SQL startup needs no Supabase configuration, and a selected v2 runtime without its database URL fails closed. | mechanical | enforced | yes | pytest `tests/unit/v2/test_import_side_effects.py` | `pr.yml:test` | **unproven** | -| `api.startup.no-import-side-effects` | Importing v2 modules, starting up and serving a request open no network connections and create no files or runtime SQLite databases. | mechanical | enforced | yes | pytest `tests/unit/v2/test_import_side_effects.py` | `pr.yml:test` | **unproven** | +| `api.startup.no-import-side-effects` | Importing v2 modules opens no network connections, starting up and serving a request open none unless the process runs on Cloud Run or has a v2 runtime database configured, and none of them creates files or runtime SQLite databases. | mechanical | enforced | yes | pytest `tests/unit/v2/test_import_side_effects.py` | `pr.yml:test` | **unproven** | | `api.startup.no-runtime-sqlite-modules` | Runtime SQLite modules are absent from the production package. | mechanical | enforced | yes | pytest `tests/unit/v2/test_import_side_effects.py` | `pr.yml:test` | **unproven** | ```mermaid @@ -294,7 +294,7 @@ flowchart LR | `api.quality.command-line-modules-ruff-exemptions` | Application modules that also run as command-line programs (if __name__ == "__main__") may print their output (T201); every other ruff rule applies to them. | mechanical | enforced | yes | ruff per-file ignore of T201 for `policyengine_api/{data/v2/catalog/initialization,data/v2/household_migration_qualification,data/v2/policy_migration_qualification,data/v2/privilege_reconciliation,data/v2/user_association_migration_qualification,worker_spm_release}.py` | `pr.yml:lint` | — | | `api.quality.command-line-scripts-ruff-exemptions` | Command-line scripts under scripts/ may print their output, run other programs, fetch fixed URLs, stamp today's date and pick random samples (T201, S603, S607, S310, DTZ011, S311); every other ruff rule applies to them. | mechanical | enforced | yes | ruff per-file ignore of DTZ011, S310, S311, S603, S607, T201 for `scripts/**` | `pr.yml:lint` | — | | `api.quality.environment-variables-load` | Required environment variables and credentials load in a deployed-like environment. | mechanical | enforced | yes | CI job `test_env_vars` | `pr.yml:test_env_vars` | — | -| `api.quality.mypy-v2` | Shared query models, FastAPI dependencies, v2 routes and services, and the user sign-in and account-deletion libraries pass mypy with untyped definitions disallowed. | mechanical | enforced | yes | mypy | `pr.yml:typecheck-v2` | — | +| `api.quality.mypy-v2` | Shared query models, FastAPI dependencies, v2 routes and services, the user sign-in and account-deletion libraries, the country list and its startup check pass mypy with untyped definitions disallowed. | mechanical | enforced | yes | mypy | `pr.yml:typecheck-v2` | — | | `api.quality.ruff-format` | All Python files are formatted with ruff format. | mechanical | enforced | yes | ruff format | `pr.yml:lint` | — | | `api.quality.ruff-lint` | All Python code passes ruff check with every ruff rule selected (ALL), with no ratchet, except the ignores in pyproject.toml: COM812 and ISC001, which conflict with ruff format, and D203 and D213, which conflict with the kept D211 and D212. | mechanical | enforced | yes | ruff check (ALL) | `pr.yml:lint` | — | | `api.quality.runtime-evaluated-models-resolve` | Every package model is one ruff treats as runtime-evaluated (a base resolves, through the import in its own module, to a name in runtime-evaluated-base-classes, or is a same-module class that does), and every Pydantic model and SQLAlchemy mapper resolves its annotations once the package is imported. | mechanical | enforced | yes | pytest `tests/unit/test_runtime_evaluated_annotations.py` | `pr.yml:test` | **unproven** | diff --git a/migrations/v2/versions/fb1a3e40d3d6_add_countries_table.py b/migrations/v2/versions/fb1a3e40d3d6_add_countries_table.py new file mode 100644 index 000000000..1b02ec819 --- /dev/null +++ b/migrations/v2/versions/fb1a3e40d3d6_add_countries_table.py @@ -0,0 +1,195 @@ +"""add countries table + +Revision ID: fb1a3e40d3d6 +Revises: fbb7488ebe8a +Create Date: 2026-10-22 02:16:50.126715 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel + + +revision: str = "fb1a3e40d3d6" +down_revision: Union[str, None] = "fbb7488ebe8a" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + countries = op.create_table( + "countries", + sa.Column("id", sqlmodel.sql.sqltypes.AutoString(length=2), nullable=False), + sa.Column("household_enabled", sa.Boolean(), nullable=False), + sa.Column("society_wide_enabled", sa.Boolean(), nullable=False), + sa.CheckConstraint( + "household_enabled OR NOT society_wide_enabled", + name=op.f("ck_countries_society_wide_requires_household"), + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_countries")), + ) + # Approved hand-written data operation: the one exception to generated-only + # revisions, decided by the maintainers for this table alone. The foreign + # keys below fail on any database whose rows already name the US or the UK + # unless both countries are listed first, so their rows are inserted here, + # in the same revision. They are the countries the replaced checks + # allowed, and both support household and society-wide calculations. The + # generated downgrade removes them with the table. Listing another + # country later needs its own revision and the same kind of decision. + op.bulk_insert( + countries, + [ + {"id": "uk", "household_enabled": True, "society_wide_enabled": True}, + {"id": "us", "household_enabled": True, "society_wide_enabled": True}, + ], + ) + op.create_foreign_key( + op.f("fk_households_country_id_countries"), + "households", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint(op.f("ck_households_country"), "households", type_="check") + op.create_foreign_key( + op.f("fk_policies_country_id_countries"), + "policies", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint(op.f("ck_policies_country"), "policies", type_="check") + op.create_foreign_key( + op.f("fk_user_household_associations_country_id_countries"), + "user_household_associations", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint( + op.f("ck_user_household_associations_country"), + "user_household_associations", + type_="check", + ) + op.create_foreign_key( + op.f("fk_user_policies_country_id_countries"), + "user_policies", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint(op.f("ck_user_policies_country"), "user_policies", type_="check") + op.create_foreign_key( + op.f("fk_user_report_associations_country_id_countries"), + "user_report_associations", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint( + op.f("ck_user_report_associations_country"), + "user_report_associations", + type_="check", + ) + op.create_foreign_key( + op.f("fk_user_simulation_associations_country_id_countries"), + "user_simulation_associations", + "countries", + ["country_id"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint( + op.f("ck_user_simulation_associations_country"), + "user_simulation_associations", + type_="check", + ) + op.create_foreign_key( + op.f("fk_users_primary_country_countries"), + "users", + "countries", + ["primary_country"], + ["id"], + ondelete="RESTRICT", + ) + op.drop_constraint(op.f("ck_users_primary_country"), "users", type_="check") + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_check_constraint( + op.f("ck_users_primary_country"), + "users", + "primary_country::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_users_primary_country_countries"), "users", type_="foreignkey" + ) + op.create_check_constraint( + op.f("ck_user_simulation_associations_country"), + "user_simulation_associations", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_user_simulation_associations_country_id_countries"), + "user_simulation_associations", + type_="foreignkey", + ) + op.create_check_constraint( + op.f("ck_user_report_associations_country"), + "user_report_associations", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_user_report_associations_country_id_countries"), + "user_report_associations", + type_="foreignkey", + ) + op.create_check_constraint( + op.f("ck_user_policies_country"), + "user_policies", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_user_policies_country_id_countries"), + "user_policies", + type_="foreignkey", + ) + op.create_check_constraint( + op.f("ck_user_household_associations_country"), + "user_household_associations", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_user_household_associations_country_id_countries"), + "user_household_associations", + type_="foreignkey", + ) + op.create_check_constraint( + op.f("ck_policies_country"), + "policies", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_policies_country_id_countries"), "policies", type_="foreignkey" + ) + op.create_check_constraint( + op.f("ck_households_country"), + "households", + "country_id::text = ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])", + ) + op.drop_constraint( + op.f("fk_households_country_id_countries"), "households", type_="foreignkey" + ) + op.drop_table("countries") + # ### end Alembic commands ### diff --git a/policyengine_api/asgi.py b/policyengine_api/asgi.py index 1f981c56a..fc5809cf1 100644 --- a/policyengine_api/asgi.py +++ b/policyengine_api/asgi.py @@ -10,6 +10,7 @@ from policyengine_api.observability import get_runtime from policyengine_api.readiness import mark_not_ready, mark_ready from policyengine_api.runtime_cache.client import close_runtime_cache_clients +from policyengine_api.startup_country_check import check_countries_table_at_startup from policyengine_api.warmup import run_startup_warmup @@ -24,6 +25,10 @@ def _close_runtime_resources() -> None: shutdown_callback=_close_runtime_resources, ) +# Refuse to start when the v2 countries table and the country list in code +# disagree (see policyengine_api.startup_country_check). +check_countries_table_at_startup() + # Warm the simulation machinery before serving (see policyengine_api.warmup). # POLICYENGINE_API_STARTUP_WARMUP=0 skips it. if os.environ.get("POLICYENGINE_API_STARTUP_WARMUP", "1") != "0": diff --git a/policyengine_api/data/v2/catalog/catalog_selection.py b/policyengine_api/data/v2/catalog/catalog_selection.py index d9b50fc76..1e97b6e3c 100644 --- a/policyengine_api/data/v2/catalog/catalog_selection.py +++ b/policyengine_api/data/v2/catalog/catalog_selection.py @@ -12,8 +12,9 @@ TaxBenefitModel, TaxBenefitModelVersion, ) +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS -SUPPORTED_V2_COUNTRY_IDS = frozenset({"us", "uk"}) +SUPPORTED_V2_COUNTRY_IDS = frozenset(SUPPORTED_COUNTRY_IDS) MAXIMUM_POLICYENGINE_VERSION_LENGTH = 128 diff --git a/policyengine_api/data/v2/catalog/extraction.py b/policyengine_api/data/v2/catalog/extraction.py index 0872eea09..0817e41ae 100644 --- a/policyengine_api/data/v2/catalog/extraction.py +++ b/policyengine_api/data/v2/catalog/extraction.py @@ -27,15 +27,15 @@ RegionRecord, VariableRecord, ) +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS if TYPE_CHECKING: from policyengine_api.json_types import JSONValue -SUPPORTED_COUNTRIES = ("us", "uk") +SUPPORTED_COUNTRIES = SUPPORTED_COUNTRY_IDS REQUIRED_DEPENDENCIES = ( "policyengine-core", - "policyengine-us", - "policyengine-uk", + *(f"policyengine-{country_id}" for country_id in SUPPORTED_COUNTRY_IDS), ) REVIEWED_DEFAULT_DATASETS = { "us": "populace_us_2024", @@ -928,7 +928,7 @@ def extract_installed_catalog() -> NormalizedCatalog: bundle=get_current_bundle(), policyengine_version=policyengine_version, models={ - "us": policyengine.us.model, - "uk": policyengine.uk.model, + country_id: getattr(policyengine, country_id).model + for country_id in SUPPORTED_COUNTRIES }, ) diff --git a/policyengine_api/data/v2/migration_target.py b/policyengine_api/data/v2/migration_target.py index fd4c05fdf..2c7b6986d 100644 --- a/policyengine_api/data/v2/migration_target.py +++ b/policyengine_api/data/v2/migration_target.py @@ -24,7 +24,7 @@ from collections.abc import Mapping V2_ALEMBIC_DISPOSABLE_TEST = "V2_ALEMBIC_DISPOSABLE_TEST" -V2_ALEMBIC_HEAD_REVISION = "fbb7488ebe8a" +V2_ALEMBIC_HEAD_REVISION = "fb1a3e40d3d6" DISPOSABLE_DATABASE_NAME = "policyengine_v2_alembic_test" DISPOSABLE_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "postgres"}) MIGRATION_ROLE = "policyengine_v2_migrator" diff --git a/policyengine_api/data/v2/models/__init__.py b/policyengine_api/data/v2/models/__init__.py index c63655bb6..a3aedae09 100644 --- a/policyengine_api/data/v2/models/__init__.py +++ b/policyengine_api/data/v2/models/__init__.py @@ -31,6 +31,7 @@ Stage12RunStatus, Stage12SimulationRole, ) +from policyengine_api.data.v2.models.countries import Country # noqa: E402 from policyengine_api.data.v2.models.household_mappings import ( # noqa: E402 LegacyHouseholdMapping, ) @@ -102,6 +103,7 @@ "ChangeAggregate", "CongressionalDistrictImpact", "ConstituencyImpact", + "Country", "Dataset", "DatasetVersion", "DecileImpact", diff --git a/policyengine_api/data/v2/models/associations.py b/policyengine_api/data/v2/models/associations.py index 2c5cd21b1..dc7e7d757 100644 --- a/policyengine_api/data/v2/models/associations.py +++ b/policyengine_api/data/v2/models/associations.py @@ -51,10 +51,6 @@ class UserHouseholdAssociation(TimestampedModel, table=True): "country_id", name="uq_user_household_associations_id_country", ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_user_household_associations_country", - ), sa.CheckConstraint( "household_id IS NOT NULL OR legacy_household_id IS NOT NULL", name="ck_user_household_associations_household_ref", @@ -91,7 +87,11 @@ class UserHouseholdAssociation(TimestampedModel, table=True): ) household_id: UUID | None = Field(default=None, index=True) legacy_household_id: int | None = Field(default=None, sa_type=sa.BigInteger) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) name: str | None = Field(default=None, max_length=255) description: str | None = Field(default=None, sa_type=sa.Text) @@ -113,10 +113,6 @@ class UserPolicy(TimestampedModel, table=True): "country_id", name="uq_user_policies_id_country", ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_user_policies_country", - ), sa.CheckConstraint( "policy_id IS NOT NULL OR legacy_policy_id IS NOT NULL", name="ck_user_policies_policy_ref", @@ -157,7 +153,11 @@ class UserPolicy(TimestampedModel, table=True): ) policy_id: UUID | None = Field(default=None, index=True) legacy_policy_id: int | None = Field(default=None, sa_type=sa.BigInteger) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) name: str | None = Field(default=None, max_length=255) description: str | None = Field(default=None, sa_type=sa.Text) baseline: str | None = Field(default=None, max_length=16) @@ -185,10 +185,6 @@ class UserSimulationAssociation(TimestampedModel, table=True): "country_id", name="uq_user_simulation_associations_id_country", ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_user_simulation_associations_country", - ), sa.CheckConstraint( "simulation_id IS NOT NULL OR legacy_simulation_id IS NOT NULL", name="ck_user_simulation_associations_simulation_ref", @@ -214,7 +210,11 @@ class UserSimulationAssociation(TimestampedModel, table=True): ) simulation_id: UUID | None = Field(default=None, index=True) legacy_simulation_id: int | None = Field(default=None, sa_type=sa.BigInteger) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) name: str | None = Field(default=None, max_length=255) description: str | None = Field(default=None, sa_type=sa.Text) # Retired; see the module docstring. @@ -239,10 +239,6 @@ class UserReportAssociation(TimestampedModel, table=True): "country_id", name="uq_user_report_associations_id_country", ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_user_report_associations_country", - ), sa.CheckConstraint( "report_id IS NOT NULL OR legacy_report_id IS NOT NULL", name="ck_user_report_associations_report_ref", @@ -273,7 +269,11 @@ class UserReportAssociation(TimestampedModel, table=True): ) report_id: UUID | None = Field(default=None, index=True) legacy_report_id: int | None = Field(default=None, sa_type=sa.BigInteger) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) name: str | None = Field(default=None, max_length=255) description: str | None = Field(default=None, sa_type=sa.Text) user_policy_id: UUID | None = Field( diff --git a/policyengine_api/data/v2/models/countries.py b/policyengine_api/data/v2/models/countries.py new file mode 100644 index 000000000..fc3ae8677 --- /dev/null +++ b/policyengine_api/data/v2/models/countries.py @@ -0,0 +1,37 @@ +"""Canonical SQLModel table recording what each served country supports. + +Which countries exist is decided in code: ``policyengine_api.supported_countries`` +lists them, and this table must hold exactly one row per listed country. A +row records the calculations its country supports: a household-only country +has ``household_enabled`` set and ``society_wide_enabled`` unset. + +``users.primary_country`` and the ``country_id`` of households, policies and +the four user association tables reference ``countries.id``. Other v2 country +columns do not: the mapping tables that pair v1 records with their v2 copies +keep their US-and-UK checks, and the simulation-domain tables, the comparison +run tables and the retired association ``country`` columns store unconstrained +text. +""" + +import sqlalchemy as sa +from sqlmodel import Field, SQLModel + + +class Country(SQLModel, table=True): + """One served country and the calculations it supports. + + A society-wide country must also be household capable. A country that a + user, household, policy or user association names cannot be deleted. + """ + + __tablename__ = "countries" + __table_args__ = ( + sa.CheckConstraint( + "household_enabled OR NOT society_wide_enabled", + name="ck_countries_society_wide_requires_household", + ), + ) + + id: str = Field(primary_key=True, max_length=2) + household_enabled: bool + society_wide_enabled: bool diff --git a/policyengine_api/data/v2/models/households.py b/policyengine_api/data/v2/models/households.py index 78e76b2ab..181266d9d 100644 --- a/policyengine_api/data/v2/models/households.py +++ b/policyengine_api/data/v2/models/households.py @@ -53,10 +53,6 @@ class Household(TimestampedModel, table=True): "content_hash", name="uq_households_canonicalization_content_hash", ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_households_country", - ), sa.CheckConstraint( "default_year IS NULL OR default_year BETWEEN 1900 AND 2200", name="ck_households_default_year", @@ -78,7 +74,11 @@ class Household(TimestampedModel, table=True): ), ) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) default_year: int | None = None household_data: dict[str, Any] = Field( sa_type=sa.JSON().with_variant(sa.dialects.postgresql.JSONB(), "postgresql") diff --git a/policyengine_api/data/v2/models/policies.py b/policyengine_api/data/v2/models/policies.py index 97e4a619e..f0da6e8cc 100644 --- a/policyengine_api/data/v2/models/policies.py +++ b/policyengine_api/data/v2/models/policies.py @@ -40,10 +40,6 @@ class Policy(TimestampedModel, table=True): "content_hash", name="uq_policies_canonicalization_content_hash", ), - sa.CheckConstraint( - "country_id IN ('us', 'uk')", - name="ck_policies_country", - ), sa.CheckConstraint( "canonicalization_version > 0", name="ck_policies_canonicalization_version", @@ -64,7 +60,11 @@ class Policy(TimestampedModel, table=True): ), ) - country_id: str = Field(max_length=2) + country_id: str = Field( + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) tax_benefit_model_id: UUID = Field( foreign_key="tax_benefit_models.id", ondelete="RESTRICT", diff --git a/policyengine_api/data/v2/models/users.py b/policyengine_api/data/v2/models/users.py index 85c0ea19a..f3b987123 100644 --- a/policyengine_api/data/v2/models/users.py +++ b/policyengine_api/data/v2/models/users.py @@ -44,10 +44,6 @@ class User(IdentifiedModel, table=True): __tablename__ = "users" __table_args__ = ( sa.UniqueConstraint("email", name="uq_users_email"), - sa.CheckConstraint( - "primary_country IN ('us', 'uk')", - name="ck_users_primary_country", - ), sa.CheckConstraint( "user_type IN ('unauthenticated', 'standard')", name="ck_users_user_type", @@ -61,7 +57,12 @@ class User(IdentifiedModel, table=True): first_name: str | None = Field(default=None, max_length=255) last_name: str | None = Field(default=None, max_length=255) email: str | None = Field(default=None, max_length=320, index=True) - primary_country: str | None = Field(default=None, max_length=2) + primary_country: str | None = Field( + default=None, + max_length=2, + foreign_key="countries.id", + ondelete="RESTRICT", + ) # The server default keeps inserts by earlier Public API revisions, which # do not map this column, creating standard users. user_type: str = Field( diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index 8f8c34515..cec2fa66e 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -132,6 +132,9 @@ def _access( # Stage 10 saved-policy mirror. "legacy_user_mappings": _access(runtime=READ_WRITE), "legacy_user_policy_mappings": _access(runtime=READ_WRITE), + # Served countries; the migration that creates the table inserts its rows, + # and the Public API reads them. + "countries": _access(runtime=READ), # Users and their saved work (Stage 12A). "user_household_associations": _access(runtime=READ_WRITE), "user_identities": _access(runtime=READ_WRITE), diff --git a/policyengine_api/fastapi_routes/dependencies.py b/policyengine_api/fastapi_routes/dependencies.py index 5d5a1a38d..b2fcbc0ff 100644 --- a/policyengine_api/fastapi_routes/dependencies.py +++ b/policyengine_api/fastapi_routes/dependencies.py @@ -22,6 +22,7 @@ from policyengine_api.services.v2.associations.types import ( LegacyReferenceChecker, ) + from policyengine_api.services.v2.countries.types import CountryRead from policyengine_api.services.v2.households.types import ( HouseholdPage, HouseholdRead, @@ -312,6 +313,14 @@ def get_economy_options( ... +class V2CountryResourceService(Protocol): + """Route-independent countries list read for one request.""" + + def list_countries(self) -> list[CountryRead]: + """Return every served country and its capability, ordered by ID.""" + ... + + class V2PolicyResourceService(Protocol): """Route-independent native policy operations for one request.""" @@ -742,6 +751,18 @@ def _default_v2_metadata_reader_factory() -> V2MetadataResourceReader: ) +def _default_v2_country_service_factory() -> V2CountryResourceService: + from policyengine_api.data.v2.database import ( + get_v2_session_factory, + ) + from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, + ) + from policyengine_api.services.v2.countries.services import V2CountryService + + return V2CountryService(CountryDatabaseSession(get_v2_session_factory())) + + def _default_v2_policy_service_factory() -> V2PolicyResourceService: from policyengine_api.data.v2.database import ( get_v2_session_factory, @@ -917,6 +938,7 @@ class NativeRouteDependencies: metadata_reader_factory: Callable[[], MetadataReader] specification_provider: Callable[[], JSONObject] v2_metadata_reader_factory: Callable[[], V2MetadataResourceReader] | None = None + v2_country_service_factory: Callable[[], V2CountryResourceService] | None = None v2_policy_service_factory: Callable[[], V2PolicyResourceService] | None = None v2_household_service_factory: Callable[[], V2HouseholdResourceService] | None = None v2_user_policy_service_factory: Callable[[], V2UserPolicyResourceService] | None = ( @@ -947,6 +969,7 @@ def defaults(cls) -> NativeRouteDependencies: metadata_reader_factory=_default_metadata_reader_factory, specification_provider=_default_specification_provider, v2_metadata_reader_factory=_default_v2_metadata_reader_factory, + v2_country_service_factory=_default_v2_country_service_factory, v2_policy_service_factory=_default_v2_policy_service_factory, v2_household_service_factory=_default_v2_household_service_factory, v2_user_policy_service_factory=(_default_v2_user_policy_service_factory), diff --git a/policyengine_api/fastapi_routes/v2/countries/__init__.py b/policyengine_api/fastapi_routes/v2/countries/__init__.py new file mode 100644 index 000000000..3d7f8eaf0 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/countries/__init__.py @@ -0,0 +1 @@ +"""FastAPI adapters for the v2 countries list.""" diff --git a/policyengine_api/fastapi_routes/v2/countries/response_models.py b/policyengine_api/fastapi_routes/v2/countries/response_models.py new file mode 100644 index 000000000..3ea3c7ee8 --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/countries/response_models.py @@ -0,0 +1,71 @@ +"""Strict HTTP response models for the native v2 countries list.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any, Literal + +from pydantic import BaseModel, ConfigDict + +from policyengine_api.fastapi_routes.v2.errors import V2ErrorResponse + +if TYPE_CHECKING: + from policyengine_api.services.v2.countries.types import CountryRead + + +class StrictCountryAPIModel(BaseModel): + """Strict response base with dataclass conversion support.""" + + model_config = ConfigDict(extra="forbid", from_attributes=True) + + +class CountryItem(StrictCountryAPIModel): + """One served country and the calculations it supports. + + ``id`` is the value other v2 resources take as ``country_id``. + ``household_enabled`` means household calculations and saved households + are available; ``society_wide_enabled`` means society-wide reports are, + and is only set for a country that is also household enabled. + """ + + id: str + household_enabled: bool + society_wide_enabled: bool + + @classmethod + def from_read(cls, country: CountryRead) -> CountryItem: + """Build the response item from the service read record.""" + return cls.model_validate(country) + + +class CountryListResult(StrictCountryAPIModel): + """Every served country, ordered by ``id``.""" + + items: list[CountryItem] + + +class CountryListResponse(StrictCountryAPIModel): + """Success envelope: ``status`` "ok", a null ``message`` and the countries.""" + + status: Literal["ok"] = "ok" + message: None = None + result: CountryListResult + + +COUNTRY_ERROR_RESPONSES: dict[int | str, dict[str, Any]] = { + 405: { + "model": V2ErrorResponse, + "description": "The countries list supports GET only.", + }, + 422: { + "model": V2ErrorResponse, + "description": "The request has query parameters; the list takes none.", + }, + 500: { + "model": V2ErrorResponse, + "description": "The countries query failed internally.", + }, + 503: { + "model": V2ErrorResponse, + "description": "The v2 database is not configured or cannot be reached.", + }, +} diff --git a/policyengine_api/fastapi_routes/v2/countries/routes.py b/policyengine_api/fastapi_routes/v2/countries/routes.py new file mode 100644 index 000000000..279aaf54f --- /dev/null +++ b/policyengine_api/fastapi_routes/v2/countries/routes.py @@ -0,0 +1,84 @@ +"""Native FastAPI route for the v2 countries list.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Annotated + +from fastapi import APIRouter, Depends +from sqlalchemy.exc import SQLAlchemyError +from starlette.responses import JSONResponse + +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.dependencies import ( + _default_v2_country_service_factory, +) +from policyengine_api.fastapi_routes.query_parameters import query_dependency +from policyengine_api.fastapi_routes.v2.countries.response_models import ( + COUNTRY_ERROR_RESPONSES, + CountryItem, + CountryListResponse, + CountryListResult, +) +from policyengine_api.fastapi_routes.v2.errors import V2ErrorMapping, run_v2_operation +from policyengine_api.query_parameters import CountryListQuery + +if TYPE_CHECKING: + from collections.abc import Callable + + from policyengine_api.fastapi_routes.dependencies import ( + NativeRouteDependencies, + V2CountryResourceService, + ) + + +CountryListQueryDependency = Annotated[ + CountryListQuery, Depends(query_dependency(CountryListQuery)) +] + +COUNTRY_ERROR_MAPPINGS = ( + V2ErrorMapping( + (V2ConfigurationError, SQLAlchemyError), + 503, + "V2 countries are unavailable", + ), +) + + +def _service_factory( + dependencies: NativeRouteDependencies, +) -> Callable[[], V2CountryResourceService]: + if dependencies.v2_country_service_factory is not None: + return dependencies.v2_country_service_factory + return _default_v2_country_service_factory + + +def build_v2_country_router( + dependencies: NativeRouteDependencies, +) -> APIRouter: + """Build the countries route without opening a database connection.""" + router = APIRouter(prefix="/v2", responses=COUNTRY_ERROR_RESPONSES) + service_factory = _service_factory(dependencies) + + @router.get( + "/countries", + response_model=CountryListResponse, + summary="List the served countries and the calculations each supports", + ) + def list_countries( + _query: CountryListQueryDependency, + ) -> CountryListResponse | JSONResponse: + def read() -> CountryListResponse: + countries = service_factory().list_countries() + return CountryListResponse( + result=CountryListResult( + items=[CountryItem.from_read(country) for country in countries] + ) + ) + + return run_v2_operation( + read, + COUNTRY_ERROR_MAPPINGS, + fallback_message="V2 countries query failed", + ) + + return router diff --git a/policyengine_api/fastapi_routes/v2/routes.py b/policyengine_api/fastapi_routes/v2/routes.py index 0521719be..fa39d7783 100644 --- a/policyengine_api/fastapi_routes/v2/routes.py +++ b/policyengine_api/fastapi_routes/v2/routes.py @@ -7,6 +7,9 @@ from fastapi import APIRouter, Request from starlette.responses import JSONResponse +from policyengine_api.fastapi_routes.v2.countries.routes import ( + build_v2_country_router, +) from policyengine_api.fastapi_routes.v2.errors import V2ErrorResponse from policyengine_api.fastapi_routes.v2.households.routes import ( build_v2_household_router, @@ -53,6 +56,7 @@ def build_v2_router( router.include_router(build_v2_user_policy_router(dependencies)) router.include_router(build_v2_household_router(dependencies)) router.include_router(build_v2_policy_router(dependencies)) + router.include_router(build_v2_country_router(dependencies)) router.include_router(build_v2_metadata_model_router(dependencies)) router.include_router(build_v2_metadata_parameter_router(dependencies)) router.include_router(build_v2_metadata_geography_router(dependencies)) diff --git a/policyengine_api/migration_registry.py b/policyengine_api/migration_registry.py index e49de6986..5daadd05a 100644 --- a/policyengine_api/migration_registry.py +++ b/policyengine_api/migration_registry.py @@ -37,6 +37,7 @@ class RouteGroupConfig: name="metadata", path_segments=( "metadata", + "countries", "datasets", "economy-options", "parameter-values", diff --git a/policyengine_api/query_parameters.py b/policyengine_api/query_parameters.py index 06e38cf01..086d33131 100644 --- a/policyengine_api/query_parameters.py +++ b/policyengine_api/query_parameters.py @@ -29,6 +29,7 @@ validate_policyengine_version, ) from policyengine_api.spm import SPMSelection +from policyengine_api.supported_countries import SupportedCountryId from policyengine_api.utils.budget_window import ( BUDGET_WINDOW_MAX_END_YEAR, BUDGET_WINDOW_MAX_YEARS, @@ -40,7 +41,6 @@ DEFAULT_QUERY_LIMIT = 100 MAXIMUM_QUERY_LIMIT = 500 MAXIMUM_USER_ID_LENGTH = 255 -SUPPORTED_COUNTRY_IDS = frozenset({"us", "uk"}) def normalize_country_id(value: object) -> object: @@ -57,7 +57,7 @@ def validate_legacy_user_id(value: str) -> str: CountryId = Annotated[ - Literal["us", "uk"], + SupportedCountryId, BeforeValidator(normalize_country_id), Field(description="Supported PolicyEngine country ID"), ] @@ -214,6 +214,10 @@ class PaginationQuery(StrictQueryParameters): limit: Limit = DEFAULT_QUERY_LIMIT +class CountryListQuery(StrictQueryParameters): + """Query contract for the countries list, which takes no parameters.""" + + class PolicyCreateQuery(CatalogQuery): """Query contract for native immutable policy creation.""" diff --git a/policyengine_api/services/v2/countries/__init__.py b/policyengine_api/services/v2/countries/__init__.py new file mode 100644 index 000000000..f0f7b972b --- /dev/null +++ b/policyengine_api/services/v2/countries/__init__.py @@ -0,0 +1 @@ +"""Application services for the v2 countries list.""" diff --git a/policyengine_api/services/v2/countries/database_connectors/__init__.py b/policyengine_api/services/v2/countries/database_connectors/__init__.py new file mode 100644 index 000000000..01c1fa018 --- /dev/null +++ b/policyengine_api/services/v2/countries/database_connectors/__init__.py @@ -0,0 +1 @@ +"""SQL-facing connector functions for the v2 countries service.""" diff --git a/policyengine_api/services/v2/countries/database_connectors/reads.py b/policyengine_api/services/v2/countries/database_connectors/reads.py new file mode 100644 index 000000000..a13a8f3e4 --- /dev/null +++ b/policyengine_api/services/v2/countries/database_connectors/reads.py @@ -0,0 +1,17 @@ +"""Database selections used by the v2 countries list and the startup check.""" + +from __future__ import annotations + +from sqlmodel import Session, col, select + +from policyengine_api.data.v2.models import Country + + +def read_countries(session: Session) -> list[Country]: + """Return every countries row, ordered by country ID.""" + return list(session.exec(select(Country).order_by(col(Country.id))).all()) + + +def read_country_ids(session: Session) -> list[str]: + """Return the ID of every countries row, ordered.""" + return list(session.exec(select(Country.id).order_by(col(Country.id))).all()) diff --git a/policyengine_api/services/v2/countries/database_session.py b/policyengine_api/services/v2/countries/database_session.py new file mode 100644 index 000000000..455d4aa39 --- /dev/null +++ b/policyengine_api/services/v2/countries/database_session.py @@ -0,0 +1,26 @@ +"""Database session lifetime management for the v2 countries service.""" + +from __future__ import annotations + +from contextlib import contextmanager +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterator + + from sqlalchemy.orm import sessionmaker + from sqlmodel import Session + + +class CountryDatabaseSession: + """Open the read sessions the countries service uses.""" + + def __init__(self, session_factory: sessionmaker[Session]) -> None: + """Use ``session_factory`` to open every session.""" + self._session_factory = session_factory + + @contextmanager + def read(self) -> Iterator[Session]: + """Yield a session for reads; nothing is committed.""" + with self._session_factory() as session: + yield session diff --git a/policyengine_api/services/v2/countries/services.py b/policyengine_api/services/v2/countries/services.py new file mode 100644 index 000000000..96e0ad601 --- /dev/null +++ b/policyengine_api/services/v2/countries/services.py @@ -0,0 +1,49 @@ +"""Router-facing and startup service for the v2 countries table.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from policyengine_api.services.v2.countries.database_connectors.reads import ( + read_countries, + read_country_ids, +) +from policyengine_api.services.v2.countries.transformations import country_read +from policyengine_api.services.v2.countries.validators import ( + validate_countries_match_code_list, +) + +if TYPE_CHECKING: + from collections.abc import Iterable + + from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, + ) + from policyengine_api.services.v2.countries.types import CountryRead + + +class V2CountryService: + """List the served countries, and check them against the list in code.""" + + def __init__(self, database_session: CountryDatabaseSession) -> None: + """Read through ``database_session``, which opens one session per call.""" + self._database_session = database_session + + def list_countries(self) -> list[CountryRead]: + """Return every countries row, ordered by country ID.""" + with self._database_session.read() as session: + return [country_read(country) for country in read_countries(session)] + + def verify_countries_match_code_list( + self, listed_country_ids: Iterable[str] + ) -> None: + """Raise ``CountryListMismatchError`` unless the rows name exactly these. + + The application runs this at startup with the country list in code. + """ + with self._database_session.read() as session: + stored_country_ids = read_country_ids(session) + validate_countries_match_code_list( + stored_country_ids=stored_country_ids, + listed_country_ids=listed_country_ids, + ) diff --git a/policyengine_api/services/v2/countries/transformations.py b/policyengine_api/services/v2/countries/transformations.py new file mode 100644 index 000000000..fd1d65fe2 --- /dev/null +++ b/policyengine_api/services/v2/countries/transformations.py @@ -0,0 +1,19 @@ +"""Conversions between stored countries rows and service read records.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from policyengine_api.services.v2.countries.types import CountryRead + +if TYPE_CHECKING: + from policyengine_api.data.v2.models import Country + + +def country_read(country: Country) -> CountryRead: + """Convert one stored countries row to the service's read record.""" + return CountryRead( + id=country.id, + household_enabled=country.household_enabled, + society_wide_enabled=country.society_wide_enabled, + ) diff --git a/policyengine_api/services/v2/countries/types.py b/policyengine_api/services/v2/countries/types.py new file mode 100644 index 000000000..b4a1e74fc --- /dev/null +++ b/policyengine_api/services/v2/countries/types.py @@ -0,0 +1,19 @@ +"""Framework-independent data exchanged by the v2 countries layers.""" + +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class CountryRead: + """One served country and the calculations it supports. + + ``household_enabled`` allows household calculations and saved households; + ``society_wide_enabled`` allows society-wide reports, and implies + ``household_enabled``. + """ + + id: str + household_enabled: bool + society_wide_enabled: bool diff --git a/policyengine_api/services/v2/countries/validators.py b/policyengine_api/services/v2/countries/validators.py new file mode 100644 index 000000000..fbb30ca5b --- /dev/null +++ b/policyengine_api/services/v2/countries/validators.py @@ -0,0 +1,48 @@ +"""Checks of the stored countries rows against the country list in code.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from collections.abc import Iterable + + +class CountryListMismatchError(RuntimeError): + """The countries table and the country list in code name different countries. + + ``listed_without_row`` holds the listed countries that have no row, and + ``rows_not_listed`` the rows for countries the list does not name, each + sorted by ID. + """ + + def __init__( + self, + *, + listed_without_row: tuple[str, ...], + rows_not_listed: tuple[str, ...], + ) -> None: + """Record both differences and describe them in the message.""" + self.listed_without_row = listed_without_row + self.rows_not_listed = rows_not_listed + super().__init__( + "the countries table does not match the country list in code: " + f"listed without a row: {', '.join(listed_without_row) or 'none'}; " + f"rows not listed: {', '.join(rows_not_listed) or 'none'}" + ) + + +def validate_countries_match_code_list( + *, + stored_country_ids: Iterable[str], + listed_country_ids: Iterable[str], +) -> None: + """Raise ``CountryListMismatchError`` unless both name the same countries.""" + stored = set(stored_country_ids) + listed = set(listed_country_ids) + if stored == listed: + return + raise CountryListMismatchError( + listed_without_row=tuple(sorted(listed - stored)), + rows_not_listed=tuple(sorted(stored - listed)), + ) diff --git a/policyengine_api/startup_country_check.py b/policyengine_api/startup_country_check.py new file mode 100644 index 000000000..69f233988 --- /dev/null +++ b/policyengine_api/startup_country_check.py @@ -0,0 +1,106 @@ +"""Refuse to start when the v2 countries table and the country list differ. + +Which countries the Public API serves is decided in code +(``policyengine_api.supported_countries``). The v2 ``countries`` table records +what each supports, and must hold one row for each listed country and no +other. ``policyengine_api.asgi`` runs this check while the application loads, +so a disagreement stops the worker before it serves a request. + +The check runs only where a v2 runtime database is selected: on Cloud Run +(``K_SERVICE`` is set), or when a v2 runtime database URL or its secret +resource is configured. A local or test startup without v2 configuration +opens no connection. + +Startup stops when the database answers and its answer disagrees with the +list: different countries, a missing table, or no privilege to read it. +When the check cannot get an answer, it logs why and startup continues: the +v1 routes do not read the v2 database, and their copies into it already +continue without it. That covers a database that cannot be reached or +refuses the credentials, and v2 configuration that cannot be used, including +a secret that cannot be resolved; the readiness check already reports the +last as not ready, and recovers once the secret resolves. +""" + +from __future__ import annotations + +import logging +import os +from typing import TYPE_CHECKING, Protocol + +from sqlalchemy.exc import InterfaceError, OperationalError + +from policyengine_api.data.v2.settings import ( + V2_RUNTIME_DATABASE_URL, + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE, + V2ConfigurationError, +) +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS + +if TYPE_CHECKING: + from collections.abc import Callable, Iterable, Mapping + +logger = logging.getLogger(__name__) + +CLOUD_RUN_SERVICE = "K_SERVICE" +V2_RUNTIME_SELECTION_VARIABLES = ( + CLOUD_RUN_SERVICE, + V2_RUNTIME_DATABASE_URL, + V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE, +) + + +class CountryListVerifier(Protocol): + """Compare the stored countries rows with a list of country IDs.""" + + def verify_countries_match_code_list( + self, listed_country_ids: Iterable[str] + ) -> None: + """Raise ``CountryListMismatchError`` unless the rows name exactly these.""" + ... + + +def _default_country_list_verifier() -> CountryListVerifier: + from policyengine_api.data.v2.database import get_v2_session_factory + from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, + ) + from policyengine_api.services.v2.countries.services import V2CountryService + + return V2CountryService(CountryDatabaseSession(get_v2_session_factory())) + + +def v2_runtime_database_selected(environ: Mapping[str, str]) -> bool: + """Whether this process runs on Cloud Run or has a v2 runtime database.""" + return any(environ.get(name, "").strip() for name in V2_RUNTIME_SELECTION_VARIABLES) + + +def check_countries_table_at_startup( + environ: Mapping[str, str] | None = None, + verifier_factory: Callable[[], CountryListVerifier] | None = None, +) -> None: + """Raise unless the countries table names exactly the countries in code. + + Raises ``CountryListMismatchError`` on a disagreement and SQLAlchemy's + ``ProgrammingError`` when the table or the privilege to read it is + missing. Unusable v2 configuration is logged with its message, which + never contains a secret, and a database connection failure by its error + type only, as its detail can name the host; both skip the check. + """ + values = os.environ if environ is None else environ + if not v2_runtime_database_selected(values): + return + factory = verifier_factory or _default_country_list_verifier + try: + factory().verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + except V2ConfigurationError as error: + logger.warning( + "Skipped the countries table check at startup: the v2 runtime " + "configuration cannot be used (%s)", + error, + ) + except (OperationalError, InterfaceError) as error: + logger.warning( + "Skipped the countries table check at startup: the v2 database " + "could not be queried (%s)", + type(error).__name__, + ) diff --git a/policyengine_api/supported_countries.py b/policyengine_api/supported_countries.py new file mode 100644 index 000000000..0c5ad01a1 --- /dev/null +++ b/policyengine_api/supported_countries.py @@ -0,0 +1,33 @@ +"""The countries the Public API serves, listed once in code. + +``SupportedCountryId`` is the one country list. Every other list derives from +it: the v2 routes' ``CountryId`` type, the countries catalog extraction +publishes, and the countries the v1 routes copy into v2 (dual-write). Adding +a country starts here. + +The v2 ``countries`` table records what each country supports, and must hold +one row for each listed country and no other: a v2 Alembic revision inserts +the row when a country is added. The application refuses to start when they +differ (``policyengine_api.startup_country_check``). Because a deploy runs +that revision before it starts the new release, instances of the previous +release that start afterwards, or a rollback to it, refuse to start until +the revision is downgraded. + +A new country also needs the US-and-UK checks widened on the tables that +pair v1 records with their v2 copies (``legacy_policy_mappings``, +``legacy_user_policy_mappings``, ``legacy_household_mappings``) and on the +v1 ``household_mirror_events`` table, which the v1 household insert writes +in its own transaction. + +The module imports nothing from the package, so modules that the query +models import, such as the catalog package, can import it too. +""" + +from __future__ import annotations + +from typing import Final, Literal, get_args + +SupportedCountryId = Literal["us", "uk"] +SUPPORTED_COUNTRY_IDS: Final[tuple[SupportedCountryId, ...]] = get_args( + SupportedCountryId +) diff --git a/pyproject.toml b/pyproject.toml index ed101271e..ae1ce0e70 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -158,6 +158,8 @@ files = [ "policyengine_api/services/v2", "policyengine_api/libs/user_auth.py", "policyengine_api/libs/auth0_management.py", + "policyengine_api/startup_country_check.py", + "policyengine_api/supported_countries.py", ] explicit_package_bases = true follow_imports = "silent" @@ -212,6 +214,8 @@ banned-module-level-imports = [ "policyengine", "policyengine_api.data.v2.database", "policyengine_api.libs.auth0_management", + "policyengine_api.services.v2.countries.database_session", + "policyengine_api.services.v2.countries.services", "policyengine_api.services.v2.households.database_session", "policyengine_api.services.v2.households.services", "policyengine_api.services.v2.legacy_refs.database_session", diff --git a/rules/registry.toml b/rules/registry.toml index e517ec50a..46718196b 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -396,7 +396,7 @@ invocation = "make test" [[rule]] id = "api.startup.no-import-side-effects" category = "startup" -statement = "Importing v2 modules, starting up and serving a request open no network connections and create no files or runtime SQLite databases." +statement = "Importing v2 modules opens no network connections, starting up and serving a request open none unless the process runs on Cloud Run or has a v2 runtime database configured, and none of them creates files or runtime SQLite databases." rationale = "docs/engineering/skills/testing.md#focused-commands" scope = ["policyengine_api/**/*.py"] tier = "mechanical" @@ -919,9 +919,9 @@ invocation = "ruff check ." [[rule]] id = "api.quality.mypy-v2" category = "quality" -statement = "Shared query models, FastAPI dependencies, v2 routes and services, and the user sign-in and account-deletion libraries pass mypy with untyped definitions disallowed." +statement = "Shared query models, FastAPI dependencies, v2 routes and services, the user sign-in and account-deletion libraries, the country list and its startup check pass mypy with untyped definitions disallowed." rationale = "docs/engineering/skills/v2-code-organization.md" -scope = ["policyengine_api/fastapi_routes/v2/**/*.py", "policyengine_api/services/v2/**/*.py", "policyengine_api/libs/user_auth.py", "policyengine_api/libs/auth0_management.py"] +scope = ["policyengine_api/fastapi_routes/v2/**/*.py", "policyengine_api/services/v2/**/*.py", "policyengine_api/libs/user_auth.py", "policyengine_api/libs/auth0_management.py", "policyengine_api/startup_country_check.py", "policyengine_api/supported_countries.py"] tier = "mechanical" status = "enforced" autofix = false @@ -935,6 +935,8 @@ paths = [ "policyengine_api/services/v2", "policyengine_api/libs/user_auth.py", "policyengine_api/libs/auth0_management.py", + "policyengine_api/startup_country_check.py", + "policyengine_api/supported_countries.py", ] [[rule.ci]] workflow = ".github/workflows/pr.yml" diff --git a/scripts/process_v1_household_mirror_event.py b/scripts/process_v1_household_mirror_event.py index f7af31072..49717019b 100755 --- a/scripts/process_v1_household_mirror_event.py +++ b/scripts/process_v1_household_mirror_event.py @@ -20,6 +20,7 @@ process_household_event_after_commit, ) from policyengine_api.services.household_service import HouseholdService +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS if TYPE_CHECKING: from collections.abc import Mapping, Sequence @@ -149,7 +150,7 @@ def _parser() -> argparse.ArgumentParser: required=True, choices=("staging", "production"), ) - parser.add_argument("--country-id", required=True, choices=("us", "uk")) + parser.add_argument("--country-id", required=True, choices=SUPPORTED_COUNTRY_IDS) parser.add_argument("--legacy-household-id", required=True, type=int) return parser diff --git a/tests/contract/registry.py b/tests/contract/registry.py index 778c3da26..a04522cbf 100644 --- a/tests/contract/registry.py +++ b/tests/contract/registry.py @@ -587,6 +587,24 @@ class WorkflowContract: ), ), ), + WorkflowContract( + name="country_selection_v2", + current_contract="typed_v2_resources", + future_owner_pr="Countries table and one country list in code", + requests=( + ContractRequest( + method="GET", + path="/v2/countries", + expected_status=200, + stable_response_fields=( + "status", + "message", + "result.items", + ), + route_group="metadata", + ), + ), + ), WorkflowContract( name="metadata_resources_v2_preview", current_contract="typed_v2_resources", diff --git a/tests/contract/test_app_v2_workflow_contracts.py b/tests/contract/test_app_v2_workflow_contracts.py index ff861db14..f74640678 100644 --- a/tests/contract/test_app_v2_workflow_contracts.py +++ b/tests/contract/test_app_v2_workflow_contracts.py @@ -20,6 +20,7 @@ def test_app_v2_workflow_contract_registry_is_complete(): "user_account_v2", "household_calculate", "region_selection", + "country_selection_v2", "metadata_resources_v2_preview", "simulation_submit_poll", "report_create_poll", @@ -31,6 +32,7 @@ def test_app_v2_workflow_contract_registry_is_complete(): "typed_v2_resources" if workflow.name in { + "country_selection_v2", "metadata_resources_v2_preview", "policy_resources_v2", "user_policy_associations_v2", diff --git a/tests/fixtures/v2_countries.py b/tests/fixtures/v2_countries.py new file mode 100644 index 000000000..36d4b6604 --- /dev/null +++ b/tests/fixtures/v2_countries.py @@ -0,0 +1,28 @@ +"""Countries rows for SQLite tests that enforce v2 foreign keys. + +``V2_METADATA.create_all`` creates an empty countries table, while the v2 +Alembic revision that adds the table inserts the US and UK rows. Tests that +turn on SQLite foreign-key enforcement add the same rows before writing +records that name a country. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +from policyengine_api.data.v2.models import Country + +if TYPE_CHECKING: + from sqlmodel import Session + +# The rows the countries revision inserts, ordered by country ID. +LISTED_COUNTRY_ROWS = ( + {"id": "uk", "household_enabled": True, "society_wide_enabled": True}, + {"id": "us", "household_enabled": True, "society_wide_enabled": True}, +) + + +def add_listed_countries(session: Session) -> None: + """Insert and commit the US and UK countries rows.""" + session.add_all(Country(**row) for row in LISTED_COUNTRY_ROWS) + session.commit() diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index 814971236..2df169511 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -3,6 +3,7 @@ import itertools import os from collections.abc import Iterator +from dataclasses import dataclass from datetime import UTC, datetime, timedelta from uuid import UUID, uuid4 @@ -30,6 +31,7 @@ STAGE_12_PREVIOUS_REVISION = "724b1b11a33e" STAGE_12_COMPARISON_PREVIOUS_REVISION = "439303be14fe" STAGE_12A_PREVIOUS_REVISION = "91af4fc8d8c1" +COUNTRIES_PREVIOUS_REVISION = "fbb7488ebe8a" V2_TABLE_NAMES = frozenset(table.name for table in V2_METADATA.tables.values()) @@ -187,7 +189,6 @@ def _assert_stage_12a_association_shape(engine) -> None: for constraint in inspect(engine).get_check_constraints("users") } assert { - "ck_users_primary_country", "ck_users_user_type", "ck_users_standard_primary_country", } <= user_checks @@ -352,6 +353,75 @@ def _assert_base(engine: Engine) -> None: ) +# Every column that names a record's country, with the check it replaced. +COUNTRY_REFERENCE_COLUMNS = { + ("users", "primary_country"): "ck_users_primary_country", + ("households", "country_id"): "ck_households_country", + ("policies", "country_id"): "ck_policies_country", + ("user_household_associations", "country_id"): ( + "ck_user_household_associations_country" + ), + ("user_policies", "country_id"): "ck_user_policies_country", + ("user_simulation_associations", "country_id"): ( + "ck_user_simulation_associations_country" + ), + ("user_report_associations", "country_id"): "ck_user_report_associations_country", +} +# The rows the countries revision inserts: the countries the replaced checks +# allowed, both household and society-wide capable. +LISTED_COUNTRIES = [("uk", True, True), ("us", True, True)] + + +def _country_foreign_keys(engine, table_name: str) -> list[dict]: + return [ + foreign_key + for foreign_key in inspect(engine).get_foreign_keys(table_name) + if foreign_key["referred_table"] == "countries" + ] + + +def _check_names(engine, table_name: str) -> set[str]: + return { + constraint["name"] + for constraint in inspect(engine).get_check_constraints(table_name) + } + + +def _listed_countries(engine) -> list[tuple]: + with engine.connect() as connection: + return [ + tuple(row) + for row in connection.execute( + text( + "SELECT id, household_enabled, society_wide_enabled " + "FROM countries ORDER BY id" + ) + ) + ] + + +def _assert_countries_shape(engine) -> None: + assert _listed_countries(engine) == LISTED_COUNTRIES + assert "ck_countries_society_wide_requires_household" in _check_names( + engine, "countries" + ) + for (table_name, column_name), check_name in COUNTRY_REFERENCE_COLUMNS.items(): + country_keys = _country_foreign_keys(engine, table_name) + assert len(country_keys) == 1, table_name + assert country_keys[0]["name"] == f"fk_{table_name}_{column_name}_countries" + assert country_keys[0]["constrained_columns"] == [column_name] + assert country_keys[0]["referred_columns"] == ["id"] + assert country_keys[0]["options"]["ondelete"] == "RESTRICT" + assert check_name not in _check_names(engine, table_name) + + +def _assert_pre_countries_shape(engine) -> None: + assert "countries" not in inspect(engine).get_table_names(schema="public") + for (table_name, _column_name), check_name in COUNTRY_REFERENCE_COLUMNS.items(): + assert _country_foreign_keys(engine, table_name) == [] + assert check_name in _check_names(engine, table_name) + + def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: database_url = _disposable_url() config = _config() @@ -364,6 +434,13 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: } _upgrade_and_assert_head(config, engine) + _assert_countries_shape(engine) + + command.downgrade(config, COUNTRIES_PREVIOUS_REVISION) + _assert_revision(engine, COUNTRIES_PREVIOUS_REVISION) + _assert_pre_countries_shape(engine) + _upgrade_and_assert_head(config, engine) + _assert_countries_shape(engine) command.downgrade(config, STAGE_12A_PREVIOUS_REVISION) _assert_revision(engine, STAGE_12A_PREVIOUS_REVISION) @@ -396,10 +473,10 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: engine.dispose() -# Dormant at the parent revision: its models map these tables, and deleting a -# user cascades through them, but no parent-revision code inserts into them. -# They are the only tables Stage 12A may give a new required column. -PARENT_REVISION_READ_ONLY_TABLES = frozenset( +# Dormant at the Stage 12A parent revision: its models map these tables, and +# deleting a user cascades through them, but no parent-revision code inserts +# into them. They are the only tables Stage 12A may give a new required column. +STAGE_12A_PARENT_READ_ONLY_TABLES = frozenset( {"user_simulation_associations", "user_report_associations"} ) USER_ASSOCIATION_TABLES = ( @@ -409,8 +486,18 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: "user_report_associations", ) COUNTRY_COLUMN_NAMES = frozenset({"country", "country_id", "primary_country"}) -# Nullable references that a parent-revision check constraint requires. -REFERENCES_REQUIRED_BY_CHECKS = frozenset({("simulations", "household_id")}) +# Nullable references that a parent-revision check constraint requires: a +# standard user's country, and one of each association's two references. +REFERENCES_REQUIRED_BY_CHECKS = frozenset( + { + ("simulations", "household_id"), + ("users", "primary_country"), + ("user_policies", "policy_id"), + ("user_household_associations", "household_id"), + ("user_simulation_associations", "simulation_id"), + ("user_report_associations", "report_id"), + } +) # Required values that a parent-revision check constraint restricts. CHECKED_VALUES = {("datasets", "year"): 2026} @@ -547,14 +634,33 @@ def _rewrite_mapped_columns(connection, table: sa.Table, row: dict) -> None: assert result.rowcount == 1, table.name +@dataclass(frozen=True) +class PreviousRevisionSchemas: + """A previous revision's reflected schema and head's, after the upgrade. + + ``read_only_tables`` are the previous revision's tables that only new code + writes, so their test rows are inserted through head's table. + """ + + parent: sa.MetaData + head: sa.MetaData + read_only_tables: frozenset[str] + + def insert_table(self, table_name: str) -> sa.Table | None: + """Head's table for a read-only table, otherwise ``None``.""" + if table_name in self.read_only_tables: + return self.head.tables[table_name] + return None + + def _assert_user_deletion_cascades( connection, - parent: sa.MetaData, - head: sa.MetaData, + schemas: PreviousRevisionSchemas, stored_rows: dict[str, dict], sequence: Iterator[int], ) -> None: - """Delete, through the parent's users table, a user who owns Stage 12A rows.""" + """Delete, through the parent's users table, a user who owns association rows.""" + parent, head = schemas.parent, schemas.head users = parent.tables["users"] owner = _insert_and_read_back( connection, users, _row_values(users, stored_rows, sequence) @@ -562,11 +668,7 @@ def _assert_user_deletion_cascades( owner_rows = {**stored_rows, "users": owner} for table_name in USER_ASSOCIATION_TABLES: table = parent.tables[table_name] - head_table = ( - head.tables[table_name] - if table_name in PARENT_REVISION_READ_ONLY_TABLES - else None - ) + head_table = schemas.insert_table(table_name) _insert_and_read_back( connection, table, @@ -596,22 +698,26 @@ def _assert_user_deletion_cascades( assert remaining == 0, table_name -def test_stage_12a_schema_serves_the_previous_revision() -> None: - """Run the previous Public API revision's table usage against head. +def _assert_head_serves_previous_revision( + previous_revision: str, + read_only_tables: frozenset[str], +) -> None: + """Run a previous Public API revision's table usage against head. The parent revision's schema is reflected and then used, after upgrading, to insert a row into every table it maps (supplying only what it requires), read the row back with every mapped column, rewrite those - columns, delete it, and delete a user who owns Stage 12A rows so the + columns, delete it, and delete a user who owns association rows so the cascades run. Head may not give any table the parent revision writes a - new required column without a server default. + new required column without a server default; ``read_only_tables`` are the + parent's tables that only new code writes. """ database_url = _disposable_url() config = _config() engine = create_engine(database_url) try: - command.downgrade(config, STAGE_12A_PREVIOUS_REVISION) + command.downgrade(config, previous_revision) parent = sa.MetaData() parent.reflect(engine) command.upgrade(config, "head") @@ -634,21 +740,18 @@ def test_stage_12a_schema_serves_the_previous_revision() -> None: for table in parent_tables if _new_required_columns(table, head.tables[table.name]) } - assert set(tables_with_new_required_columns) <= ( - PARENT_REVISION_READ_ONLY_TABLES - ), tables_with_new_required_columns + assert set(tables_with_new_required_columns) <= read_only_tables, ( + tables_with_new_required_columns + ) + schemas = PreviousRevisionSchemas(parent, head, read_only_tables) sequence = itertools.count() with engine.connect() as connection: transaction = connection.begin() try: stored_rows: dict[str, dict] = {} for table in parent_tables: - head_table = ( - head.tables[table.name] - if table.name in PARENT_REVISION_READ_ONLY_TABLES - else None - ) + head_table = schemas.insert_table(table.name) stored_rows[table.name] = _insert_and_read_back( connection, table, @@ -660,7 +763,7 @@ def test_stage_12a_schema_serves_the_previous_revision() -> None: _rewrite_mapped_columns(connection, table, stored_rows[table.name]) _assert_user_deletion_cascades( - connection, parent, head, stored_rows, sequence + connection, schemas, stored_rows, sequence ) for table in reversed(parent_tables): @@ -677,6 +780,21 @@ def test_stage_12a_schema_serves_the_previous_revision() -> None: engine.dispose() +def test_stage_12a_schema_serves_the_previous_revision() -> None: + _assert_head_serves_previous_revision( + STAGE_12A_PREVIOUS_REVISION, + STAGE_12A_PARENT_READ_ONLY_TABLES, + ) + + +def test_countries_schema_serves_the_previous_revision() -> None: + """The release before the countries table keeps working during a deploy.""" + _assert_head_serves_previous_revision( + COUNTRIES_PREVIOUS_REVISION, + frozenset(), + ) + + def test_upgrade_to_head_validates_the_resulting_schema_against_metadata() -> None: database_url = _disposable_url() config = _config() @@ -957,3 +1075,176 @@ def reflected(name: str) -> sa.Table: connection.execute(sa.delete(table).where(table.c.id == row_id)) command.upgrade(config, "head") engine.dispose() + + +def test_countries_revision_keeps_existing_rows_and_rejects_unlisted_countries() -> ( + None +): + """Rows saved before the revision survive it, and only listed countries pass.""" + database_url = _disposable_url() + config = _config() + engine = create_engine(database_url) + user_id = uuid4() + + def reflected(name: str) -> sa.Table: + return sa.Table(name, sa.MetaData(), autoload_with=engine) + + try: + command.downgrade(config, COUNTRIES_PREVIOUS_REVISION) + with engine.begin() as connection: + connection.execute( + sa.insert(reflected("users")).values(id=user_id, primary_country="uk") + ) + connection.execute( + sa.insert(reflected("user_report_associations")).values( + id=uuid4(), + user_id=user_id, + country_id="uk", + legacy_report_id=7, + ) + ) + + command.upgrade(config, "head") + command.check(config) + _assert_countries_shape(engine) + + with engine.connect() as connection: + stored_countries = connection.execute( + text( + "SELECT users.primary_country, " + "user_report_associations.country_id FROM users " + "JOIN user_report_associations " + "ON user_report_associations.user_id = users.id " + "WHERE users.id = :user_id" + ), + {"user_id": user_id}, + ).one() + assert tuple(stored_countries) == ("uk", "uk") + + with engine.connect() as connection, pytest.raises(sa.exc.IntegrityError): + connection.execute( + sa.insert(reflected("user_report_associations")).values( + id=uuid4(), + user_id=user_id, + country_id="ca", + legacy_report_id=8, + ) + ) + + with engine.connect() as connection, pytest.raises(sa.exc.IntegrityError): + connection.execute(text("DELETE FROM countries WHERE id = 'uk'")) + finally: + with engine.begin() as connection: + users = reflected("users") + connection.execute(sa.delete(users).where(users.c.id == user_id)) + command.upgrade(config, "head") + engine.dispose() + + +def test_countries_downgrade_refuses_records_of_an_unlisted_country() -> None: + """The restored US-and-UK checks stop a downgrade that would orphan records.""" + database_url = _disposable_url() + config = _config() + engine = create_engine(database_url) + user_id = uuid4() + + try: + command.upgrade(config, "head") + with engine.begin() as connection: + connection.execute( + text( + "INSERT INTO countries " + "(id, household_enabled, society_wide_enabled) " + "VALUES ('be', true, false)" + ) + ) + connection.execute( + text("INSERT INTO users (id, primary_country) VALUES (:user_id, 'be')"), + {"user_id": user_id}, + ) + + with pytest.raises(sa.exc.IntegrityError): + command.downgrade(config, COUNTRIES_PREVIOUS_REVISION) + + with engine.connect() as connection: + context = MigrationContext.configure(connection) + assert context.get_current_revision() == V2_ALEMBIC_HEAD_REVISION + finally: + with engine.begin() as connection: + connection.execute( + text("DELETE FROM users WHERE id = :user_id"), {"user_id": user_id} + ) + connection.execute(text("DELETE FROM countries WHERE id = 'be'")) + command.upgrade(config, "head") + engine.dispose() + + +def test_a_household_only_country_row_lets_records_name_that_country() -> None: + """Once a household-only country has a row, its records can be stored.""" + database_url = _disposable_url() + config = _config() + engine = create_engine(database_url) + user_id = uuid4() + household_id = uuid4() + + try: + command.upgrade(config, "head") + with engine.connect() as connection: + transaction = connection.begin() + try: + with pytest.raises(sa.exc.IntegrityError), connection.begin_nested(): + connection.execute( + text( + "INSERT INTO users (id, primary_country) " + "VALUES (:user_id, 'be')" + ), + {"user_id": user_id}, + ) + + connection.execute( + text( + "INSERT INTO countries " + "(id, household_enabled, society_wide_enabled) " + "VALUES ('be', true, false)" + ) + ) + connection.execute( + text( + "INSERT INTO users (id, primary_country) " + "VALUES (:user_id, 'be')" + ), + {"user_id": user_id}, + ) + connection.execute( + text( + "INSERT INTO households (id, country_id, household_data, " + "canonicalization_version, content_hash) VALUES " + "(:household_id, 'be', '{}'::jsonb, 1, :content_hash)" + ), + {"household_id": household_id, "content_hash": "c" * 64}, + ) + connection.execute( + text( + "INSERT INTO user_household_associations " + "(id, user_id, household_id, country_id) VALUES " + "(:association_id, :user_id, :household_id, 'be')" + ), + { + "association_id": uuid4(), + "user_id": user_id, + "household_id": household_id, + }, + ) + + with pytest.raises(sa.exc.IntegrityError), connection.begin_nested(): + connection.execute( + text( + "INSERT INTO countries " + "(id, household_enabled, society_wide_enabled) " + "VALUES ('xx', false, true)" + ) + ) + finally: + transaction.rollback() + finally: + engine.dispose() diff --git a/tests/integration/test_v2_country_routes.py b/tests/integration/test_v2_country_routes.py new file mode 100644 index 000000000..ee6408ddc --- /dev/null +++ b/tests/integration/test_v2_country_routes.py @@ -0,0 +1,111 @@ +"""Serve and check the v2 countries from a migrated disposable Postgres database. + +The route and the startup check run as the Public API's runtime role, so the +tests prove the role's declared privileges cover them (rule +``api.v2.runtime-privileges-tested-as-runtime-role``). +""" + +from __future__ import annotations + +import pytest +from alembic import command +from alembic.config import Config +from fastapi.testclient import TestClient +from flask import Flask +from sqlalchemy import create_engine, text +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import NullPool +from sqlmodel import Session + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.constants import REPO +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, +) +from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, +) +from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.countries.validators import ( + CountryListMismatchError, +) +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS +from tests.fixtures.v2_countries import LISTED_COUNTRY_ROWS + + +def test_postgres_countries_list_the_rows_the_revision_inserted( + managed_v2_roles, + v2_engine_as_role, +) -> None: + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + runtime_engine = v2_engine_as_role(managed_v2_roles.runtime, poolclass=NullPool) + sessions = sessionmaker(runtime_engine, class_=Session) + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + specification_provider=dict, + v2_country_service_factory=lambda: V2CountryService( + CountryDatabaseSession(sessions) + ), + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + client = TestClient( + create_asgi_app( + Flask(__name__), + dependencies=dependencies, + route_settings=settings, + ) + ) + + try: + response = client.get("/v2/countries") + finally: + runtime_engine.dispose() + + assert response.status_code == 200 + assert response.json() == { + "status": "ok", + "message": None, + "result": {"items": list(LISTED_COUNTRY_ROWS)}, + } + + +def test_postgres_countries_match_the_code_list_as_the_runtime_role( + managed_v2_roles, + v2_engine_as_role, + disposable_v2_database_url: str, +) -> None: + """The startup check passes on the revision's rows and fails on an extra row.""" + command.upgrade(Config(str(REPO / "alembic-v2.ini")), "head") + runtime_engine = v2_engine_as_role(managed_v2_roles.runtime, poolclass=NullPool) + service = V2CountryService( + CountryDatabaseSession(sessionmaker(runtime_engine, class_=Session)) + ) + owner_engine = create_engine(disposable_v2_database_url, poolclass=NullPool) + + try: + service.verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + + with owner_engine.begin() as connection: + connection.execute( + text( + "INSERT INTO countries " + "(id, household_enabled, society_wide_enabled) " + "VALUES ('ca', true, false)" + ) + ) + with pytest.raises(CountryListMismatchError) as raised: + service.verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + assert raised.value.rows_not_listed == ("ca",) + finally: + with owner_engine.begin() as connection: + connection.execute(text("DELETE FROM countries WHERE id = 'ca'")) + owner_engine.dispose() + runtime_engine.dispose() diff --git a/tests/unit/test_alembic_workflows.py b/tests/unit/test_alembic_workflows.py index 797c82848..fa69a8378 100644 --- a/tests/unit/test_alembic_workflows.py +++ b/tests/unit/test_alembic_workflows.py @@ -221,6 +221,7 @@ def test_reusable_v2_integration_check_uses_disposable_databases_and_coverage(): assert "RUN_V2_CATALOG_COMPATIBILITY" in workflow assert "test_v2_catalog_publication.py" in workflow assert "test_v2_metadata_routes.py" in workflow + assert "test_v2_country_routes.py" in workflow assert "test_v2_policy_persistence.py" in workflow assert "test_v1_policy_dual_write.py" in workflow assert "test_mysql_policy_dual_write.py" in workflow diff --git a/tests/unit/test_migration_contract_artifacts.py b/tests/unit/test_migration_contract_artifacts.py index d08090feb..62ddabd71 100644 --- a/tests/unit/test_migration_contract_artifacts.py +++ b/tests/unit/test_migration_contract_artifacts.py @@ -13,8 +13,8 @@ def test_migration_contract_payload_summarizes_route_contracts(): assert payload["version"] == 1 assert payload["metadata"] == { "route_group_count": 9, - "workflow_count": 16, - "request_count": 89, + "workflow_count": 17, + "request_count": 90, "db_entity_count": 6, "sim_flow_count": 3, } @@ -31,6 +31,7 @@ def test_migration_contract_payload_summarizes_route_contracts(): "user_account_v2", "household_calculate", "region_selection", + "country_selection_v2", "metadata_resources_v2_preview", "simulation_submit_poll", "report_create_poll", diff --git a/tests/unit/test_startup_country_check.py b/tests/unit/test_startup_country_check.py new file mode 100644 index 000000000..64bd0f219 --- /dev/null +++ b/tests/unit/test_startup_country_check.py @@ -0,0 +1,229 @@ +"""The startup check that the v2 countries table matches the country list in code.""" + +from __future__ import annotations + +import logging +import os +import subprocess +import sys +from typing import TYPE_CHECKING + +import pytest +from sqlalchemy.exc import InterfaceError, OperationalError, ProgrammingError + +from policyengine_api import startup_country_check +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.services.v2.countries.validators import ( + CountryListMismatchError, +) +from policyengine_api.startup_country_check import check_countries_table_at_startup +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS + +if TYPE_CHECKING: + from collections.abc import Iterable + from pathlib import Path + +V2_RUNTIME_SECRET_RESOURCE = "projects/example/secrets/v2-runtime/versions/latest" + + +@pytest.fixture(autouse=True) +def enabled_startup_check_logger(monkeypatch: pytest.MonkeyPatch) -> None: + """Re-enable the check's logger, which Alembic's logging setup disables. + + Tests that run Alembic commands load its logging configuration, which + disables every logger that already exists, including this module's. + """ + monkeypatch.setattr(startup_country_check.logger, "disabled", False) + + +class FakeCountryListVerifier: + """Record each verification, then raise ``error`` if one is set.""" + + def __init__(self, error: Exception | None = None) -> None: + self.error = error + self.verified_country_lists: list[tuple[str, ...]] = [] + + def verify_countries_match_code_list( + self, listed_country_ids: Iterable[str] + ) -> None: + self.verified_country_lists.append(tuple(listed_country_ids)) + if self.error is not None: + raise self.error + + +class CountingVerifierFactory: + """Return one verifier and count how often startup asks for it.""" + + def __init__(self, verifier: FakeCountryListVerifier) -> None: + self.verifier = verifier + self.calls = 0 + + def __call__(self) -> FakeCountryListVerifier: + self.calls += 1 + return self.verifier + + +def test_startup_without_a_v2_runtime_database_skips_the_check() -> None: + factory = CountingVerifierFactory(FakeCountryListVerifier()) + + check_countries_table_at_startup( + environ={"FLASK_DEBUG": "1"}, verifier_factory=factory + ) + + assert factory.calls == 0 + + +def test_blank_selection_variables_skip_the_check() -> None: + factory = CountingVerifierFactory(FakeCountryListVerifier()) + + check_countries_table_at_startup( + environ={ + "K_SERVICE": "", + "V2_RUNTIME_DATABASE_URL": " ", + "V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE": "", + }, + verifier_factory=factory, + ) + + assert factory.calls == 0 + + +@pytest.mark.parametrize( + "environ", + [ + {"K_SERVICE": "policyengine-api"}, + {"V2_RUNTIME_DATABASE_URL": "postgresql+psycopg://configured"}, + {"V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE": V2_RUNTIME_SECRET_RESOURCE}, + ], + ids=["cloud-run", "runtime-url", "runtime-secret"], +) +def test_a_selected_v2_runtime_database_is_checked_against_the_code_list( + environ: dict[str, str], +) -> None: + verifier = FakeCountryListVerifier() + + check_countries_table_at_startup( + environ=environ, verifier_factory=CountingVerifierFactory(verifier) + ) + + assert verifier.verified_country_lists == [SUPPORTED_COUNTRY_IDS] + + +@pytest.mark.parametrize( + "error", + [ + CountryListMismatchError(listed_without_row=("us",), rows_not_listed=()), + ProgrammingError( + "SELECT countries.id FROM countries", + {}, + Exception('relation "countries" does not exist'), + ), + ProgrammingError( + "SELECT countries.id FROM countries", + {}, + Exception("permission denied for table countries"), + ), + ], + ids=["mismatch", "missing-table", "missing-privilege"], +) +def test_a_failed_check_stops_startup(error: Exception) -> None: + verifier = FakeCountryListVerifier(error=error) + + with pytest.raises(type(error)): + check_countries_table_at_startup( + environ={"K_SERVICE": "policyengine-api"}, + verifier_factory=CountingVerifierFactory(verifier), + ) + + +@pytest.mark.parametrize( + "message", + [ + "V2_RUNTIME_DATABASE_URL must name a non-local Postgres host", + "V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE could not be resolved", + ], + ids=["invalid-url", "secret-unavailable"], +) +def test_unusable_v2_configuration_is_logged_and_left_to_readiness( + message: str, + caplog: pytest.LogCaptureFixture, +) -> None: + """Readiness already reports not ready for it, and recovers when it does.""" + + def unusable_configuration() -> FakeCountryListVerifier: + raise V2ConfigurationError(message) + + with caplog.at_level(logging.WARNING): + check_countries_table_at_startup( + environ={"K_SERVICE": "policyengine-api"}, + verifier_factory=unusable_configuration, + ) + + assert [record.levelno for record in caplog.records] == [logging.WARNING] + assert message in caplog.records[0].getMessage() + + +@pytest.mark.parametrize( + "error_type", + [OperationalError, InterfaceError], + ids=["operational", "interface"], +) +def test_an_unreachable_database_is_logged_and_startup_continues( + error_type: type[OperationalError | InterfaceError], + caplog: pytest.LogCaptureFixture, +) -> None: + verifier = FakeCountryListVerifier( + error=error_type( + "SELECT countries.id FROM countries", + {}, + Exception("connection to db.secret-host failed: password=hunter2"), + ) + ) + + with caplog.at_level(logging.WARNING): + check_countries_table_at_startup( + environ={"K_SERVICE": "policyengine-api"}, + verifier_factory=CountingVerifierFactory(verifier), + ) + + assert verifier.verified_country_lists == [SUPPORTED_COUNTRY_IDS] + assert [record.levelno for record in caplog.records] == [logging.WARNING] + message = caplog.records[0].getMessage() + assert "countries table" in message + assert error_type.__name__ in message + assert "secret-host" not in message + assert "hunter2" not in message + + +def test_importing_the_asgi_application_runs_the_check(tmp_path: Path) -> None: + """The check runs at import; an unusable v2 runtime URL is only logged.""" + environment = os.environ.copy() + environment.update( + { + "POLICYENGINE_API_STARTUP_WARMUP": "0", + "V2_RUNTIME_DATABASE_URL": ( + "postgresql+psycopg://runtime:hunter2@localhost:5432/postgres" + "?sslmode=require" + ), + "V2_SUPABASE_PROJECT_REF": "abcdefghijklmnopqrst", + "V2_SUPABASE_ENVIRONMENT": "test-startup", + } + ) + for name in ("V2_RUNTIME_DATABASE_URL_SECRET_RESOURCE", "K_SERVICE"): + environment.pop(name, None) + + result = subprocess.run( + [sys.executable, "-c", "import policyengine_api.asgi"], + cwd=tmp_path, + env=environment, + capture_output=True, + text=True, + check=False, + ) + + # The application's log handler writes structured records to stdout. + output = result.stdout + result.stderr + assert result.returncode == 0, result.stderr + assert "Skipped the countries table check" in output + assert "V2_RUNTIME_DATABASE_URL must name a non-local Postgres host" in output + assert "hunter2" not in output diff --git a/tests/unit/test_supported_countries.py b/tests/unit/test_supported_countries.py new file mode 100644 index 000000000..429f27657 --- /dev/null +++ b/tests/unit/test_supported_countries.py @@ -0,0 +1,139 @@ +"""The Public API's one country list, and the lists that derive from it.""" + +from __future__ import annotations + +import json +import subprocess +import sys +from typing import get_args + +import pytest +from pydantic import TypeAdapter, ValidationError + +from policyengine_api.constants import REPO +from policyengine_api.data.v2.catalog import extraction +from policyengine_api.query_parameters import CountryId +from policyengine_api.supported_countries import ( + SUPPORTED_COUNTRY_IDS, + SupportedCountryId, +) + +# Runs in a fresh interpreter: it adds a country to the one list before any +# module that derives from it is imported, then reports what each derived +# list contains. PolicyEngine.py is replaced by a stand-in, because importing +# the real package builds every country model. +EXTENDED_LIST_SCRIPT = """ +import json +import sys +from types import ModuleType, SimpleNamespace +from typing import Literal, get_args + +import policyengine_api.supported_countries as supported_countries + +supported_countries.SupportedCountryId = Literal["us", "uk", "zz"] +supported_countries.SUPPORTED_COUNTRY_IDS = get_args( + supported_countries.SupportedCountryId +) + +installed_policyengine = ModuleType("policyengine") +for country_id in ("us", "uk", "zz", "ca"): + setattr( + installed_policyengine, + country_id, + SimpleNamespace(model=f"{country_id} model"), + ) +installed_bundle = ModuleType("policyengine.bundle") +installed_bundle.get_current_bundle = dict +sys.modules["policyengine"] = installed_policyengine +sys.modules["policyengine.bundle"] = installed_bundle + +from pydantic import TypeAdapter + +from policyengine_api.data.v2.catalog import catalog_selection, extraction +from policyengine_api.query_parameters import CountryId +from scripts import process_v1_household_mirror_event + +captured_arguments = {} + + +def capture_extraction(**arguments): + captured_arguments.update(arguments) + + +extraction.extract_catalog = capture_extraction +extraction.extract_installed_catalog() +country_option = next( + action + for action in process_v1_household_mirror_event._parser()._actions + if action.dest == "country_id" +) +print( + json.dumps( + { + "country_type": TypeAdapter(CountryId).validate_python("ZZ"), + "extracted_countries": list(extraction.SUPPORTED_COUNTRIES), + "required_dependencies": list(extraction.REQUIRED_DEPENDENCIES), + "installed_models": captured_arguments["models"], + "dual_write_countries": sorted(catalog_selection.SUPPORTED_V2_COUNTRY_IDS), + "household_copy_countries": list(country_option.choices), + } + ) +) +""" + + +def test_the_country_list_is_the_us_and_the_uk_in_catalog_order() -> None: + assert SUPPORTED_COUNTRY_IDS == ("us", "uk") + assert get_args(SupportedCountryId) == SUPPORTED_COUNTRY_IDS + + +def test_the_v2_country_type_accepts_exactly_the_listed_countries() -> None: + country_id_adapter = TypeAdapter(CountryId) + + for country_id in SUPPORTED_COUNTRY_IDS: + assert country_id_adapter.validate_python(country_id) == country_id + assert country_id_adapter.validate_python(country_id.upper()) == country_id + for unlisted_country_id in ("ca", "ng", "il", ""): + with pytest.raises(ValidationError): + country_id_adapter.validate_python(unlisted_country_id) + + +def test_every_country_list_follows_a_country_added_to_the_one_list() -> None: + result = subprocess.run( + [sys.executable, "-c", EXTENDED_LIST_SCRIPT], + cwd=REPO, + capture_output=True, + text=True, + check=False, + ) + + assert result.returncode == 0, result.stderr + derived_lists = json.loads(result.stdout.strip().splitlines()[-1]) + assert derived_lists == { + "country_type": "zz", + "extracted_countries": ["us", "uk", "zz"], + "required_dependencies": [ + "policyengine-core", + "policyengine-us", + "policyengine-uk", + "policyengine-zz", + ], + # The installed package's other countries are not extracted. + "installed_models": { + "us": "us model", + "uk": "uk model", + "zz": "zz model", + }, + "dual_write_countries": ["uk", "us", "zz"], + "household_copy_countries": ["us", "uk", "zz"], + } + + +def test_catalog_settings_cover_exactly_the_listed_countries() -> None: + # Each listed country needs reviewed catalog settings, so adding a country + # without them fails here rather than during publication. + listed_countries = set(SUPPORTED_COUNTRY_IDS) + + assert set(extraction.REVIEWED_DEFAULT_DATASETS) == listed_countries + assert set(extraction.CURRENT_LAW_IDS) == listed_countries + assert set(extraction.METADATA_TIME_PERIODS) == listed_countries diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index 4c0e6ee09..56e31410b 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -40,6 +40,7 @@ V2_TABLE_NAMES = frozenset(table.name for table in V2_METADATA.tables.values()) POST_BASELINE_TABLE_NAMES = frozenset( { + "countries", "legacy_household_mappings", "legacy_policy_mappings", "legacy_user_mappings", @@ -235,6 +236,7 @@ def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> N assert script.get_heads() == [V2_ALEMBIC_HEAD_REVISION] assert [revision.revision for revision in script.walk_revisions()] == [ V2_ALEMBIC_HEAD_REVISION, + "fbb7488ebe8a", "91af4fc8d8c1", "60d6518b6a98", "439303be14fe", @@ -495,7 +497,6 @@ def test_stage_12a_user_identity_and_association_revision_is_generated() -> None / "migrations/v2/versions/fbb7488ebe8a_link_signed_in_users_to_v1_anchored_.py" ).read_text(encoding="utf-8") - assert V2_ALEMBIC_HEAD_REVISION == "fbb7488ebe8a" assert ( "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" in revision @@ -565,6 +566,94 @@ def test_stage_12a_user_identity_and_association_revision_is_generated() -> None ) in upgrade +COUNTRY_REFERENCE_COLUMNS = ( + ("users", "primary_country"), + ("households", "country_id"), + ("policies", "country_id"), + ("user_household_associations", "country_id"), + ("user_policies", "country_id"), + ("user_simulation_associations", "country_id"), + ("user_report_associations", "country_id"), +) +# The generated downgrade restores each check in PostgreSQL's reflected form +# of `` IN ('us', 'uk')``. +US_AND_UK = "ANY (ARRAY['us'::character varying, 'uk'::character varying]::text[])" +REPLACED_COUNTRY_CHECKS = { + "ck_users_primary_country": ("users", f"primary_country::text = {US_AND_UK}"), + "ck_households_country": ("households", f"country_id::text = {US_AND_UK}"), + "ck_policies_country": ("policies", f"country_id::text = {US_AND_UK}"), + "ck_user_household_associations_country": ( + "user_household_associations", + f"country_id::text = {US_AND_UK}", + ), + "ck_user_policies_country": ("user_policies", f"country_id::text = {US_AND_UK}"), + "ck_user_simulation_associations_country": ( + "user_simulation_associations", + f"country_id::text = {US_AND_UK}", + ), + "ck_user_report_associations_country": ( + "user_report_associations", + f"country_id::text = {US_AND_UK}", + ), +} + + +def _countries_revision_source() -> str: + script = ScriptDirectory.from_config(Config(str(REPO / "alembic-v2.ini"))) + countries_revision = next( + revision + for revision in script.walk_revisions() + if revision.down_revision == "fbb7488ebe8a" + ) + assert countries_revision.revision == V2_ALEMBIC_HEAD_REVISION + assert countries_revision.path is not None + return Path(countries_revision.path).read_text(encoding="utf-8") + + +def test_countries_revision_inserts_us_and_uk_and_replaces_country_checks() -> None: + revision = _countries_revision_source() + upgrade = revision[ + revision.index("def upgrade()") : revision.index("def downgrade()") + ] + downgrade = revision[revision.index("def downgrade()") :] + + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert revision.count("op.create_table(") == 1 + assert 'op.create_table(\n "countries"' in upgrade + assert "ck_countries_society_wide_requires_household" in upgrade + assert 'op.drop_table("countries")' in downgrade + # The one hand-written operation, an approved exception to generated-only + # revisions: the US and UK rows, inserted before the foreign keys that + # existing rows must satisfy. The generated downgrade drops them with the + # table. + assert revision.count("Approved hand-written data operation") == 1 + assert revision.count("Post-generation correction") == 0 + assert revision.count("op.bulk_insert(") == 1 + assert "op.execute(" not in revision + inserted_rows = upgrade[ + upgrade.index("op.bulk_insert(") : upgrade.index("op.create_foreign_key(") + ] + for country_id in ("uk", "us"): + assert f'"id": "{country_id}"' in inserted_rows + assert inserted_rows.count('"household_enabled": True') == 2 + assert inserted_rows.count('"society_wide_enabled": True') == 2 + for table_name, column_name in COUNTRY_REFERENCE_COLUMNS: + key_name = f"fk_{table_name}_{column_name}_countries" + assert key_name in upgrade + assert key_name in downgrade + assert upgrade.count('ondelete="RESTRICT"') == len(COUNTRY_REFERENCE_COLUMNS) + for check_name, (table_name, expression) in REPLACED_COUNTRY_CHECKS.items(): + assert check_name in upgrade + assert f'"{table_name}"' in upgrade + assert check_name in downgrade + assert expression in downgrade + assert "op.drop_column(" not in upgrade + assert "op.add_column(" not in upgrade + + def test_alembic_rejects_unknown_missing_and_divergent_history(tmp_path: Path) -> None: original = REPO / "migrations/v2" missing = tmp_path / "missing" diff --git a/tests/unit/v2/test_country_routes.py b/tests/unit/v2/test_country_routes.py new file mode 100644 index 000000000..7fd2e2b91 --- /dev/null +++ b/tests/unit/v2/test_country_routes.py @@ -0,0 +1,198 @@ +"""Route coverage for the native v2 countries list.""" + +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient +from flask import Flask, jsonify +from sqlalchemy.exc import OperationalError + +from policyengine_api.asgi_factory import create_asgi_app +from policyengine_api.data.v2.settings import V2ConfigurationError +from policyengine_api.fastapi_routes.dependencies import NativeRouteDependencies +from policyengine_api.migration_flags import ( + RouteImplementation, + RouteImplementationSettings, + infer_route_group, +) +from policyengine_api.services.v2.countries.types import CountryRead + +UNITED_KINGDOM = CountryRead(id="uk", household_enabled=True, society_wide_enabled=True) +UNITED_STATES = CountryRead(id="us", household_enabled=True, society_wide_enabled=True) +HOUSEHOLD_ONLY_COUNTRY = CountryRead( + id="be", household_enabled=True, society_wide_enabled=False +) + + +class FakeCountryService: + """Return fixed countries, or raise ``error``, and count the calls.""" + + def __init__( + self, + countries: list[CountryRead], + *, + error: Exception | None = None, + ) -> None: + self.countries = countries + self.error = error + self.list_calls = 0 + + def list_countries(self) -> list[CountryRead]: + self.list_calls += 1 + if self.error is not None: + raise self.error + return self.countries + + +def _client(service: FakeCountryService) -> tuple[TestClient, dict[str, int]]: + flask_calls = {"count": 0} + flask_app = Flask(__name__) + + @flask_app.route("/", methods=["GET", "POST"]) + def fallback(resource: str): + flask_calls["count"] += 1 + return jsonify({"source": "flask", "resource": resource}) + + dependencies = NativeRouteDependencies( + readiness_probe=lambda: True, + gateway_client_factory=lambda: None, + metadata_reader_factory=lambda: None, + specification_provider=dict, + v2_country_service_factory=lambda: service, + ) + settings = RouteImplementationSettings( + health=RouteImplementation.FLASK_FALLBACK, + specification=RouteImplementation.FLASK_FALLBACK, + metadata=RouteImplementation.FLASK_FALLBACK, + ) + client = TestClient( + create_asgi_app(flask_app, dependencies=dependencies, route_settings=settings), + raise_server_exceptions=False, + ) + return client, flask_calls + + +def test_countries_list_every_country_with_its_capability() -> None: + service = FakeCountryService( + [HOUSEHOLD_ONLY_COUNTRY, UNITED_KINGDOM, UNITED_STATES] + ) + client, flask_calls = _client(service) + + response = client.get("/v2/countries") + + assert response.status_code == 200 + assert response.json() == { + "status": "ok", + "message": None, + "result": { + "items": [ + { + "id": "be", + "household_enabled": True, + "society_wide_enabled": False, + }, + { + "id": "uk", + "household_enabled": True, + "society_wide_enabled": True, + }, + { + "id": "us", + "household_enabled": True, + "society_wide_enabled": True, + }, + ] + }, + } + assert service.list_calls == 1 + assert flask_calls["count"] == 0 + + +def test_countries_list_needs_no_user_credential() -> None: + client, _flask_calls = _client(FakeCountryService([UNITED_STATES])) + + response = client.get("/v2/countries", headers={"Authorization": ""}) + + assert response.status_code == 200 + + +def test_countries_list_rejects_query_parameters() -> None: + service = FakeCountryService([UNITED_STATES]) + client, _flask_calls = _client(service) + + response = client.get("/v2/countries?country_id=us") + + assert response.status_code == 422 + assert response.json()["status"] == "error" + assert service.list_calls == 0 + + +@pytest.mark.parametrize( + "error", + [ + V2ConfigurationError("V2_RUNTIME_DATABASE_URL is required"), + OperationalError("SELECT", {}, Exception("connection refused")), + ], + ids=["configuration", "database"], +) +def test_countries_list_reports_an_unavailable_database_as_503( + error: Exception, +) -> None: + client, _flask_calls = _client(FakeCountryService([], error=error)) + + response = client.get("/v2/countries") + + assert response.status_code == 503 + assert response.json() == { + "status": "error", + "message": "V2 countries are unavailable", + } + + +def test_countries_list_hides_unexpected_failures_behind_a_500() -> None: + client, _flask_calls = _client( + FakeCountryService([], error=RuntimeError("secret detail")) + ) + + response = client.get("/v2/countries") + + assert response.status_code == 500 + assert response.json() == { + "status": "error", + "message": "V2 countries query failed", + } + + +def test_countries_list_supports_get_only() -> None: + service = FakeCountryService([UNITED_STATES]) + client, flask_calls = _client(service) + + response = client.post("/v2/countries", json={}) + + assert response.status_code == 405 + assert service.list_calls == 0 + assert flask_calls["count"] == 0 + + +def test_countries_list_is_documented_in_the_v2_openapi_document() -> None: + client, _flask_calls = _client(FakeCountryService([])) + + document = client.get("/v2/openapi.json").json() + operation = document["paths"]["/v2/countries"]["get"] + + assert operation.get("parameters", []) == [] + assert operation["responses"]["200"]["content"]["application/json"]["schema"] == { + "$ref": "#/components/schemas/CountryListResponse" + } + assert {"422", "500", "503"} <= operation["responses"].keys() + item_schema = document["components"]["schemas"]["CountryItem"] + assert item_schema["required"] == [ + "id", + "household_enabled", + "society_wide_enabled", + ] + assert item_schema["additionalProperties"] is False + + +def test_countries_requests_are_logged_with_the_metadata_route_group() -> None: + assert infer_route_group("/v2/countries") == "metadata" diff --git a/tests/unit/v2/test_country_service.py b/tests/unit/v2/test_country_service.py new file mode 100644 index 000000000..253278599 --- /dev/null +++ b/tests/unit/v2/test_country_service.py @@ -0,0 +1,85 @@ +"""SQLite-backed coverage for the v2 countries service: the list and the check.""" + +from __future__ import annotations + +import pytest +from sqlalchemy.orm import sessionmaker +from sqlmodel import Session, create_engine + +from policyengine_api.data.v2.models import V2_METADATA, Country +from policyengine_api.services.v2.countries.database_session import ( + CountryDatabaseSession, +) +from policyengine_api.services.v2.countries.services import V2CountryService +from policyengine_api.services.v2.countries.types import CountryRead +from policyengine_api.services.v2.countries.validators import ( + CountryListMismatchError, +) +from policyengine_api.supported_countries import SUPPORTED_COUNTRY_IDS +from tests.fixtures.v2_countries import add_listed_countries + + +def _service_with_countries(*extra_countries: Country) -> V2CountryService: + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + sessions = sessionmaker(engine, class_=Session) + with sessions() as session: + add_listed_countries(session) + session.add_all(extra_countries) + session.commit() + return V2CountryService(CountryDatabaseSession(sessions)) + + +def test_countries_are_listed_by_id_with_their_capability_flags() -> None: + service = _service_with_countries( + Country(id="be", household_enabled=True, society_wide_enabled=False) + ) + + countries = service.list_countries() + + assert countries == [ + CountryRead(id="be", household_enabled=True, society_wide_enabled=False), + CountryRead(id="uk", household_enabled=True, society_wide_enabled=True), + CountryRead(id="us", household_enabled=True, society_wide_enabled=True), + ] + + +def test_an_empty_countries_table_lists_no_countries() -> None: + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + service = V2CountryService( + CountryDatabaseSession(sessionmaker(engine, class_=Session)) + ) + + assert service.list_countries() == [] + + +def test_rows_for_exactly_the_listed_countries_pass_verification() -> None: + service = _service_with_countries() + + service.verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + + +def test_a_row_for_an_unlisted_country_fails_verification() -> None: + service = _service_with_countries( + Country(id="ca", household_enabled=True, society_wide_enabled=False) + ) + + with pytest.raises(CountryListMismatchError) as raised: + service.verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + + assert raised.value.rows_not_listed == ("ca",) + assert raised.value.listed_without_row == () + + +def test_an_empty_countries_table_fails_verification() -> None: + engine = create_engine("sqlite://") + V2_METADATA.create_all(engine) + service = V2CountryService( + CountryDatabaseSession(sessionmaker(engine, class_=Session)) + ) + + with pytest.raises(CountryListMismatchError) as raised: + service.verify_countries_match_code_list(SUPPORTED_COUNTRY_IDS) + + assert raised.value.listed_without_row == tuple(sorted(SUPPORTED_COUNTRY_IDS)) diff --git a/tests/unit/v2/test_country_validators.py b/tests/unit/v2/test_country_validators.py new file mode 100644 index 000000000..d56db5c00 --- /dev/null +++ b/tests/unit/v2/test_country_validators.py @@ -0,0 +1,68 @@ +"""The comparison of stored countries rows with the country list in code.""" + +from __future__ import annotations + +import pytest + +from policyengine_api.services.v2.countries.validators import ( + CountryListMismatchError, + validate_countries_match_code_list, +) + + +def test_the_same_countries_in_any_order_match() -> None: + validate_countries_match_code_list( + stored_country_ids=["us", "uk"], + listed_country_ids=("uk", "us"), + ) + + +def test_a_listed_country_without_a_row_is_a_mismatch() -> None: + with pytest.raises(CountryListMismatchError) as raised: + validate_countries_match_code_list( + stored_country_ids=["uk"], + listed_country_ids=("uk", "us"), + ) + + assert raised.value.listed_without_row == ("us",) + assert raised.value.rows_not_listed == () + assert str(raised.value) == ( + "the countries table does not match the country list in code: " + "listed without a row: us; rows not listed: none" + ) + + +def test_a_row_for_an_unlisted_country_is_a_mismatch() -> None: + with pytest.raises(CountryListMismatchError) as raised: + validate_countries_match_code_list( + stored_country_ids=["ca", "uk", "us"], + listed_country_ids=("uk", "us"), + ) + + assert raised.value.listed_without_row == () + assert raised.value.rows_not_listed == ("ca",) + + +def test_every_difference_is_reported_in_order() -> None: + with pytest.raises(CountryListMismatchError) as raised: + validate_countries_match_code_list( + stored_country_ids=["ng", "ca"], + listed_country_ids=("us", "uk"), + ) + + assert raised.value.listed_without_row == ("uk", "us") + assert raised.value.rows_not_listed == ("ca", "ng") + assert str(raised.value) == ( + "the countries table does not match the country list in code: " + "listed without a row: uk, us; rows not listed: ca, ng" + ) + + +def test_an_empty_table_misses_every_listed_country() -> None: + with pytest.raises(CountryListMismatchError) as raised: + validate_countries_match_code_list( + stored_country_ids=[], + listed_country_ids=("uk", "us"), + ) + + assert raised.value.listed_without_row == ("uk", "us") diff --git a/tests/unit/v2/test_household_models.py b/tests/unit/v2/test_household_models.py index f0798e35e..bef59c843 100644 --- a/tests/unit/v2/test_household_models.py +++ b/tests/unit/v2/test_household_models.py @@ -18,6 +18,7 @@ User, UserHouseholdAssociation, ) +from tests.fixtures.v2_countries import add_listed_countries def _engine(): @@ -30,6 +31,8 @@ def enable_foreign_keys(dbapi_connection, _connection_record) -> None: cursor.close() V2_METADATA.create_all(engine) + with Session(engine) as session: + add_listed_countries(session) return engine diff --git a/tests/unit/v2/test_import_side_effects.py b/tests/unit/v2/test_import_side_effects.py index 7b753004d..af6913897 100644 --- a/tests/unit/v2/test_import_side_effects.py +++ b/tests/unit/v2/test_import_side_effects.py @@ -69,7 +69,7 @@ def reject_ddl(*args, **kwargs): import sys after = set(pathlib.Path.cwd().iterdir()) assert before == after -assert len(V2_METADATA.tables) == 39 +assert len(V2_METADATA.tables) == 40 assert "policyengine_api.data.v2.catalog.initialization" not in sys.modules assert "policyengine_api.data.v2.policy_migration_qualification" not in sys.modules assert "policyengine_api.data.v2.household_migration_qualification" not in sys.modules diff --git a/tests/unit/v2/test_metadata_routes.py b/tests/unit/v2/test_metadata_routes.py index 79ec7c1ed..1c2e542a5 100644 --- a/tests/unit/v2/test_metadata_routes.py +++ b/tests/unit/v2/test_metadata_routes.py @@ -538,6 +538,7 @@ def test_openapi_references_explicit_resource_response_schemas() -> None: "/v2/variables/{variable_id}", } native_paths = { + "/v2/countries", "/v2/households", "/v2/households/{household_id}", "/v2/policies", diff --git a/tests/unit/v2/test_model_persistence.py b/tests/unit/v2/test_model_persistence.py index be7879424..732fa7d98 100644 --- a/tests/unit/v2/test_model_persistence.py +++ b/tests/unit/v2/test_model_persistence.py @@ -10,7 +10,9 @@ from policyengine_api.data.v2.models import ( V2_METADATA, + Country, Dynamic, + Household, LegacyPolicyMapping, LegacyUserMapping, LegacyUserPolicyMapping, @@ -20,9 +22,11 @@ TaxBenefitModel, TaxBenefitModelVersion, User, + UserHouseholdAssociation, UserPolicy, ) from policyengine_api.data.v2.models.base import DIRECT_SQLALCHEMY_EXCEPTIONS +from tests.fixtures.v2_countries import add_listed_countries def _relational_sqlite_engine(): @@ -35,6 +39,8 @@ def enable_foreign_keys(dbapi_connection, _connection_record) -> None: cursor.close() V2_METADATA.create_all(engine) + with Session(engine) as session: + add_listed_countries(session) return engine @@ -99,9 +105,8 @@ def test_direct_sqlalchemy_categories_are_complete_and_documented() -> None: assert all(DIRECT_SQLALCHEMY_EXCEPTIONS.values()) -def test_user_primary_country_can_change_between_us_and_uk_only() -> None: - engine = create_engine("sqlite://") - V2_METADATA.create_all(engine) +def test_user_primary_country_must_be_a_listed_country() -> None: + engine = _relational_sqlite_engine() with Session(engine) as session: user = User( @@ -126,6 +131,73 @@ def test_user_primary_country_can_change_between_us_and_uk_only() -> None: engine.dispose() +def test_a_country_row_lets_records_use_a_household_only_country() -> None: + engine = _relational_sqlite_engine() + household_only_country = Country( + id="be", + household_enabled=True, + society_wide_enabled=False, + ) + + with Session(engine) as session: + session.add(User(primary_country="be")) + with pytest.raises(IntegrityError): + session.commit() + session.rollback() + + session.add(household_only_country) + session.commit() + user = User(primary_country="be") + household = Household( + country_id="be", + household_data={"people": {}}, + canonicalization_version=1, + content_hash="b" * 64, + ) + session.add_all([user, household]) + session.flush() + session.add( + UserHouseholdAssociation( + user_id=user.id, + household_id=household.id, + country_id="be", + ) + ) + session.commit() + + stored_country = session.exec(select(UserHouseholdAssociation.country_id)).one() + assert stored_country == "be" + + engine.dispose() + + +def test_a_listed_country_cannot_be_deleted_while_records_use_it() -> None: + engine = _relational_sqlite_engine() + + with Session(engine) as session: + session.add(User(primary_country="uk")) + session.commit() + + session.delete(session.get(Country, "uk")) + with pytest.raises(IntegrityError): + session.commit() + + engine.dispose() + + +def test_a_society_wide_country_must_also_be_household_capable() -> None: + engine = _relational_sqlite_engine() + + with Session(engine) as session: + session.add( + Country(id="be", household_enabled=False, society_wide_enabled=True) + ) + with pytest.raises(IntegrityError): + session.commit() + + engine.dispose() + + def test_only_unauthenticated_users_may_omit_a_primary_country() -> None: engine = create_engine("sqlite://") V2_METADATA.create_all(engine) diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index 20947ffcb..baa0e2590 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -8,6 +8,7 @@ from policyengine_api.data.v1_models import V1Base from policyengine_api.data.v2.models import ( V2_METADATA, + Country, DatasetVersion, Dynamic, Household, @@ -29,6 +30,18 @@ UserType, ) +# Every column that names the country a record belongs to. Each references the +# countries table instead of checking a fixed list of country IDs. +COUNTRY_REFERENCE_COLUMNS = ( + ("users", "primary_country"), + ("households", "country_id"), + ("policies", "country_id"), + ("user_household_associations", "country_id"), + ("user_policies", "country_id"), + ("user_simulation_associations", "country_id"), + ("user_report_associations", "country_id"), +) + RUN_OUTPUT_TABLES = frozenset( { "aggregates", @@ -48,6 +61,7 @@ def test_domain_models_are_grouped_into_topic_scoped_modules() -> None: expected_modules = { + Country: "countries", User: "users", UserIdentity: "users", Policy: "policies", @@ -354,8 +368,8 @@ def test_standard_users_require_a_supported_primary_country() -> None: for constraint in users.constraints if isinstance(constraint, sa.CheckConstraint) } + # The countries table, not a check, lists the countries a user may choose. assert checks == { - "ck_users_primary_country": "primary_country IN ('us', 'uk')", "ck_users_user_type": "user_type IN ('unauthenticated', 'standard')", "ck_users_standard_primary_country": ( "user_type = 'unauthenticated' OR primary_country IS NOT NULL" @@ -474,7 +488,8 @@ def test_named_checks_and_required_indexes_cover_core_invariants() -> None: assert { "ck_regions_required_filter_values", "ck_simulations_type_input", - "ck_users_primary_country", + "ck_users_standard_primary_country", + "ck_countries_society_wide_requires_household", "ck_report_runs_terminal_completion", "ck_parameter_values_single_owner", }.issubset(constraint_names) @@ -607,10 +622,11 @@ def test_simulation_and_report_associations_have_no_cross_service_keys() -> None assert table.c.label.nullable assert { foreign_key.target_fullname for foreign_key in simulations.foreign_keys - } == {"users.id"} + } == {"users.id", "countries.id"} assert {foreign_key.target_fullname for foreign_key in reports.foreign_keys} == { "users.id", "user_policies.id", + "countries.id", } user_policy_key = next(iter(reports.c.user_policy_id.foreign_keys)) assert user_policy_key.ondelete == "SET NULL" @@ -635,3 +651,71 @@ def test_user_policy_records_v3_reform_fields() -> None: assert "current-policy" in str(baseline_check.sqltext) postgres_type = policies.c.provenance.type.dialect_impl(postgresql.dialect()) assert isinstance(postgres_type, postgresql.JSONB) + + +def test_countries_record_household_and_society_wide_capability() -> None: + countries = V2_METADATA.tables["countries"] + + assert countries.c.keys() == ["id", "household_enabled", "society_wide_enabled"] + assert countries.primary_key.columns.keys() == ["id"] + assert countries.primary_key.name == "pk_countries" + assert countries.c.id.type.length == 2 + for flag_name in ("household_enabled", "society_wide_enabled"): + assert isinstance(countries.c[flag_name].type, sa.Boolean) + assert not countries.c[flag_name].nullable + assert countries.c[flag_name].server_default is None + checks = { + constraint.name: str(constraint.sqltext) + for constraint in countries.constraints + if isinstance(constraint, sa.CheckConstraint) + } + # Society-wide reports need household calculations, so a society-wide + # country is always household capable too. + assert checks == { + "ck_countries_society_wide_requires_household": ( + "household_enabled OR NOT society_wide_enabled" + ), + } + + +def test_country_columns_reference_the_countries_table_instead_of_a_fixed_list() -> ( + None +): + for table_name, column_name in COUNTRY_REFERENCE_COLUMNS: + table = V2_METADATA.tables[table_name] + country_keys = [ + foreign_key + for foreign_key in table.c[column_name].foreign_keys + if foreign_key.column.table.name == "countries" + ] + assert len(country_keys) == 1, table_name + assert country_keys[0].target_fullname == "countries.id" + assert country_keys[0].ondelete == "RESTRICT" + assert country_keys[0].constraint.name == ( + f"fk_{table_name}_{column_name}_countries" + ) + fixed_country_checks = [ + constraint.name + for constraint in table.constraints + if isinstance(constraint, sa.CheckConstraint) + and "'us', 'uk'" in str(constraint.sqltext) + ] + assert fixed_country_checks == [], table_name + + +def test_v1_copy_mappings_keep_their_us_and_uk_checks() -> None: + # These tables pair v1 records with their v2 copies. Each already + # references a v2 record of the same country, and they are removed with + # dual-write, so they keep their fixed checks. + for table_name in ( + "legacy_household_mappings", + "legacy_policy_mappings", + "legacy_user_policy_mappings", + ): + table = V2_METADATA.tables[table_name] + assert f"ck_{table_name}_country" in { + constraint.name for constraint in table.constraints + } + assert "countries.id" not in { + foreign_key.target_fullname for foreign_key in table.foreign_keys + } diff --git a/tests/unit/v2/user_service_support.py b/tests/unit/v2/user_service_support.py index ce54237e3..209a63dc5 100644 --- a/tests/unit/v2/user_service_support.py +++ b/tests/unit/v2/user_service_support.py @@ -31,6 +31,7 @@ from policyengine_api.services.v2.legacy_refs.validators import ( require_legacy_references, ) +from tests.fixtures.v2_countries import add_listed_countries if TYPE_CHECKING: from collections.abc import Collection, Iterator, Mapping @@ -128,6 +129,8 @@ def begin_transaction(connection) -> None: V2_METADATA.create_all(engine) sessions = sessionmaker(engine, class_=Session, expire_on_commit=False) + with sessions() as session: + add_listed_countries(session) with sessions.begin() as session: model = TaxBenefitModel(name="policyengine-us") version = TaxBenefitModelVersion(