From d4497debfdd40d7e02f109efec887a44b359ce67 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:54:48 +0400 Subject: [PATCH 1/4] Drop the simulation-domain tables, their enum types and retired columns The simulation service owns household jobs, simulations, reports, report runs and their outputs in its own simulation schema. The previous release removed the Public API's models of those 15 tables and stopped mapping the retired country and label columns of the simulation and report associations, keeping them only as schema-only declarations. Delete those declarations. Revision 363f34954a14, autogenerated, drops the 15 tables and the four columns. Its post-generation corrections, commented in the revision and documented in the Alembic migrations skill: - drop each table before the tables its foreign keys reference, and create them in the reverse order in the downgrade (Alembic emits removed tables in arbitrary order, which fails on the foreign keys); - drop the eight native enum types the tables used (Alembic leaves them, and the generated downgrade could not then create them again); - restore the parentheses that reflection strips from ck_simulations_filter_pair, which left invalid SQL in the downgrade. V2_ALEMBIC_HEAD_REVISION moves to the new revision. The v2 metadata validation rejects the 15 table names, and the privilege manifest no longer lists them. The lifecycle test covers the upgrade, the downgrade to the previous revision and the re-upgrade, and its test of the previous application revision now runs against the revision before this one: head drops exactly these tables and columns, which that release no longer maps, and serves everything else it maps. The Stage 12A rollback steps and the Stage 12 worker contract say what this removal changes. This must deploy only after the previous release is deployed in both environments and an operator has confirmed that the tables are empty. Co-Authored-By: Claude Opus 5.5 --- changelog.d/drop-simulation-tables.removed.md | 1 + docs/engineering/skills/alembic-migrations.md | 33 + docs/migration/stage-12-worker-contract.md | 4 +- docs/migration/stage-12a-user-login.md | 17 +- ...4a14_drop_simulation_domain_tables_and_.py | 1361 +++++++++++++++++ .../v2/household_migration_qualification.py | 4 +- .../data/v2/metadata_validation.py | 38 +- policyengine_api/data/v2/migration_target.py | 2 +- policyengine_api/data/v2/models/__init__.py | 5 - .../data/v2/models/associations.py | 5 - policyengine_api/data/v2/models/base.py | 5 - .../data/v2/models/retired_tables.py | 519 ------- policyengine_api/data/v2/privileges.py | 17 - .../integration/test_alembic_v2_lifecycle.py | 165 +- .../test_v2_catalog_publication.py | 33 - tests/unit/v2/test_alembic_v2.py | 125 +- tests/unit/v2/test_import_side_effects.py | 2 +- tests/unit/v2/test_metadata_validation.py | 36 +- tests/unit/v2/test_model_persistence.py | 1 - tests/unit/v2/test_models.py | 45 +- 20 files changed, 1753 insertions(+), 665 deletions(-) create mode 100644 changelog.d/drop-simulation-tables.removed.md create mode 100644 migrations/v2/versions/363f34954a14_drop_simulation_domain_tables_and_.py delete mode 100644 policyengine_api/data/v2/models/retired_tables.py diff --git a/changelog.d/drop-simulation-tables.removed.md b/changelog.d/drop-simulation-tables.removed.md new file mode 100644 index 000000000..ddf8e37c5 --- /dev/null +++ b/changelog.d/drop-simulation-tables.removed.md @@ -0,0 +1 @@ +Dropped the Public API's copies of the 15 simulation-domain v2 tables, their 8 PostgreSQL enum types and the retired `country` and `label` columns of the simulation and report associations; the simulation service owns those tables in its own schema. diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 9bcf0139a..2f02a6712 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -382,3 +382,36 @@ last dependent table is dropped. Its only post-generation correction drops the nine generated `v2_*` enum types at the end of the baseline downgrade. The v2 Postgres lifecycle test covers empty upgrade, downgrade to base, and re-upgrade so stale enum types cannot make the generated baseline non-reversible. + +### Simulation-domain table removal + +Revision `363f34954a14` was autogenerated after the last schema-only +declarations of the simulation-domain tables were deleted (an earlier release +had already removed their models and stopped reading or writing them). It +drops the 15 simulation-domain tables (`household_jobs`, `simulations`, +`reports`, `report_runs` and the 11 report-run output tables), which the +simulation service owns in its own `simulation` schema, and the retired +`country` and `label` columns of `user_simulation_associations` and +`user_report_associations`. Its post-generation corrections: + +- Alembic emits removed tables in arbitrary order. The upgrade drops each + table before the tables its foreign keys reference, and the downgrade + creates them in the reverse order. +- Alembic does not generate the removal of the dropped tables' eight native + enum types, and the generated downgrade could not create them again while + they remained. The upgrade drops them. +- Reflection strips the outer parentheses of `ck_simulations_filter_pair`, + which leaves invalid SQL in the generated downgrade. They are restored as + the model declared them. + +The v2 lifecycle test covers the upgrade, the downgrade to the previous +revision and the re-upgrade, and checks that head serves the previous +application revision, which no longer maps any dropped table or column. The +downgrade recreates the tables and columns empty. + +The revision must not deploy while an application revision that maps the +dropped tables or columns can serve traffic, and it may run only after an +operator has confirmed in each environment that the tables are empty. Once it +has run, an application rollback to a revision that maps them, including any +pre-Stage 12A revision, needs the schema downgraded to `fbb7488ebe8a` first. +The v2 metadata validation rejects their table names. diff --git a/docs/migration/stage-12-worker-contract.md b/docs/migration/stage-12-worker-contract.md index 1e830a360..3d44dd921 100644 --- a/docs/migration/stage-12-worker-contract.md +++ b/docs/migration/stage-12-worker-contract.md @@ -222,8 +222,8 @@ the temporary schema. The 30-day artifact lifecycle is distinct from final removal of the two temporary table schemas. After Stage 12 comparison running has ended, a separately reviewed, autogenerated v2 Alembic revision in Stage 14 removes -those tables. Production `simulations`, `reports`, `report_runs`, and -user-association records must not depend on them. +those tables. No production table, including the user associations, may +depend on them. The tracked follow-up change identifier is `retire-stage-12-comparison-schema`. That Stage 14 change may begin only after diff --git a/docs/migration/stage-12a-user-login.md b/docs/migration/stage-12a-user-login.md index d73b3efd0..03cac230d 100644 --- a/docs/migration/stage-12a-user-login.md +++ b/docs/migration/stage-12a-user-login.md @@ -311,8 +311,8 @@ from the final SQLModel metadata. It has no post-generation corrections. kept: `country` becomes nullable and nothing reads or writes either column. The previous Public API revision still maps them, so dropping them would break it during the deploy window or an application rollback (see - "Rollback"). A later release stopped mapping them: they stay declared on the - tables, unmapped, only until a revision drops them. + "Rollback"). A later release stopped mapping them, and revision + `363f34954a14` then dropped them (see the Alembic migrations skill). The models drop the `user_associations` relationships from v2 simulations and reports. `V2_ALEMBIC_HEAD_REVISION` is `fbb7488ebe8a`. @@ -345,6 +345,13 @@ caller can list a user's saved items by `user_id`, read, rename or delete one by record ID, and create rows for any user. Rolling the application back would therefore hand users' saved work to anyone. +- **Once revision `363f34954a14` has run in an environment, no pre-Stage 12A + revision can serve traffic there.** Those revisions map the simulation-domain + tables and the retired `country` and `label` association columns that it + drops; the oldest revision that can run is the release that stopped mapping + them. A rollback to a pre-Stage 12A revision then first needs the v2 schema + downgraded to `fbb7488ebe8a`, which recreates those tables and columns + empty, and only after that do the steps below apply. - **Before any user-owned record exists, an application rollback is safe without a database downgrade.** Signed-out saving needs no configuration, so treat user-owned records as existing from the moment this revision serves @@ -531,7 +538,8 @@ Configure staging first, then production: (`docs/migration/armor/proposed/20261013-pol-api-lb-stage-12a-rollback-guard.yaml`) ready. From the first user-owned record on, signed in or not, it is a precondition for any application rollback to a pre-Stage 12A revision (see - "Rollback"). + "Rollback"). Once revision `363f34954a14` has run, such a rollback also + needs the v2 schema downgraded to `fbb7488ebe8a` first. User-owned routes must never become less restrictive during a cutover. The migration-contract guard rejects any `/v2/me` or `/v2/user-` contract that @@ -578,9 +586,6 @@ The disposable-Postgres suite proves the following: ## Deferred -- A later autogenerated revision that drops the retired `country` and `label` - columns from `user_simulation_associations` and `user_report_associations`, - once no deployable Public API revision maps them. - A retention policy for unauthenticated users whose browsers never return. Nothing expires them today. - An explicit claim flow for legacy-app users (`POST /v2/me/legacy-claims`). diff --git a/migrations/v2/versions/363f34954a14_drop_simulation_domain_tables_and_.py b/migrations/v2/versions/363f34954a14_drop_simulation_domain_tables_and_.py new file mode 100644 index 000000000..c3db4d547 --- /dev/null +++ b/migrations/v2/versions/363f34954a14_drop_simulation_domain_tables_and_.py @@ -0,0 +1,1361 @@ +"""drop simulation-domain tables and retired association columns + +Revision ID: 363f34954a14 +Revises: fbb7488ebe8a +Create Date: 2026-10-22 17:38:22.373978 +Generation: uv run alembic -c alembic-v2.ini revision --autogenerate +""" + +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +import sqlmodel +from sqlalchemy.dialects import postgresql + +revision: str = "363f34954a14" +down_revision: Union[str, None] = "fbb7488ebe8a" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + # Post-generation correction: Alembic emits removed tables in arbitrary + # order; each table is dropped before the tables its foreign keys + # reference. + op.drop_table("budget_summary") + op.drop_table("constituency_impacts") + op.drop_index(op.f("ix_aggregates_report_run_status"), table_name="aggregates") + op.drop_table("aggregates") + op.drop_table("program_statistics") + op.drop_table("congressional_district_impacts") + op.drop_index( + op.f("ix_change_aggregates_report_run_status"), table_name="change_aggregates" + ) + op.drop_table("change_aggregates") + op.drop_table("intra_decile_impacts") + op.drop_index(op.f("ix_poverty_run_simulation_type"), table_name="poverty") + op.drop_table("poverty") + op.drop_table("decile_impacts") + op.drop_table("inequality") + op.drop_table("local_authority_impacts") + op.drop_index( + op.f("ix_household_jobs_status_created_at"), table_name="household_jobs" + ) + op.drop_table("household_jobs") + op.drop_index(op.f("ix_report_runs_current_output"), table_name="report_runs") + op.drop_table("report_runs") + op.drop_index(op.f("ix_reports_country_type_created_at"), table_name="reports") + op.drop_index(op.f("ix_reports_tax_benefit_model_id"), table_name="reports") + op.drop_table("reports") + op.drop_index(op.f("ix_simulations_status_created_at"), table_name="simulations") + op.drop_index( + op.f("ix_simulations_tax_benefit_model_version_id"), table_name="simulations" + ) + op.drop_table("simulations") + op.drop_column("user_report_associations", "country") + op.drop_column("user_report_associations", "label") + op.drop_column("user_simulation_associations", "country") + op.drop_column("user_simulation_associations", "label") + # Post-generation reversibility correction: Alembic removes the tables + # that use these native PostgreSQL enum types but leaves the types, + # which the generated downgrade could not then create again. + sa.Enum(name="v2_aggregate_type").drop(op.get_bind()) + sa.Enum(name="v2_decile_type").drop(op.get_bind()) + sa.Enum(name="v2_household_job_status").drop(op.get_bind()) + sa.Enum(name="v2_output_status").drop(op.get_bind()) + sa.Enum(name="v2_report_run_status").drop(op.get_bind()) + sa.Enum(name="v2_report_run_trigger").drop(op.get_bind()) + sa.Enum(name="v2_simulation_status").drop(op.get_bind()) + sa.Enum(name="v2_simulation_type").drop(op.get_bind()) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "user_simulation_associations", + sa.Column("label", sa.VARCHAR(length=255), autoincrement=False, nullable=True), + ) + op.add_column( + "user_simulation_associations", + sa.Column("country", sa.VARCHAR(length=16), autoincrement=False, nullable=True), + ) + op.add_column( + "user_report_associations", + sa.Column("label", sa.VARCHAR(length=255), autoincrement=False, nullable=True), + ) + op.add_column( + "user_report_associations", + sa.Column("country", sa.VARCHAR(length=16), autoincrement=False, nullable=True), + ) + # Post-generation correction: each table is created after the tables + # its foreign keys reference, the reverse of the corrected upgrade order. + op.create_table( + "simulations", + sa.Column("id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column( + "updated_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column( + "simulation_type", + postgresql.ENUM("household", "economy", name="v2_simulation_type"), + autoincrement=False, + nullable=False, + ), + sa.Column("dataset_id", sa.UUID(), autoincrement=False, nullable=True), + sa.Column("household_id", sa.UUID(), autoincrement=False, nullable=True), + sa.Column("policy_id", sa.UUID(), autoincrement=False, nullable=True), + sa.Column("dynamic_id", sa.UUID(), autoincrement=False, nullable=True), + sa.Column( + "tax_benefit_model_version_id", + sa.UUID(), + autoincrement=False, + nullable=False, + ), + sa.Column("output_dataset_id", sa.UUID(), autoincrement=False, nullable=True), + sa.Column("region_id", sa.UUID(), autoincrement=False, nullable=True), + sa.Column( + "status", + postgresql.ENUM( + "pending", "running", "succeeded", "failed", name="v2_simulation_status" + ), + autoincrement=False, + nullable=False, + ), + sa.Column("error_message", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column( + "filter_field", sa.VARCHAR(length=128), autoincrement=False, nullable=True + ), + sa.Column( + "filter_value", sa.VARCHAR(length=255), autoincrement=False, nullable=True + ), + sa.Column( + "filter_strategy", sa.VARCHAR(length=64), autoincrement=False, nullable=True + ), + sa.Column("year", sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column( + "started_at", + postgresql.TIMESTAMP(timezone=True), + autoincrement=False, + nullable=True, + ), + sa.Column( + "completed_at", + postgresql.TIMESTAMP(timezone=True), + autoincrement=False, + nullable=True, + ), + sa.Column( + "household_result", + postgresql.JSON(astext_type=sa.Text()), + autoincrement=False, + nullable=True, + ), + sa.CheckConstraint( + "simulation_type = 'household'::v2_simulation_type AND household_id IS NOT NULL AND dataset_id IS NULL OR simulation_type = 'economy'::v2_simulation_type AND dataset_id IS NOT NULL AND household_id IS NULL", + name=op.f("ck_simulations_type_input"), + ), + sa.CheckConstraint( + # Post-generation correction: reflection strips the outer + # parentheses of this check, leaving invalid SQL; they are + # restored as the model declared them. + "(filter_field IS NULL) = (filter_value IS NULL)", + name=op.f("ck_simulations_filter_pair"), + ), + sa.CheckConstraint( + "year IS NULL OR year >= 1900 AND year <= 2200", + name=op.f("ck_simulations_year"), + ), + sa.ForeignKeyConstraint( + ["dataset_id"], + ["datasets.id"], + name=op.f("fk_simulations_dataset_id_datasets"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["dynamic_id"], + ["dynamics.id"], + name=op.f("fk_simulations_dynamic_id_dynamics"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["household_id"], + ["households.id"], + name=op.f("fk_simulations_household_id_households"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["output_dataset_id"], + ["datasets.id"], + name=op.f("fk_simulations_output_dataset_id_datasets"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["policy_id"], + ["policies.id"], + name=op.f("fk_simulations_policy_id_policies"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["region_id"], + ["regions.id"], + name=op.f("fk_simulations_region_id_regions"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["tax_benefit_model_version_id"], + ["tax_benefit_model_versions.id"], + name=op.f("fk_simulations_tax_benefit_model_version_id_tax_benefit_84f8"), + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_simulations")), + ) + op.create_index( + op.f("ix_simulations_tax_benefit_model_version_id"), + "simulations", + ["tax_benefit_model_version_id"], + unique=False, + ) + op.create_index( + op.f("ix_simulations_status_created_at"), + "simulations", + ["status", "created_at"], + unique=False, + ) + op.create_table( + "reports", + sa.Column("id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column( + "updated_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column("label", sa.VARCHAR(length=255), autoincrement=False, nullable=False), + sa.Column("description", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column( + "country", sa.VARCHAR(length=16), autoincrement=False, nullable=False + ), + sa.Column("type", sa.VARCHAR(length=128), autoincrement=False, nullable=True), + sa.Column("user_id", sa.UUID(), autoincrement=False, nullable=True), + sa.Column( + "tax_benefit_model_id", sa.UUID(), autoincrement=False, nullable=False + ), + sa.Column("policy_id", sa.UUID(), autoincrement=False, nullable=True), + sa.Column( + "baseline_simulation_id", sa.UUID(), autoincrement=False, nullable=True + ), + sa.Column( + "reform_simulation_id", sa.UUID(), autoincrement=False, nullable=True + ), + sa.Column("household_id", sa.UUID(), autoincrement=False, nullable=True), + sa.Column("dataset_id", sa.UUID(), autoincrement=False, nullable=True), + sa.Column("region_id", sa.UUID(), autoincrement=False, nullable=True), + sa.Column("year", sa.INTEGER(), autoincrement=False, nullable=True), + sa.Column( + "inputs", + postgresql.JSON(astext_type=sa.Text()), + autoincrement=False, + nullable=False, + ), + sa.CheckConstraint( + "year IS NULL OR year >= 1900 AND year <= 2200", + name=op.f("ck_reports_year"), + ), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f("fk_reports_baseline_simulation_id_simulations"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["dataset_id"], + ["datasets.id"], + name=op.f("fk_reports_dataset_id_datasets"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["household_id"], + ["households.id"], + name=op.f("fk_reports_household_id_households"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["policy_id"], + ["policies.id"], + name=op.f("fk_reports_policy_id_policies"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f("fk_reports_reform_simulation_id_simulations"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["region_id"], + ["regions.id"], + name=op.f("fk_reports_region_id_regions"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["tax_benefit_model_id"], + ["tax_benefit_models.id"], + name=op.f("fk_reports_tax_benefit_model_id_tax_benefit_models"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["user_id"], + ["users.id"], + name=op.f("fk_reports_user_id_users"), + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_reports")), + ) + op.create_index( + op.f("ix_reports_tax_benefit_model_id"), + "reports", + ["tax_benefit_model_id"], + unique=False, + ) + op.create_index( + op.f("ix_reports_country_type_created_at"), + "reports", + ["country", "type", "created_at"], + unique=False, + ) + op.create_table( + "report_runs", + sa.Column("id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column( + "updated_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column("report_id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "country_package_version", + sa.VARCHAR(length=128), + autoincrement=False, + nullable=False, + ), + sa.Column( + "policyengine_version", + sa.VARCHAR(length=128), + autoincrement=False, + nullable=False, + ), + sa.Column( + "status", + postgresql.ENUM( + "pending", "running", "succeeded", "failed", name="v2_report_run_status" + ), + autoincrement=False, + nullable=False, + ), + sa.Column( + "trigger", + postgresql.ENUM( + "initial", "manual", "system", name="v2_report_run_trigger" + ), + autoincrement=False, + nullable=False, + ), + sa.Column("idempotency_key", sa.UUID(), autoincrement=False, nullable=True), + sa.Column( + "started_at", + postgresql.TIMESTAMP(timezone=True), + autoincrement=False, + nullable=True, + ), + sa.Column( + "completed_at", + postgresql.TIMESTAMP(timezone=True), + autoincrement=False, + nullable=True, + ), + sa.Column("error_message", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column("markdown", sa.TEXT(), autoincrement=False, nullable=True), + sa.CheckConstraint( + "(status <> ALL (ARRAY['succeeded'::v2_report_run_status, 'failed'::v2_report_run_status])) OR completed_at IS NOT NULL", + name=op.f("ck_report_runs_terminal_completion"), + ), + sa.ForeignKeyConstraint( + ["report_id"], + ["reports.id"], + name=op.f("fk_report_runs_report_id_reports"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_report_runs")), + sa.UniqueConstraint( + "report_id", + "idempotency_key", + name=op.f("uq_report_runs_report_idempotency_key"), + postgresql_include=[], + postgresql_nulls_not_distinct=False, + ), + ) + op.create_index( + op.f("ix_report_runs_current_output"), + "report_runs", + [ + "report_id", + "status", + "country_package_version", + "policyengine_version", + "completed_at", + "id", + ], + unique=False, + ) + op.create_table( + "household_jobs", + sa.Column("id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column( + "country", sa.VARCHAR(length=16), autoincrement=False, nullable=False + ), + sa.Column( + "request_data", + postgresql.JSON(astext_type=sa.Text()), + autoincrement=False, + nullable=False, + ), + sa.Column("policy_id", sa.UUID(), autoincrement=False, nullable=True), + sa.Column("dynamic_id", sa.UUID(), autoincrement=False, nullable=True), + sa.Column( + "status", + postgresql.ENUM( + "pending", + "running", + "succeeded", + "failed", + name="v2_household_job_status", + ), + autoincrement=False, + nullable=False, + ), + sa.Column("error_message", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column( + "result", + postgresql.JSON(astext_type=sa.Text()), + autoincrement=False, + nullable=True, + ), + sa.Column( + "started_at", + postgresql.TIMESTAMP(timezone=True), + autoincrement=False, + nullable=True, + ), + sa.Column( + "completed_at", + postgresql.TIMESTAMP(timezone=True), + autoincrement=False, + nullable=True, + ), + sa.ForeignKeyConstraint( + ["dynamic_id"], + ["dynamics.id"], + name=op.f("fk_household_jobs_dynamic_id_dynamics"), + ondelete="SET NULL", + ), + sa.ForeignKeyConstraint( + ["policy_id"], + ["policies.id"], + name=op.f("fk_household_jobs_policy_id_policies"), + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_household_jobs")), + ) + op.create_index( + op.f("ix_household_jobs_status_created_at"), + "household_jobs", + ["status", "created_at"], + unique=False, + ) + op.create_table( + "local_authority_impacts", + sa.Column("id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column("report_run_id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "baseline_simulation_id", sa.UUID(), autoincrement=False, nullable=False + ), + sa.Column( + "reform_simulation_id", sa.UUID(), autoincrement=False, nullable=False + ), + sa.Column( + "average_household_income_change", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=False, + ), + sa.Column( + "relative_household_income_change", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=False, + ), + sa.Column( + "population", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=False, + ), + sa.Column( + "local_authority_code", + sa.VARCHAR(length=64), + autoincrement=False, + nullable=False, + ), + sa.Column( + "local_authority_name", + sa.VARCHAR(length=255), + autoincrement=False, + nullable=False, + ), + sa.Column("x", sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column("y", sa.INTEGER(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f("fk_local_authority_impacts_baseline_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f("fk_local_authority_impacts_reform_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_local_authority_impacts_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_local_authority_impacts")), + sa.UniqueConstraint( + "report_run_id", + "local_authority_code", + name=op.f("uq_local_authority_impacts_run_code"), + postgresql_include=[], + postgresql_nulls_not_distinct=False, + ), + ) + op.create_table( + "inequality", + sa.Column("id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column("report_run_id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column("simulation_id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "income_variable", + sa.VARCHAR(length=512), + autoincrement=False, + nullable=False, + ), + sa.Column( + "entity", sa.VARCHAR(length=128), autoincrement=False, nullable=False + ), + sa.Column( + "gini", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "top_10_share", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "top_1_share", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "bottom_50_share", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_inequality_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["simulation_id"], + ["simulations.id"], + name=op.f("fk_inequality_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_inequality")), + sa.UniqueConstraint( + "report_run_id", + "simulation_id", + "income_variable", + "entity", + name=op.f("uq_inequality_run_simulation_measure"), + postgresql_include=[], + postgresql_nulls_not_distinct=False, + ), + ) + op.create_table( + "decile_impacts", + sa.Column("id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column("report_run_id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "baseline_simulation_id", sa.UUID(), autoincrement=False, nullable=False + ), + sa.Column( + "reform_simulation_id", sa.UUID(), autoincrement=False, nullable=False + ), + sa.Column( + "income_variable", + sa.VARCHAR(length=512), + autoincrement=False, + nullable=False, + ), + sa.Column( + "entity", sa.VARCHAR(length=128), autoincrement=False, nullable=False + ), + sa.Column("decile", sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column("quantiles", sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column( + "baseline_mean", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "reform_mean", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "absolute_change", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "relative_change", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "count_better_off", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "count_worse_off", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "count_no_change", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.CheckConstraint( + "decile >= 1 AND decile <= quantiles", name=op.f("ck_decile_impacts_decile") + ), + sa.CheckConstraint("quantiles > 0", name=op.f("ck_decile_impacts_quantiles")), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f("fk_decile_impacts_baseline_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f("fk_decile_impacts_reform_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_decile_impacts_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_decile_impacts")), + sa.UniqueConstraint( + "report_run_id", + "income_variable", + "entity", + "decile", + "quantiles", + name=op.f("uq_decile_impacts_run_measure"), + postgresql_include=[], + postgresql_nulls_not_distinct=False, + ), + ) + op.create_table( + "poverty", + sa.Column("id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column("report_run_id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column("simulation_id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "poverty_type", sa.VARCHAR(length=128), autoincrement=False, nullable=False + ), + sa.Column( + "entity", sa.VARCHAR(length=128), autoincrement=False, nullable=False + ), + sa.Column( + "filter_variable", + sa.VARCHAR(length=512), + autoincrement=False, + nullable=True, + ), + sa.Column( + "headcount", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "total_population", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "rate", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_poverty_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["simulation_id"], + ["simulations.id"], + name=op.f("fk_poverty_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_poverty")), + ) + op.create_index( + op.f("ix_poverty_run_simulation_type"), + "poverty", + ["report_run_id", "simulation_id", "poverty_type"], + unique=False, + ) + op.create_table( + "intra_decile_impacts", + sa.Column("id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column("report_run_id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "baseline_simulation_id", sa.UUID(), autoincrement=False, nullable=False + ), + sa.Column( + "reform_simulation_id", sa.UUID(), autoincrement=False, nullable=False + ), + sa.Column( + "decile_type", + postgresql.ENUM("income", "wealth", name="v2_decile_type"), + autoincrement=False, + nullable=False, + ), + sa.Column("decile", sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column( + "lose_more_than_5pct", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "lose_less_than_5pct", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "no_change", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "gain_less_than_5pct", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "gain_more_than_5pct", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.CheckConstraint( + "decile >= 0 AND decile <= 10", name=op.f("ck_intra_decile_impacts_decile") + ), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f("fk_intra_decile_impacts_baseline_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f("fk_intra_decile_impacts_reform_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_intra_decile_impacts_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_intra_decile_impacts")), + sa.UniqueConstraint( + "report_run_id", + "decile_type", + "decile", + name=op.f("uq_intra_decile_impacts_run_type_decile"), + postgresql_include=[], + postgresql_nulls_not_distinct=False, + ), + ) + op.create_table( + "change_aggregates", + sa.Column("id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column("report_run_id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "baseline_simulation_id", sa.UUID(), autoincrement=False, nullable=False + ), + sa.Column( + "reform_simulation_id", sa.UUID(), autoincrement=False, nullable=False + ), + sa.Column( + "variable", sa.VARCHAR(length=512), autoincrement=False, nullable=False + ), + sa.Column( + "aggregate_type", + postgresql.ENUM("sum", "mean", "count", name="v2_aggregate_type"), + autoincrement=False, + nullable=False, + ), + sa.Column("entity", sa.VARCHAR(length=128), autoincrement=False, nullable=True), + sa.Column( + "filter_config", + postgresql.JSON(astext_type=sa.Text()), + autoincrement=False, + nullable=False, + ), + sa.Column( + "change_geq", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "change_leq", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "status", + postgresql.ENUM( + "pending", "running", "succeeded", "failed", name="v2_output_status" + ), + autoincrement=False, + nullable=False, + ), + sa.Column("error_message", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column( + "result", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f("fk_change_aggregates_baseline_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f("fk_change_aggregates_reform_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_change_aggregates_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_change_aggregates")), + ) + op.create_index( + op.f("ix_change_aggregates_report_run_status"), + "change_aggregates", + ["report_run_id", "status"], + unique=False, + ) + op.create_table( + "congressional_district_impacts", + sa.Column("id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column("report_run_id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "baseline_simulation_id", sa.UUID(), autoincrement=False, nullable=False + ), + sa.Column( + "reform_simulation_id", sa.UUID(), autoincrement=False, nullable=False + ), + sa.Column( + "average_household_income_change", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=False, + ), + sa.Column( + "relative_household_income_change", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=False, + ), + sa.Column( + "population", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=False, + ), + sa.Column("district_geoid", sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column("state_fips", sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column("district_number", sa.INTEGER(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f("fk_congressional_district_impacts_baseline_simulation_i_d68d"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f("fk_congressional_district_impacts_reform_simulation_id__ca94"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_congressional_district_impacts_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_congressional_district_impacts")), + sa.UniqueConstraint( + "report_run_id", + "district_geoid", + name=op.f("uq_congressional_district_impacts_run_geoid"), + postgresql_include=[], + postgresql_nulls_not_distinct=False, + ), + ) + op.create_table( + "program_statistics", + sa.Column("id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column("report_run_id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "baseline_simulation_id", sa.UUID(), autoincrement=False, nullable=False + ), + sa.Column( + "reform_simulation_id", sa.UUID(), autoincrement=False, nullable=False + ), + sa.Column( + "program_name", sa.VARCHAR(length=512), autoincrement=False, nullable=False + ), + sa.Column( + "entity", sa.VARCHAR(length=128), autoincrement=False, nullable=False + ), + sa.Column("is_tax", sa.BOOLEAN(), autoincrement=False, nullable=False), + sa.Column( + "baseline_total", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "reform_total", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "change", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "baseline_count", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "reform_count", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "winners", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "losers", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f("fk_program_statistics_baseline_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f("fk_program_statistics_reform_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_program_statistics_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_program_statistics")), + sa.UniqueConstraint( + "report_run_id", + "program_name", + "entity", + name=op.f("uq_program_statistics_run_program_entity"), + postgresql_include=[], + postgresql_nulls_not_distinct=False, + ), + ) + op.create_table( + "aggregates", + sa.Column("id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column("report_run_id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column("simulation_id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "variable", sa.VARCHAR(length=512), autoincrement=False, nullable=False + ), + sa.Column( + "aggregate_type", + postgresql.ENUM("sum", "mean", "count", name="v2_aggregate_type"), + autoincrement=False, + nullable=False, + ), + sa.Column("entity", sa.VARCHAR(length=128), autoincrement=False, nullable=True), + sa.Column( + "filter_config", + postgresql.JSON(astext_type=sa.Text()), + autoincrement=False, + nullable=False, + ), + sa.Column( + "status", + postgresql.ENUM( + "pending", "running", "succeeded", "failed", name="v2_output_status" + ), + autoincrement=False, + nullable=False, + ), + sa.Column("error_message", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column( + "result", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_aggregates_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["simulation_id"], + ["simulations.id"], + name=op.f("fk_aggregates_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_aggregates")), + ) + op.create_index( + op.f("ix_aggregates_report_run_status"), + "aggregates", + ["report_run_id", "status"], + unique=False, + ) + op.create_table( + "constituency_impacts", + sa.Column("id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column("report_run_id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "baseline_simulation_id", sa.UUID(), autoincrement=False, nullable=False + ), + sa.Column( + "reform_simulation_id", sa.UUID(), autoincrement=False, nullable=False + ), + sa.Column( + "average_household_income_change", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=False, + ), + sa.Column( + "relative_household_income_change", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=False, + ), + sa.Column( + "population", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=False, + ), + sa.Column( + "constituency_code", + sa.VARCHAR(length=64), + autoincrement=False, + nullable=False, + ), + sa.Column( + "constituency_name", + sa.VARCHAR(length=255), + autoincrement=False, + nullable=False, + ), + sa.Column("x", sa.INTEGER(), autoincrement=False, nullable=False), + sa.Column("y", sa.INTEGER(), autoincrement=False, nullable=False), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f("fk_constituency_impacts_baseline_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f("fk_constituency_impacts_reform_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_constituency_impacts_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_constituency_impacts")), + sa.UniqueConstraint( + "report_run_id", + "constituency_code", + name=op.f("uq_constituency_impacts_run_code"), + postgresql_include=[], + postgresql_nulls_not_distinct=False, + ), + ) + op.create_table( + "budget_summary", + sa.Column("id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=False, + ), + sa.Column("report_run_id", sa.UUID(), autoincrement=False, nullable=False), + sa.Column( + "baseline_simulation_id", sa.UUID(), autoincrement=False, nullable=False + ), + sa.Column( + "reform_simulation_id", sa.UUID(), autoincrement=False, nullable=False + ), + sa.Column( + "variable_name", sa.VARCHAR(length=512), autoincrement=False, nullable=False + ), + sa.Column( + "entity", sa.VARCHAR(length=128), autoincrement=False, nullable=False + ), + sa.Column( + "baseline_total", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "reform_total", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.Column( + "change", + sa.DOUBLE_PRECISION(precision=53), + autoincrement=False, + nullable=True, + ), + sa.ForeignKeyConstraint( + ["baseline_simulation_id"], + ["simulations.id"], + name=op.f("fk_budget_summary_baseline_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["reform_simulation_id"], + ["simulations.id"], + name=op.f("fk_budget_summary_reform_simulation_id_simulations"), + ondelete="RESTRICT", + ), + sa.ForeignKeyConstraint( + ["report_run_id"], + ["report_runs.id"], + name=op.f("fk_budget_summary_report_run_id_report_runs"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_budget_summary")), + sa.UniqueConstraint( + "report_run_id", + "variable_name", + "entity", + name=op.f("uq_budget_summary_run_variable_entity"), + postgresql_include=[], + postgresql_nulls_not_distinct=False, + ), + ) + # ### end Alembic commands ### diff --git a/policyengine_api/data/v2/household_migration_qualification.py b/policyengine_api/data/v2/household_migration_qualification.py index 646f64d84..3b2a2977f 100644 --- a/policyengine_api/data/v2/household_migration_qualification.py +++ b/policyengine_api/data/v2/household_migration_qualification.py @@ -27,8 +27,8 @@ HOUSEHOLD_MIGRATION_REVISION = "724b1b11a33e" V2_ALEMBIC_CONFIG = "alembic-v2.ini" -# A target before the household revision has these tables, but no model maps -# them any more and a later revision drops them, so the counts name them +# A target before the household revision has these tables, but revision +# 363f34954a14 dropped them and their models, so the counts name them # directly. The existing-table check skips them on a target without them. SIMULATIONS = table("simulations", column("household_id")) REPORTS = table("reports", column("household_id")) diff --git a/policyengine_api/data/v2/metadata_validation.py b/policyengine_api/data/v2/metadata_validation.py index 7445269f1..4a7bd35b4 100644 --- a/policyengine_api/data/v2/metadata_validation.py +++ b/policyengine_api/data/v2/metadata_validation.py @@ -2,9 +2,9 @@ from collections.abc import Iterable -# These names belong only to the existing v1 database domain. Names that are -# intentionally shared between v1 and v2, such as `simulations` and -# `user_policies`, are excluded from this targeted rejection set. +# These names belong only to the existing v1 database domain. Names that v1 +# and v2 both use, such as `user_policies`, are excluded from this targeted +# rejection set. V1_ONLY_TABLES = frozenset( { "analysis", @@ -35,18 +35,44 @@ ) +# The simulation service owns these tables in its own ``simulation`` schema. +# A v2 revision dropped the Public API's copies, and its metadata must not +# declare them again. +SIMULATION_SERVICE_TABLES = frozenset( + { + "aggregates", + "budget_summary", + "change_aggregates", + "congressional_district_impacts", + "constituency_impacts", + "decile_impacts", + "household_jobs", + "inequality", + "intra_decile_impacts", + "local_authority_impacts", + "poverty", + "program_statistics", + "report_runs", + "reports", + "simulations", + } +) + + class V2MetadataValidationError(RuntimeError): """Raised when v2 metadata contains a specifically rejected table.""" def validate_v2_metadata_table_names(table_names: Iterable[str]) -> None: - """Reject known v1-only and explicitly prohibited v2 table names.""" + """Reject known v1-only, simulation-service and prohibited v2 table names.""" actual = frozenset(table_names) prohibited = sorted(actual & PROHIBITED_V2_TABLES) v1_only = sorted(actual & V1_ONLY_TABLES) - if prohibited or v1_only: + simulation_service = sorted(actual & SIMULATION_SERVICE_TABLES) + if prohibited or v1_only or simulation_service: msg = ( "API v2-alpha metadata contains rejected tables: " - f"prohibited={prohibited}, v1_only={v1_only}" + f"prohibited={prohibited}, v1_only={v1_only}, " + f"simulation_service={simulation_service}" ) raise V2MetadataValidationError(msg) diff --git a/policyengine_api/data/v2/migration_target.py b/policyengine_api/data/v2/migration_target.py index fd4c05fdf..399a055f2 100644 --- a/policyengine_api/data/v2/migration_target.py +++ b/policyengine_api/data/v2/migration_target.py @@ -24,7 +24,7 @@ from collections.abc import Mapping V2_ALEMBIC_DISPOSABLE_TEST = "V2_ALEMBIC_DISPOSABLE_TEST" -V2_ALEMBIC_HEAD_REVISION = "fbb7488ebe8a" +V2_ALEMBIC_HEAD_REVISION = "363f34954a14" DISPOSABLE_DATABASE_NAME = "policyengine_v2_alembic_test" DISPOSABLE_HOSTS = frozenset({"localhost", "127.0.0.1", "::1", "postgres"}) MIGRATION_ROLE = "policyengine_v2_migrator" diff --git a/policyengine_api/data/v2/models/__init__.py b/policyengine_api/data/v2/models/__init__.py index 255af0d6f..a3b5aa022 100644 --- a/policyengine_api/data/v2/models/__init__.py +++ b/policyengine_api/data/v2/models/__init__.py @@ -17,11 +17,6 @@ } # Import in dependency order so every reviewed table registers exactly once. -# retired_tables declares, without models, the tables and columns a later -# revision drops; see its docstring. -from policyengine_api.data.v2.models import ( # noqa: E402 - retired_tables as retired_tables, -) from policyengine_api.data.v2.models.associations import ( # noqa: E402 UserHouseholdAssociation, UserPolicy, diff --git a/policyengine_api/data/v2/models/associations.py b/policyengine_api/data/v2/models/associations.py index acc95d433..27a183602 100644 --- a/policyengine_api/data/v2/models/associations.py +++ b/policyengine_api/data/v2/models/associations.py @@ -3,11 +3,6 @@ Every association names its target through a nullable v2 reference, a nullable country-scoped v1 (legacy) reference, or both; a database check requires at least one. - -The simulation and report association tables still have the pre-Stage 12A -``country`` and ``label`` columns, which a later revision drops. These models -don't declare them, so nothing reads or writes them; ``retired_tables`` -declares them on the tables only so the metadata matches the schema. """ from datetime import datetime diff --git a/policyengine_api/data/v2/models/base.py b/policyengine_api/data/v2/models/base.py index 27bb5317b..980972e5d 100644 --- a/policyengine_api/data/v2/models/base.py +++ b/policyengine_api/data/v2/models/base.py @@ -34,11 +34,6 @@ "Overlapping composite region-default joins need SQLAlchemy " "relationship hints exposed by SQLModel Relationship." ), - "retired_table_declarations": ( - "Tables and columns a later revision drops stay declared as plain " - "SQLAlchemy tables and columns that no model maps, so the metadata " - "matches the schema while nothing reads or writes them." - ), } ) diff --git a/policyengine_api/data/v2/models/retired_tables.py b/policyengine_api/data/v2/models/retired_tables.py deleted file mode 100644 index 9959fcdd1..000000000 --- a/policyengine_api/data/v2/models/retired_tables.py +++ /dev/null @@ -1,519 +0,0 @@ -"""Schema-only declarations of the v2 tables and columns a later revision drops. - -The simulation service owns household jobs, simulations, reports, report runs -and their outputs in its own ``simulation`` schema. The Public API's copies of -those 15 tables, and the retired ``country`` and ``label`` columns of the -simulation and report associations, stay in the database until a later v2 -revision drops them. That revision runs while this release serves traffic, so -this release must never read or write them. - -They stay declared here only so the v2 metadata keeps matching the schema: -``alembic check`` and the migration environment's head-schema validation -compare the two. They are plain SQLAlchemy tables and columns that no model -maps, so no ORM query, insert, update, relationship or cascade reaches them. -Nothing imports this module except the models package, which registers it. -Deleting it is what makes autogeneration produce the revision that drops them. -""" - -from sqlalchemy import ( - JSON, - Boolean, - CheckConstraint, - Column, - DateTime, - Enum, - Float, - ForeignKey, - Index, - Integer, - String, - Table, - Text, - UniqueConstraint, - Uuid, - func, -) -from sqlmodel import SQLModel - -from policyengine_api.data.v2.models.associations import ( - UserReportAssociation, - UserSimulationAssociation, -) - -METADATA = SQLModel.metadata - -# Each lifecycle enum has the same four values. -LIFECYCLE_STATES = ("pending", "running", "succeeded", "failed") -REPORT_RUN_TRIGGERS = ("initial", "manual", "system") -AGGREGATE_TYPES = ("sum", "mean", "count") -DECILE_TYPES = ("income", "wealth") - -# The retired association columns, by table, with their string lengths. -RETIRED_ASSOCIATION_COLUMNS = { - UserSimulationAssociation.__tablename__: {"country": 16, "label": 255}, - UserReportAssociation.__tablename__: {"country": 16, "label": 255}, -} - - -def _enum(values: tuple[str, ...], name: str) -> Enum: - return Enum(*values, name=name) - - -def _identified_columns() -> list[Column]: - return [ - Column("id", Uuid(), primary_key=True), - Column( - "created_at", - DateTime(timezone=True), - server_default=func.now(), - nullable=False, - ), - ] - - -def _timestamped_columns() -> list[Column]: - return [ - *_identified_columns(), - Column( - "updated_at", - DateTime(timezone=True), - server_default=func.now(), - nullable=False, - ), - ] - - -def _reference( - name: str, - target: str, - ondelete: str, - *, - nullable: bool, - index: bool = False, -) -> Column: - return Column( - name, - Uuid(), - ForeignKey(target, ondelete=ondelete), - nullable=nullable, - index=index, - ) - - -def _report_run_reference() -> Column: - return _reference("report_run_id", "report_runs.id", "CASCADE", nullable=False) - - -def _simulation_reference(name: str) -> Column: - return _reference(name, "simulations.id", "RESTRICT", nullable=False) - - -def _comparison_references() -> list[Column]: - return [ - _report_run_reference(), - _simulation_reference("baseline_simulation_id"), - _simulation_reference("reform_simulation_id"), - ] - - -def _geographic_impact_columns() -> list[Column]: - return [ - *_identified_columns(), - *_comparison_references(), - Column("average_household_income_change", Float(), nullable=False), - Column("relative_household_income_change", Float(), nullable=False), - Column("population", Float(), nullable=False), - ] - - -HOUSEHOLD_JOBS = Table( - "household_jobs", - METADATA, - *_identified_columns(), - Column("country", String(16), nullable=False), - Column("request_data", JSON(), nullable=False), - _reference("policy_id", "policies.id", "SET NULL", nullable=True), - _reference("dynamic_id", "dynamics.id", "SET NULL", nullable=True), - Column( - "status", - _enum(LIFECYCLE_STATES, "v2_household_job_status"), - nullable=False, - ), - Column("error_message", String(), nullable=True), - Column("result", JSON(), nullable=True), - Column("started_at", DateTime(timezone=True), nullable=True), - Column("completed_at", DateTime(timezone=True), nullable=True), - Index("ix_household_jobs_status_created_at", "status", "created_at"), -) - -SIMULATIONS = Table( - "simulations", - METADATA, - *_timestamped_columns(), - Column( - "simulation_type", - _enum(("household", "economy"), "v2_simulation_type"), - nullable=False, - ), - _reference("dataset_id", "datasets.id", "RESTRICT", nullable=True), - _reference("household_id", "households.id", "RESTRICT", nullable=True), - _reference("policy_id", "policies.id", "SET NULL", nullable=True), - _reference("dynamic_id", "dynamics.id", "SET NULL", nullable=True), - _reference( - "tax_benefit_model_version_id", - "tax_benefit_model_versions.id", - "RESTRICT", - nullable=False, - index=True, - ), - _reference("output_dataset_id", "datasets.id", "SET NULL", nullable=True), - _reference("region_id", "regions.id", "SET NULL", nullable=True), - Column("status", _enum(LIFECYCLE_STATES, "v2_simulation_status"), nullable=False), - Column("error_message", String(), nullable=True), - Column("filter_field", String(128), nullable=True), - Column("filter_value", String(255), nullable=True), - Column("filter_strategy", String(64), nullable=True), - Column("year", Integer(), nullable=True), - Column("started_at", DateTime(timezone=True), nullable=True), - Column("completed_at", DateTime(timezone=True), nullable=True), - Column("household_result", JSON(), nullable=True), - CheckConstraint( - "(simulation_type = 'household' AND household_id IS NOT NULL " - "AND dataset_id IS NULL) OR " - "(simulation_type = 'economy' AND dataset_id IS NOT NULL " - "AND household_id IS NULL)", - name="ck_simulations_type_input", - ), - CheckConstraint( - "(filter_field IS NULL) = (filter_value IS NULL)", - name="ck_simulations_filter_pair", - ), - CheckConstraint( - "year IS NULL OR year BETWEEN 1900 AND 2200", - name="ck_simulations_year", - ), - Index("ix_simulations_status_created_at", "status", "created_at"), -) - -REPORTS = Table( - "reports", - METADATA, - *_timestamped_columns(), - Column("label", String(255), nullable=False), - Column("description", String(), nullable=True), - Column("country", String(16), nullable=False), - Column("type", String(128), nullable=True), - _reference("user_id", "users.id", "SET NULL", nullable=True), - _reference( - "tax_benefit_model_id", - "tax_benefit_models.id", - "RESTRICT", - nullable=False, - index=True, - ), - _reference("policy_id", "policies.id", "SET NULL", nullable=True), - _reference("baseline_simulation_id", "simulations.id", "SET NULL", nullable=True), - _reference("reform_simulation_id", "simulations.id", "SET NULL", nullable=True), - _reference("household_id", "households.id", "RESTRICT", nullable=True), - _reference("dataset_id", "datasets.id", "SET NULL", nullable=True), - _reference("region_id", "regions.id", "SET NULL", nullable=True), - Column("year", Integer(), nullable=True), - Column("inputs", JSON(), nullable=False), - CheckConstraint( - "year IS NULL OR year BETWEEN 1900 AND 2200", - name="ck_reports_year", - ), - Index("ix_reports_country_type_created_at", "country", "type", "created_at"), -) - -REPORT_RUNS = Table( - "report_runs", - METADATA, - *_timestamped_columns(), - _reference("report_id", "reports.id", "CASCADE", nullable=False), - Column("country_package_version", String(128), nullable=False), - Column("policyengine_version", String(128), nullable=False), - Column("status", _enum(LIFECYCLE_STATES, "v2_report_run_status"), nullable=False), - Column( - "trigger", - _enum(REPORT_RUN_TRIGGERS, "v2_report_run_trigger"), - nullable=False, - ), - Column("idempotency_key", Uuid(), nullable=True), - Column("started_at", DateTime(timezone=True), nullable=True), - Column("completed_at", DateTime(timezone=True), nullable=True), - Column("error_message", String(), nullable=True), - Column("markdown", Text(), nullable=True), - UniqueConstraint( - "report_id", - "idempotency_key", - name="uq_report_runs_report_idempotency_key", - ), - CheckConstraint( - "status NOT IN ('succeeded', 'failed') OR completed_at IS NOT NULL", - name="ck_report_runs_terminal_completion", - ), - Index( - "ix_report_runs_current_output", - "report_id", - "status", - "country_package_version", - "policyengine_version", - "completed_at", - "id", - ), -) - -AGGREGATES = Table( - "aggregates", - METADATA, - *_identified_columns(), - _report_run_reference(), - _simulation_reference("simulation_id"), - Column("variable", String(512), nullable=False), - Column( - "aggregate_type", - _enum(AGGREGATE_TYPES, "v2_aggregate_type"), - nullable=False, - ), - Column("entity", String(128), nullable=True), - Column("filter_config", JSON(), nullable=False), - Column("status", _enum(LIFECYCLE_STATES, "v2_output_status"), nullable=False), - Column("error_message", String(), nullable=True), - Column("result", Float(), nullable=True), - Index("ix_aggregates_report_run_status", "report_run_id", "status"), -) - -CHANGE_AGGREGATES = Table( - "change_aggregates", - METADATA, - *_identified_columns(), - *_comparison_references(), - Column("variable", String(512), nullable=False), - Column( - "aggregate_type", - _enum(AGGREGATE_TYPES, "v2_aggregate_type"), - nullable=False, - ), - Column("entity", String(128), nullable=True), - Column("filter_config", JSON(), nullable=False), - Column("change_geq", Float(), nullable=True), - Column("change_leq", Float(), nullable=True), - Column("status", _enum(LIFECYCLE_STATES, "v2_output_status"), nullable=False), - Column("error_message", String(), nullable=True), - Column("result", Float(), nullable=True), - Index("ix_change_aggregates_report_run_status", "report_run_id", "status"), -) - -BUDGET_SUMMARY = Table( - "budget_summary", - METADATA, - *_identified_columns(), - *_comparison_references(), - Column("variable_name", String(512), nullable=False), - Column("entity", String(128), nullable=False), - Column("baseline_total", Float(), nullable=True), - Column("reform_total", Float(), nullable=True), - Column("change", Float(), nullable=True), - UniqueConstraint( - "report_run_id", - "variable_name", - "entity", - name="uq_budget_summary_run_variable_entity", - ), -) - -DECILE_IMPACTS = Table( - "decile_impacts", - METADATA, - *_identified_columns(), - *_comparison_references(), - Column("income_variable", String(512), nullable=False), - Column("entity", String(128), nullable=False), - Column("decile", Integer(), nullable=False), - Column("quantiles", Integer(), nullable=False), - Column("baseline_mean", Float(), nullable=True), - Column("reform_mean", Float(), nullable=True), - Column("absolute_change", Float(), nullable=True), - Column("relative_change", Float(), nullable=True), - Column("count_better_off", Float(), nullable=True), - Column("count_worse_off", Float(), nullable=True), - Column("count_no_change", Float(), nullable=True), - UniqueConstraint( - "report_run_id", - "income_variable", - "entity", - "decile", - "quantiles", - name="uq_decile_impacts_run_measure", - ), - CheckConstraint("quantiles > 0", name="ck_decile_impacts_quantiles"), - CheckConstraint("decile BETWEEN 1 AND quantiles", name="ck_decile_impacts_decile"), -) - -INTRA_DECILE_IMPACTS = Table( - "intra_decile_impacts", - METADATA, - *_identified_columns(), - *_comparison_references(), - Column("decile_type", _enum(DECILE_TYPES, "v2_decile_type"), nullable=False), - Column("decile", Integer(), nullable=False), - Column("lose_more_than_5pct", Float(), nullable=True), - Column("lose_less_than_5pct", Float(), nullable=True), - Column("no_change", Float(), nullable=True), - Column("gain_less_than_5pct", Float(), nullable=True), - Column("gain_more_than_5pct", Float(), nullable=True), - UniqueConstraint( - "report_run_id", - "decile_type", - "decile", - name="uq_intra_decile_impacts_run_type_decile", - ), - CheckConstraint( - "decile BETWEEN 0 AND 10", - name="ck_intra_decile_impacts_decile", - ), -) - -INEQUALITY = Table( - "inequality", - METADATA, - *_identified_columns(), - _report_run_reference(), - _simulation_reference("simulation_id"), - Column("income_variable", String(512), nullable=False), - Column("entity", String(128), nullable=False), - Column("gini", Float(), nullable=True), - Column("top_10_share", Float(), nullable=True), - Column("top_1_share", Float(), nullable=True), - Column("bottom_50_share", Float(), nullable=True), - UniqueConstraint( - "report_run_id", - "simulation_id", - "income_variable", - "entity", - name="uq_inequality_run_simulation_measure", - ), -) - -POVERTY = Table( - "poverty", - METADATA, - *_identified_columns(), - _report_run_reference(), - _simulation_reference("simulation_id"), - Column("poverty_type", String(128), nullable=False), - Column("entity", String(128), nullable=False), - Column("filter_variable", String(512), nullable=True), - Column("headcount", Float(), nullable=True), - Column("total_population", Float(), nullable=True), - Column("rate", Float(), nullable=True), - Index( - "ix_poverty_run_simulation_type", - "report_run_id", - "simulation_id", - "poverty_type", - ), -) - -PROGRAM_STATISTICS = Table( - "program_statistics", - METADATA, - *_identified_columns(), - *_comparison_references(), - Column("program_name", String(512), nullable=False), - Column("entity", String(128), nullable=False), - Column("is_tax", Boolean(), nullable=False), - Column("baseline_total", Float(), nullable=True), - Column("reform_total", Float(), nullable=True), - Column("change", Float(), nullable=True), - Column("baseline_count", Float(), nullable=True), - Column("reform_count", Float(), nullable=True), - Column("winners", Float(), nullable=True), - Column("losers", Float(), nullable=True), - UniqueConstraint( - "report_run_id", - "program_name", - "entity", - name="uq_program_statistics_run_program_entity", - ), -) - -CONGRESSIONAL_DISTRICT_IMPACTS = Table( - "congressional_district_impacts", - METADATA, - *_geographic_impact_columns(), - Column("district_geoid", Integer(), nullable=False), - Column("state_fips", Integer(), nullable=False), - Column("district_number", Integer(), nullable=False), - UniqueConstraint( - "report_run_id", - "district_geoid", - name="uq_congressional_district_impacts_run_geoid", - ), -) - -CONSTITUENCY_IMPACTS = Table( - "constituency_impacts", - METADATA, - *_geographic_impact_columns(), - Column("constituency_code", String(64), nullable=False), - Column("constituency_name", String(255), nullable=False), - Column("x", Integer(), nullable=False), - Column("y", Integer(), nullable=False), - UniqueConstraint( - "report_run_id", - "constituency_code", - name="uq_constituency_impacts_run_code", - ), -) - -LOCAL_AUTHORITY_IMPACTS = Table( - "local_authority_impacts", - METADATA, - *_geographic_impact_columns(), - Column("local_authority_code", String(64), nullable=False), - Column("local_authority_name", String(255), nullable=False), - Column("x", Integer(), nullable=False), - Column("y", Integer(), nullable=False), - UniqueConstraint( - "report_run_id", - "local_authority_code", - name="uq_local_authority_impacts_run_code", - ), -) - -RETIRED_TABLES = ( - HOUSEHOLD_JOBS, - SIMULATIONS, - REPORTS, - REPORT_RUNS, - AGGREGATES, - CHANGE_AGGREGATES, - BUDGET_SUMMARY, - DECILE_IMPACTS, - INTRA_DECILE_IMPACTS, - INEQUALITY, - POVERTY, - PROGRAM_STATISTICS, - CONGRESSIONAL_DISTRICT_IMPACTS, - CONSTITUENCY_IMPACTS, - LOCAL_AUTHORITY_IMPACTS, -) - - -def _declare_retired_association_columns() -> None: - """Add the retired columns to the association tables, but not their models. - - The association models no longer declare these columns, so their mappers - never select, insert or update them: a column appended to a table after - its mapper exists stays unmapped. - """ - for table_name, columns in RETIRED_ASSOCIATION_COLUMNS.items(): - for column_name, length in columns.items(): - METADATA.tables[table_name].append_column( - Column(column_name, String(length), nullable=True) - ) - - -_declare_retired_association_columns() diff --git a/policyengine_api/data/v2/privileges.py b/policyengine_api/data/v2/privileges.py index a43e0c00d..5b869e57e 100644 --- a/policyengine_api/data/v2/privileges.py +++ b/policyengine_api/data/v2/privileges.py @@ -93,23 +93,6 @@ def _access( TABLE_ACCESS: Final[dict[str, TableAccess]] = { # Alembic's revision record; the catalog writer checks the schema revision. "alembic_version": _access(catalog_writer=READ), - # Simulation-domain tables, which the simulation service owns in its own - # schema. No managed role uses them, and a later revision drops them. - "aggregates": _access(), - "budget_summary": _access(), - "change_aggregates": _access(), - "congressional_district_impacts": _access(), - "constituency_impacts": _access(), - "decile_impacts": _access(), - "household_jobs": _access(), - "inequality": _access(), - "intra_decile_impacts": _access(), - "local_authority_impacts": _access(), - "poverty": _access(), - "program_statistics": _access(), - "report_runs": _access(), - "reports": _access(), - "simulations": _access(), # Metadata catalog: the catalog writer publishes it, the Public API reads it. "dataset_versions": _access(catalog_writer=READ), "datasets": _access(runtime=READ, catalog_writer=READ_INSERT_UPDATE), diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index 814971236..5b43419b0 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -12,10 +12,12 @@ from alembic.autogenerate import compare_metadata from alembic.config import Config from alembic.migration import MigrationContext +from alembic.script import ScriptDirectory from sqlalchemy import Engine, create_engine, inspect, text from sqlalchemy.dialects import postgresql from policyengine_api.constants import REPO +from policyengine_api.data.v2.metadata_validation import SIMULATION_SERVICE_TABLES from policyengine_api.data.v2.migration_target import ( V2_ALEMBIC_DISPOSABLE_TEST, V2_ALEMBIC_HEAD_REVISION, @@ -30,7 +32,26 @@ STAGE_12_PREVIOUS_REVISION = "724b1b11a33e" STAGE_12_COMPARISON_PREVIOUS_REVISION = "439303be14fe" STAGE_12A_PREVIOUS_REVISION = "91af4fc8d8c1" +SIMULATION_TABLE_REMOVAL_REVISION = "363f34954a14" V2_TABLE_NAMES = frozenset(table.name for table in V2_METADATA.tables.values()) +# The retired association columns the removal revision drops. +RETIRED_ASSOCIATION_COLUMNS = { + "user_simulation_associations": frozenset({"country", "label"}), + "user_report_associations": frozenset({"country", "label"}), +} +# The native enum types only the dropped tables used. +SIMULATION_DOMAIN_ENUM_TYPES = frozenset( + { + "v2_aggregate_type", + "v2_decile_type", + "v2_household_job_status", + "v2_output_status", + "v2_report_run_status", + "v2_report_run_trigger", + "v2_simulation_status", + "v2_simulation_type", + } +) def _disposable_url() -> str: @@ -52,6 +73,14 @@ def _config() -> Config: return Config(str(REPO / "alembic-v2.ini")) +def _simulation_table_removal_parent() -> str: + """Return the revision the simulation-table removal applies to.""" + removal = ScriptDirectory.from_config(_config()).get_revision( + SIMULATION_TABLE_REMOVAL_REVISION + ) + return removal.down_revision + + def _assert_head(engine) -> None: assert set(inspect(engine).get_table_names(schema="public")) == ( V2_TABLE_NAMES | {"alembic_version"} @@ -174,10 +203,6 @@ def _assert_stage_12a_association_shape(engine) -> None: if foreign_key["referred_table"] == "user_policies" ) assert report_policy_key["options"]["ondelete"] == "SET NULL" - for table_name in ("user_simulation_associations", "user_report_associations"): - columns = _columns(engine, table_name) - assert columns["country"]["nullable"] - assert columns["label"]["nullable"] user_columns = _columns(engine, "users") assert not user_columns["user_type"]["nullable"] assert "standard" in user_columns["user_type"]["default"] @@ -193,6 +218,42 @@ def _assert_stage_12a_association_shape(engine) -> None: } <= user_checks +def _enum_type_names(engine) -> set[str]: + with engine.connect() as connection: + return set( + connection.execute( + text("SELECT typname FROM pg_type WHERE typtype = 'e'") + ).scalars() + ) + + +def _assert_simulation_domain_schema_present(engine) -> None: + assert set(inspect(engine).get_table_names(schema="public")) >= ( + SIMULATION_SERVICE_TABLES + ) + assert _enum_type_names(engine) >= SIMULATION_DOMAIN_ENUM_TYPES + for table_name, retired_columns in RETIRED_ASSOCIATION_COLUMNS.items(): + columns = _columns(engine, table_name) + assert retired_columns <= columns.keys() + assert all(columns[name]["nullable"] for name in retired_columns) + filter_pair = next( + constraint + for constraint in inspect(engine).get_check_constraints("simulations") + if constraint["name"] == "ck_simulations_filter_pair" + ) + assert "filter_field IS NULL" in filter_pair["sqltext"] + assert "filter_value IS NULL" in filter_pair["sqltext"] + + +def _assert_simulation_domain_schema_removed(engine) -> None: + assert SIMULATION_SERVICE_TABLES.isdisjoint( + inspect(engine).get_table_names(schema="public") + ) + assert SIMULATION_DOMAIN_ENUM_TYPES.isdisjoint(_enum_type_names(engine)) + for table_name, retired_columns in RETIRED_ASSOCIATION_COLUMNS.items(): + assert retired_columns.isdisjoint(_columns(engine, table_name)) + + def _upgrade_and_assert_head(config: Config, engine: Engine) -> None: command.upgrade(config, "head") command.check(config) @@ -364,6 +425,13 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: } _upgrade_and_assert_head(config, engine) + _assert_simulation_domain_schema_removed(engine) + + command.downgrade(config, _simulation_table_removal_parent()) + _assert_revision(engine, _simulation_table_removal_parent()) + _assert_simulation_domain_schema_present(engine) + _upgrade_and_assert_head(config, engine) + _assert_simulation_domain_schema_removed(engine) command.downgrade(config, STAGE_12A_PREVIOUS_REVISION) _assert_revision(engine, STAGE_12A_PREVIOUS_REVISION) @@ -396,12 +464,10 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: engine.dispose() -# Dormant at the parent revision: its models map these tables, and deleting a -# user cascades through them, but no parent-revision code inserts into them. -# They are the only tables Stage 12A may give a new required column. -PARENT_REVISION_READ_ONLY_TABLES = frozenset( - {"user_simulation_associations", "user_report_associations"} -) +# Tables the previous revision maps but never writes, which head may therefore +# give a new required column. There are none: the revision before the +# simulation-table removal writes every table it still maps. +PARENT_REVISION_READ_ONLY_TABLES: frozenset[str] = frozenset() USER_ASSOCIATION_TABLES = ( "user_policies", "user_household_associations", @@ -409,8 +475,17 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: "user_report_associations", ) COUNTRY_COLUMN_NAMES = frozenset({"country", "country_id", "primary_country"}) -# Nullable references that a parent-revision check constraint requires. -REFERENCES_REQUIRED_BY_CHECKS = frozenset({("simulations", "household_id")}) +# Nullable columns that a parent-revision check constraint requires: a +# standard user's country, and one reference of each association. +NULLABLE_COLUMNS_REQUIRED_BY_CHECKS = frozenset( + { + ("users", "primary_country"), + ("user_policies", "legacy_policy_id"), + ("user_household_associations", "legacy_household_id"), + ("user_simulation_associations", "legacy_simulation_id"), + ("user_report_associations", "legacy_report_id"), + } +) # Required values that a parent-revision check constraint restricts. CHECKED_VALUES = {("datasets", "year"): 2026} @@ -481,7 +556,7 @@ def _row_values( column.name: _test_value(table, column, sequence) for column in table.columns if _is_required_without_default(column) - or (table.name, column.name) in REFERENCES_REQUIRED_BY_CHECKS + or (table.name, column.name) in NULLABLE_COLUMNS_REQUIRED_BY_CHECKS } if head_table is not None: for column_name in _new_required_columns(table, head_table): @@ -596,24 +671,52 @@ def _assert_user_deletion_cascades( assert remaining == 0, table_name -def test_stage_12a_schema_serves_the_previous_revision() -> None: +def _reflect_previous_revision_usage(engine) -> sa.MetaData: + """Reflect the tables and columns the previous application revision maps. + + It maps neither the simulation-domain tables nor the retired association + columns, so the removal revision may drop them while it serves traffic. + """ + parent = sa.MetaData() + for table_name, retired_columns in RETIRED_ASSOCIATION_COLUMNS.items(): + sa.Table( + table_name, + parent, + autoload_with=engine, + include_columns=[ + name + for name in _columns(engine, table_name) + if name not in retired_columns + ], + ) + parent.reflect( + engine, + only=lambda table_name, _metadata: table_name not in SIMULATION_SERVICE_TABLES, + ) + return parent + + +def test_simulation_table_removal_serves_the_previous_revision() -> None: """Run the previous Public API revision's table usage against head. - The parent revision's schema is reflected and then used, after upgrading, - to insert a row into every table it maps (supplying only what it - requires), read the row back with every mapped column, rewrite those - columns, delete it, and delete a user who owns Stage 12A rows so the - cascades run. Head may not give any table the parent revision writes a - new required column without a server default. + The removal revision drops exactly the simulation-domain tables and the + retired association columns, which the previous application revision no + longer maps. The rest of the parent schema is reflected and then used, + after upgrading, to insert a row into every table it maps (supplying only + what it requires), read the row back with every mapped column, rewrite + those columns, delete it, and delete a user who owns association rows so + the cascades run. Head may not give any table the parent revision writes + a new required column without a server default. """ database_url = _disposable_url() config = _config() engine = create_engine(database_url) try: - command.downgrade(config, STAGE_12A_PREVIOUS_REVISION) - parent = sa.MetaData() - parent.reflect(engine) + command.downgrade(config, _simulation_table_removal_parent()) + full_parent = sa.MetaData() + full_parent.reflect(engine) + parent = _reflect_previous_revision_usage(engine) command.upgrade(config, "head") head = sa.MetaData() head.reflect(engine) @@ -621,6 +724,19 @@ def test_stage_12a_schema_serves_the_previous_revision() -> None: table for table in parent.sorted_tables if table.name != "alembic_version" ] + assert set(full_parent.tables) - set(head.tables) == SIMULATION_SERVICE_TABLES + assert sorted( + f"{table.name}.{column.name}" + for table in full_parent.tables.values() + if table.name in head.tables + for column in table.columns + if column.name not in head.tables[table.name].columns + ) == sorted( + f"{table_name}.{column_name}" + for table_name, retired_columns in RETIRED_ASSOCIATION_COLUMNS.items() + for column_name in retired_columns + ) + dropped_columns = sorted( f"{table.name}.{column.name}" for table in parent_tables @@ -722,7 +838,8 @@ def report_run_checks() -> set[str]: } try: - command.upgrade(config, "head") + # The removal revision drops report_runs, so check it before that. + command.downgrade(config, _simulation_table_removal_parent()) assert isinstance(idempotency_column_type(), postgresql.UUID) assert "ck_report_runs_idempotency_key_nonblank" not in report_run_checks() diff --git a/tests/integration/test_v2_catalog_publication.py b/tests/integration/test_v2_catalog_publication.py index d2efb3c7f..c3980d85a 100644 --- a/tests/integration/test_v2_catalog_publication.py +++ b/tests/integration/test_v2_catalog_publication.py @@ -25,7 +25,6 @@ TaxBenefitModel, TaxBenefitModelVersion, ) -from policyengine_api.data.v2.privileges import CATALOG_WRITER_ROLE from policyengine_api.data.v2.settings import V2_MIGRATION_DATABASE_URL from tests.fixtures.v2_catalog import normalized_catalog @@ -78,9 +77,6 @@ def _counts(engine: Engine) -> dict[str, int]: "datasets", "dataset_versions", "regions", - "simulations", - "reports", - "report_runs", ) } @@ -132,9 +128,6 @@ def test_publish_and_republish_preserve_complete_catalog( "datasets": 3, "dataset_versions": 0, "regions": 5, - "simulations": 0, - "reports": 0, - "report_runs": 0, } ) assert _identifiers(publication_engine) == first_ids @@ -257,29 +250,6 @@ def test_republish_preserves_existing_dataset_version_records( assert session.get(Dataset, protected_ids[1]) is not None -def test_catalog_writer_publishes_without_access_to_simulation_tables( - publication_engine: Engine, - v2_engine_as_role, -) -> None: - # A later v2 revision drops simulations, reports and report_runs while - # this release serves traffic, so publication must work without them. - with publication_engine.begin() as connection: - connection.execute( - text( - "REVOKE ALL ON simulations, reports, report_runs " - f"FROM {CATALOG_WRITER_ROLE}" - ) - ) - catalog_writer_engine = v2_engine_as_role(CATALOG_WRITER_ROLE, poolclass=NullPool) - try: - evidence = publish_catalog(catalog_writer_engine, normalized_catalog()) - finally: - catalog_writer_engine.dispose() - - assert evidence.as_dict()["outcome"] == "ok" - assert _counts(publication_engine)["tax_benefit_models"] == 2 - - def test_new_version_is_additive_and_concurrent_retries_serialize( publication_engine: Engine, ) -> None: @@ -309,7 +279,4 @@ def test_new_version_is_additive_and_concurrent_retries_serialize( "datasets": 6, "dataset_versions": 0, "regions": 10, - "simulations": 0, - "reports": 0, - "report_runs": 0, } diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index 4c0e6ee09..7a4c2e5d8 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -16,6 +16,7 @@ from policyengine_api.constants import REPO from policyengine_api.data.v1_models import V1Base +from policyengine_api.data.v2.metadata_validation import SIMULATION_SERVICE_TABLES from policyengine_api.data.v2.migration_target import ( DISPOSABLE_DATABASE_NAME, MIGRATION_ROLE, @@ -49,6 +50,28 @@ "user_identities", } ) +STAGE_12A_REVISION = "fbb7488ebe8a" +SIMULATION_TABLE_REMOVAL_REVISION = "363f34954a14" +# The output tables of a report run, which reference report_runs and simulations. +RUN_OUTPUT_TABLES = SIMULATION_SERVICE_TABLES - { + "household_jobs", + "report_runs", + "reports", + "simulations", +} +# The native enum types only the simulation-domain tables used. +SIMULATION_DOMAIN_ENUM_TYPES = frozenset( + { + "v2_aggregate_type", + "v2_decile_type", + "v2_household_job_status", + "v2_output_status", + "v2_report_run_status", + "v2_report_run_trigger", + "v2_simulation_status", + "v2_simulation_type", + } +) POOLER_URL = ( "postgresql+psycopg://policyengine_v2_migrator." f"{PROJECT_REF}:test-password@aws-0-us-east-2.pooler.supabase.com:5432/" @@ -235,6 +258,7 @@ def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> N assert script.get_heads() == [V2_ALEMBIC_HEAD_REVISION] assert [revision.revision for revision in script.walk_revisions()] == [ V2_ALEMBIC_HEAD_REVISION, + STAGE_12A_REVISION, "91af4fc8d8c1", "60d6518b6a98", "439303be14fe", @@ -268,7 +292,11 @@ def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> N assert "fk_regions_default_dataset_model_datasets" in baseline assert "uq_datasets_model_name" in baseline assert "ck_datasets_output_storage_path" in baseline - baseline_table_count = len(V2_TABLE_NAMES - POST_BASELINE_TABLE_NAMES) + # The baseline also created the simulation-domain tables a later revision + # dropped. + baseline_table_count = len( + (V2_TABLE_NAMES | SIMULATION_SERVICE_TABLES) - POST_BASELINE_TABLE_NAMES + ) assert baseline.count("op.create_table(") == baseline_table_count assert baseline.count("op.drop_table(") == baseline_table_count @@ -495,7 +523,6 @@ def test_stage_12a_user_identity_and_association_revision_is_generated() -> None / "migrations/v2/versions/fbb7488ebe8a_link_signed_in_users_to_v1_anchored_.py" ).read_text(encoding="utf-8") - assert V2_ALEMBIC_HEAD_REVISION == "fbb7488ebe8a" assert ( "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" in revision @@ -739,3 +766,97 @@ def test_v2_environment_commits_after_persistent_target_qualification() -> None: assert "current_heads != previous_heads" in env_source assert "current_heads == script_heads" in env_source assert "validate_v2_head_schema(connection, target_metadata)" in env_source + + +def _simulation_table_removal_revision() -> str: + (path,) = (REPO / "migrations/v2/versions").glob( + f"{SIMULATION_TABLE_REMOVAL_REVISION}_*.py" + ) + return path.read_text(encoding="utf-8") + + +def _revision_sections(revision: str) -> tuple[str, str]: + upgrade_start = revision.index("def upgrade()") + downgrade_start = revision.index("def downgrade()") + return revision[upgrade_start:downgrade_start], revision[downgrade_start:] + + +def test_simulation_table_removal_revision_is_generated_and_drops_only_them() -> None: + revision = _simulation_table_removal_revision() + upgrade, downgrade = _revision_sections(revision) + + assert V2_ALEMBIC_HEAD_REVISION == SIMULATION_TABLE_REMOVAL_REVISION + assert ( + "Generation: uv run alembic -c alembic-v2.ini revision --autogenerate" + in revision + ) + assert f'down_revision: Union[str, None] = "{STAGE_12A_REVISION}"' in revision + assert "op.execute(" not in revision + assert "op.bulk_insert(" not in revision + # The upgrade drops exactly the simulation-domain tables and the two + # retired columns of each association; the downgrade recreates them. + assert set(re.findall(r'op\.drop_table\("([a-z0-9_]+)"\)', upgrade)) == ( + SIMULATION_SERVICE_TABLES + ) + assert "op.create_table(" not in upgrade + assert set(re.findall(r'op\.create_table\(\s*"([a-z0-9_]+)"', downgrade)) == ( + SIMULATION_SERVICE_TABLES + ) + assert "op.drop_table(" not in downgrade + retired_columns = { + (table_name, column_name) + for table_name in ("user_simulation_associations", "user_report_associations") + for column_name in ("country", "label") + } + assert ( + set(re.findall(r'op\.drop_column\("([a-z_]+)", "([a-z_]+)"\)', upgrade)) + == retired_columns + ) + assert ( + set( + re.findall( + r'op\.add_column\(\s*"([a-z_]+)",\s*sa\.Column\(\s*"([a-z_]+)"', + downgrade, + ) + ) + == retired_columns + ) + assert "op.alter_column(" not in revision + + +def test_simulation_table_removal_revision_has_only_its_three_corrections() -> None: + revision = _simulation_table_removal_revision() + upgrade, downgrade = _revision_sections(revision) + + # The three corrections docs/engineering/skills/alembic-migrations.md + # describes: drop and create order, the enum types Alembic leaves behind, + # and the check constraint reflection does not reproduce. + assert revision.count("# Post-generation correction:") == 3 + assert revision.count("# Post-generation reversibility correction:") == 1 + corrected_enum_names = set( + re.findall( + r"sa\.Enum\(name=[\"']([^\"']+)[\"']\)\.drop\(op\.get_bind\(\)\)", + upgrade, + ) + ) + assert corrected_enum_names == SIMULATION_DOMAIN_ENUM_TYPES + assert ".drop(op.get_bind())" not in downgrade + assert "(filter_field IS NULL) = (filter_value IS NULL)" in downgrade + + def position(section: str, operation: str, table_name: str) -> int: + match = re.search(rf'{re.escape(operation)}\(\s*"{table_name}"', section) + assert match, (operation, table_name) + return match.start() + + for referencing, referenced in ( + *((output, "report_runs") for output in RUN_OUTPUT_TABLES), + *((output, "simulations") for output in RUN_OUTPUT_TABLES), + ("report_runs", "reports"), + ("reports", "simulations"), + ): + assert position(upgrade, "op.drop_table", referencing) < position( + upgrade, "op.drop_table", referenced + ) + assert position(downgrade, "op.create_table", referenced) < position( + downgrade, "op.create_table", referencing + ) diff --git a/tests/unit/v2/test_import_side_effects.py b/tests/unit/v2/test_import_side_effects.py index 7b753004d..a1bb4c961 100644 --- a/tests/unit/v2/test_import_side_effects.py +++ b/tests/unit/v2/test_import_side_effects.py @@ -69,7 +69,7 @@ def reject_ddl(*args, **kwargs): import sys after = set(pathlib.Path.cwd().iterdir()) assert before == after -assert len(V2_METADATA.tables) == 39 +assert len(V2_METADATA.tables) == 24 assert "policyengine_api.data.v2.catalog.initialization" not in sys.modules assert "policyengine_api.data.v2.policy_migration_qualification" not in sys.modules assert "policyengine_api.data.v2.household_migration_qualification" not in sys.modules diff --git a/tests/unit/v2/test_metadata_validation.py b/tests/unit/v2/test_metadata_validation.py index 935e4ed9b..8dc488c13 100644 --- a/tests/unit/v2/test_metadata_validation.py +++ b/tests/unit/v2/test_metadata_validation.py @@ -4,6 +4,7 @@ from policyengine_api.data.v2.metadata_validation import ( PROHIBITED_V2_TABLES, + SIMULATION_SERVICE_TABLES, V1_ONLY_TABLES, V2MetadataValidationError, validate_v2_metadata_table_names, @@ -17,7 +18,17 @@ def test_current_metadata_contains_no_rejected_tables() -> None: @pytest.mark.parametrize( "table_name", - ["runtime_bundles", "populations", "household", "user_profiles"], + [ + "runtime_bundles", + "populations", + "household", + "user_profiles", + "simulations", + "reports", + "report_runs", + "household_jobs", + "budget_summary", + ], ) def test_known_rejected_tables_are_identified(table_name: str) -> None: with pytest.raises(V2MetadataValidationError, match=table_name): @@ -29,6 +40,29 @@ def test_rejection_sets_do_not_block_current_metadata() -> None: assert table_names.isdisjoint(PROHIBITED_V2_TABLES) assert table_names.isdisjoint(V1_ONLY_TABLES) + assert table_names.isdisjoint(SIMULATION_SERVICE_TABLES) + + +def test_simulation_service_tables_are_the_fifteen_dropped_tables() -> None: + dropped_tables = { + "aggregates", + "budget_summary", + "change_aggregates", + "congressional_district_impacts", + "constituency_impacts", + "decile_impacts", + "household_jobs", + "inequality", + "intra_decile_impacts", + "local_authority_impacts", + "poverty", + "program_statistics", + "report_runs", + "reports", + "simulations", + } + + assert dropped_tables == SIMULATION_SERVICE_TABLES def test_new_table_name_requires_no_allowlist_update() -> None: diff --git a/tests/unit/v2/test_model_persistence.py b/tests/unit/v2/test_model_persistence.py index d14a774c7..b5fa6dc35 100644 --- a/tests/unit/v2/test_model_persistence.py +++ b/tests/unit/v2/test_model_persistence.py @@ -129,7 +129,6 @@ def test_direct_sqlalchemy_categories_are_complete_and_documented() -> None: "typed_json_and_text", "named_constraints_and_indexes", "ambiguous_foreign_key_relationships", - "retired_table_declarations", } assert all(DIRECT_SQLALCHEMY_EXCEPTIONS.values()) diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index ddde6b525..1d8f4e623 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -7,6 +7,7 @@ from policyengine_api.data.v1_models import V1Base from policyengine_api.data.v2 import models as v2_models +from policyengine_api.data.v2.metadata_validation import SIMULATION_SERVICE_TABLES from policyengine_api.data.v2.models import ( V2_METADATA, DatasetVersion, @@ -28,29 +29,7 @@ UserType, ) -RUN_OUTPUT_TABLES = frozenset( - { - "aggregates", - "budget_summary", - "change_aggregates", - "congressional_district_impacts", - "constituency_impacts", - "decile_impacts", - "inequality", - "intra_decile_impacts", - "local_authority_impacts", - "poverty", - "program_statistics", - } -) - -# The simulation service owns these tables in its own schema. The v2 metadata -# still declares them, so it matches the schema until a later revision drops -# them, but no model maps them: nothing this release runs reads or writes them. -RETIRED_SIMULATION_TABLES = frozenset( - {"household_jobs", "simulations", "reports", "report_runs", *RUN_OUTPUT_TABLES} -) -# Retired at Stage 12A; dropped by the same later revision. +# Retired at Stage 12A and dropped with the simulation-domain tables. RETIRED_ASSOCIATION_COLUMNS = frozenset({"country", "label"}) REMOVED_SIMULATION_MODEL_NAMES = frozenset( { @@ -113,14 +92,14 @@ def _v2_mappers(): ) -def test_every_kept_metadata_table_has_one_mapped_model() -> None: +def test_every_metadata_table_has_one_mapped_model() -> None: metadata_table_names = set(V2_METADATA.tables) model_table_names = {mapper.local_table.name for mapper in _v2_mappers()} assert V2_METADATA is not V1Base.metadata - assert metadata_table_names >= RETIRED_SIMULATION_TABLES - assert model_table_names == metadata_table_names - RETIRED_SIMULATION_TABLES - assert len(_v2_mappers()) == len(model_table_names) + assert SIMULATION_SERVICE_TABLES.isdisjoint(metadata_table_names) + assert model_table_names == metadata_table_names + assert len(_v2_mappers()) == len(metadata_table_names) def test_models_package_exports_no_simulation_domain_model() -> None: @@ -129,7 +108,7 @@ def test_models_package_exports_no_simulation_domain_model() -> None: assert REMOVED_SIMULATION_MODEL_NAMES.isdisjoint(exported_names) -def test_kept_models_have_no_relationship_to_a_retired_table() -> None: +def test_kept_models_have_no_relationship_to_a_simulation_service_table() -> None: related_tables = { f"{mapper.class_.__name__}.{relationship.key}": ( relationship.mapper.local_table.name @@ -141,18 +120,14 @@ def test_kept_models_have_no_relationship_to_a_retired_table() -> None: assert not { relationship: table_name for relationship, table_name in related_tables.items() - if table_name in RETIRED_SIMULATION_TABLES + if table_name in SIMULATION_SERVICE_TABLES } -def test_association_models_do_not_map_the_retired_columns() -> None: +def test_association_tables_no_longer_have_the_retired_columns() -> None: for model in (UserSimulationAssociation, UserReportAssociation): table = V2_METADATA.tables[model.__tablename__] - # Declared on the table, so the metadata matches the schema... - assert set(table.c.keys()) >= RETIRED_ASSOCIATION_COLUMNS - assert all(table.c[name].nullable for name in RETIRED_ASSOCIATION_COLUMNS) - # ...but neither mapped nor a model field. - assert RETIRED_ASSOCIATION_COLUMNS.isdisjoint(sa.inspect(model).columns.keys()) + assert RETIRED_ASSOCIATION_COLUMNS.isdisjoint(table.c.keys()) assert RETIRED_ASSOCIATION_COLUMNS.isdisjoint(model.model_fields) From 2932874ddfcc69052ad5c1d02c9d4ecdcee5f9c6 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 17:56:19 +0400 Subject: [PATCH 2/4] Register a rule that keeps the simulation-domain models removed The previous commit dropped the Public API's copies of the 15 simulation-domain tables, which the simulation service owns in its own schema. Rule api.v2.simulation-tables-stay-removed keeps them out: - test_simulation_domain_boundaries reads every module under policyengine_api, scripts and tests, and the v2 migration environment, and fails when one imports a removed model, the removed models/reports, models/simulations or models/retired_tables modules, or the removed report_runs module, including from the parent package and relatively; - it fails if the v2 metadata declares one of the 15 tables, or the models package exports a removed model (moved from test_models); - the metadata validation test that rejects their table names is the rule's third check. Its proof fixtures use imports the household migration qualification and the report-run tests made before the models were removed. The registry renders the rule into docs/generated/rules.md and AGENTS.md, and the Alembic migrations and testing skills cite it. Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 2 +- docs/engineering/skills/alembic-migrations.md | 3 +- docs/engineering/skills/testing.md | 5 +- docs/generated/rules.md | 9 +- .../bad_import.py | 31 ++++ .../good_import.py | 6 + rules/registry.toml | 26 +++ tests/unit/v2/test_models.py | 34 ---- .../v2/test_simulation_domain_boundaries.py | 165 ++++++++++++++++++ 9 files changed, 241 insertions(+), 40 deletions(-) create mode 100644 rules/fixtures/api.v2.simulation-tables-stay-removed/bad_import.py create mode 100644 rules/fixtures/api.v2.simulation-tables-stay-removed/good_import.py create mode 100644 tests/unit/v2/test_simulation_domain_boundaries.py diff --git a/AGENTS.md b/AGENTS.md index 1f7b8c19e..1837b50ce 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -102,7 +102,7 @@ To add or change a rule, read `docs/engineering/skills/rules-registry.md`. ### Data source isolation -- Checked: `api.catalog.extractor-import-isolation`, `api.metadata.read-modules-isolated`, `api.reports.v1-persistence-only`, `api.stage12.comparison-tables-isolated`, `api.stage12.retirement-owned`, `api.v2.single-orm-layer` +- Checked: `api.catalog.extractor-import-isolation`, `api.metadata.read-modules-isolated`, `api.reports.v1-persistence-only`, `api.stage12.comparison-tables-isolated`, `api.stage12.retirement-owned`, `api.v2.simulation-tables-stay-removed`, `api.v2.single-orm-layer` ### Migration and schema diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index 2f02a6712..fd7428d3f 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -414,4 +414,5 @@ dropped tables or columns can serve traffic, and it may run only after an operator has confirmed in each environment that the tables are empty. Once it has run, an application rollback to a revision that maps them, including any pre-Stage 12A revision, needs the schema downgraded to `fbb7488ebe8a` first. -The v2 metadata validation rejects their table names. +`api.v2.simulation-tables-stay-removed` stops code from importing the removed +models again, and the v2 metadata validation rejects their table names. diff --git a/docs/engineering/skills/testing.md b/docs/engineering/skills/testing.md index e7713ea05..e2179d9d7 100644 --- a/docs/engineering/skills/testing.md +++ b/docs/engineering/skills/testing.md @@ -75,10 +75,11 @@ python -m pytest tests/unit/test_cloud_run_deploy_scripts.py tests/unit/test_cap For v2 platform foundation work, run the smallest applicable group while iterating, then all groups before qualification. -SQLModel schema, lazy configuration, and import effects: +SQLModel schema, the removed simulation-domain models, lazy configuration, and +import effects: ```bash -uv run pytest tests/unit/v2/test_models.py tests/unit/v2/test_model_persistence.py tests/unit/v2/test_settings.py tests/unit/v2/test_import_side_effects.py -q +uv run pytest tests/unit/v2/test_models.py tests/unit/v2/test_model_persistence.py tests/unit/v2/test_simulation_domain_boundaries.py tests/unit/v2/test_settings.py tests/unit/v2/test_import_side_effects.py -q ``` The generated-only v2 Alembic chain and disposable Postgres lifecycle: diff --git a/docs/generated/rules.md b/docs/generated/rules.md index 079f41b3f..a48377314 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -8,11 +8,11 @@ Every rule that governs `policyengine-api` is listed here with how it is enforce | Tier | enforced | ratcheted | advisory | doc-only | Total | | --- | ---: | ---: | ---: | ---: | ---: | -| mechanical | 59 | 0 | 1 | 13 | 73 | +| mechanical | 60 | 0 | 1 | 13 | 74 | | metric | 1 | 0 | 0 | 0 | 1 | | judgment | 0 | 0 | 0 | 1 | 1 | -**Blocking:** 60 of 75 rules. +**Blocking:** 61 of 76 rules. ## Gaps @@ -207,6 +207,7 @@ flowchart LR | `api.reports.v1-persistence-only` | The existing report route and service import only v1 persistence. | mechanical | enforced | yes | pytest `tests/unit/v2/test_report_v1_runtime_selection.py` | `pr.yml:test` | **unproven** | | `api.stage12.comparison-tables-isolated` | Serving routes and services never read the Stage 12 comparison tables, and production tables never depend on them. | mechanical | enforced | yes | pytest `tests/unit/v2/test_stage12_comparison_isolation.py` | `pr.yml:test` | **unproven** | | `api.stage12.retirement-owned` | Retiring the Stage 12 comparison schema stays owned by a named follow-up change in the Stage 12 worker contract. | mechanical | enforced | yes | pytest `tests/unit/v2/test_stage12_comparison_isolation.py` | `pr.yml:test` | **unproven** | +| `api.v2.simulation-tables-stay-removed` | Public API code never imports the removed simulation-domain models (household jobs, simulations, reports, report runs and their outputs), and the v2 metadata never declares their tables, which the simulation service owns in its own schema. | mechanical | enforced | yes | pytest `tests/unit/v2/test_metadata_validation.py`, `tests/unit/v2/test_simulation_domain_boundaries.py` | `pr.yml:test` | yes | | `api.v2.single-orm-layer` | v2 models never create a parallel SQLAlchemy ORM layer alongside SQLModel. | mechanical | enforced | yes | pytest `tests/unit/v2/test_model_persistence.py` | `pr.yml:test` | **unproven** | ```mermaid @@ -221,6 +222,8 @@ flowchart LR r_api_stage12_comparison_tables_isolated("api.stage12.comparison-tables-isolated (enforced)") e_pytest_tests_unit_v2_test_stage12_comparison_isolation_py["pytest tests/unit/v2/test_stage12_comparison_isolation.py"] r_api_stage12_retirement_owned("api.stage12.retirement-owned (enforced)") + r_api_v2_simulation_tables_stay_removed("api.v2.simulation-tables-stay-removed (enforced)") + e_pytest_tests_unit_v2_test_metadata_validation_py__tests_unit_v2_test_simulation_domain_boundaries_py["pytest tests/unit/v2/test_metadata_validation.py, tests/unit/v2/test_simulation_domain_boundaries.py"] r_api_v2_single_orm_layer("api.v2.single-orm-layer (enforced)") e_pytest_tests_unit_v2_test_model_persistence_py["pytest tests/unit/v2/test_model_persistence.py"] r_api_catalog_extractor_import_isolation --> e_pytest_tests_unit_v2_test_catalog_extraction_py @@ -232,6 +235,8 @@ flowchart LR r_api_stage12_comparison_tables_isolated --> e_pytest_tests_unit_v2_test_stage12_comparison_isolation_py e_pytest_tests_unit_v2_test_stage12_comparison_isolation_py --> j_CI_pr_yml_test r_api_stage12_retirement_owned --> e_pytest_tests_unit_v2_test_stage12_comparison_isolation_py + r_api_v2_simulation_tables_stay_removed --> e_pytest_tests_unit_v2_test_metadata_validation_py__tests_unit_v2_test_simulation_domain_boundaries_py + e_pytest_tests_unit_v2_test_metadata_validation_py__tests_unit_v2_test_simulation_domain_boundaries_py --> j_CI_pr_yml_test r_api_v2_single_orm_layer --> e_pytest_tests_unit_v2_test_model_persistence_py e_pytest_tests_unit_v2_test_model_persistence_py --> j_CI_pr_yml_test ``` diff --git a/rules/fixtures/api.v2.simulation-tables-stay-removed/bad_import.py b/rules/fixtures/api.v2.simulation-tables-stay-removed/bad_import.py new file mode 100644 index 000000000..3e3132dbd --- /dev/null +++ b/rules/fixtures/api.v2.simulation-tables-stay-removed/bad_import.py @@ -0,0 +1,31 @@ +"""Proof fixture: imports of removed simulation-domain models; the rule rejects it.""" + +# The household migration qualification imported these before the models were +# removed, and the report-run tests imported the report_runs module. +import policyengine_api.data.v2.models.retired_tables +from policyengine_api.data.v2.report_runs import ReportRunPackageVersions + +from policyengine_api.data.v2 import report_runs +from policyengine_api.data.v2.models import ( + Household, + HouseholdJob, + Report, + Simulation, + UserHouseholdAssociation, +) + +from . import report_runs as sibling_report_runs +from .models.simulations import SimulationStatus + +__all__ = [ + "Household", + "HouseholdJob", + "Report", + "ReportRunPackageVersions", + "Simulation", + "SimulationStatus", + "UserHouseholdAssociation", + "policyengine_api", + "report_runs", + "sibling_report_runs", +] diff --git a/rules/fixtures/api.v2.simulation-tables-stay-removed/good_import.py b/rules/fixtures/api.v2.simulation-tables-stay-removed/good_import.py new file mode 100644 index 000000000..b95b23710 --- /dev/null +++ b/rules/fixtures/api.v2.simulation-tables-stay-removed/good_import.py @@ -0,0 +1,6 @@ +"""Proof fixture: kept v2 models and a same-named v1 model; the rule accepts it.""" + +from policyengine_api.data.v1_models import Simulation +from policyengine_api.data.v2.models import Household, Policy, UserReportAssociation + +__all__ = ["Household", "Policy", "Simulation", "UserReportAssociation"] diff --git a/rules/registry.toml b/rules/registry.toml index 37a41e923..e0cb247a0 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -612,6 +612,32 @@ workflow = ".github/workflows/pr.yml" job = "test" invocation = "make test" +[[rule]] +id = "api.v2.simulation-tables-stay-removed" +category = "isolation" +statement = "Public API code never imports the removed simulation-domain models (household jobs, simulations, reports, report runs and their outputs), and the v2 metadata never declares their tables, which the simulation service owns in its own schema." +rationale = "docs/engineering/skills/alembic-migrations.md#simulation-domain-table-removal" +scope = ["policyengine_api/**/*.py", "scripts/**/*.py", "tests/**/*.py", "migrations/v2/env.py"] +tier = "mechanical" +status = "enforced" +autofix = false +[rule.enforcer] +kind = "pytest" +nodes = [ + "tests/unit/v2/test_simulation_domain_boundaries.py::test_no_module_imports_a_removed_simulation_model", + "tests/unit/v2/test_simulation_domain_boundaries.py::test_v2_metadata_declares_no_simulation_service_table", + "tests/unit/v2/test_simulation_domain_boundaries.py::test_models_package_exports_no_simulation_domain_model", + "tests/unit/v2/test_metadata_validation.py::test_known_rejected_tables_are_identified", +] +[[rule.ci]] +workflow = ".github/workflows/pr.yml" +job = "test" +invocation = "make test" +[rule.proof] +bad_fixture = "rules/fixtures/api.v2.simulation-tables-stay-removed/bad_import.py" +good_fixture = "rules/fixtures/api.v2.simulation-tables-stay-removed/good_import.py" +test = "tests/unit/v2/test_simulation_domain_boundaries.py::test_rule_proof_simulation_tables_stay_removed" + # --- Migration and schema -------------------------------------------------- [[rule]] diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index 1d8f4e623..dd92985ca 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -6,7 +6,6 @@ from sqlalchemy.schema import CreateTable from policyengine_api.data.v1_models import V1Base -from policyengine_api.data.v2 import models as v2_models from policyengine_api.data.v2.metadata_validation import SIMULATION_SERVICE_TABLES from policyengine_api.data.v2.models import ( V2_METADATA, @@ -31,33 +30,6 @@ # Retired at Stage 12A and dropped with the simulation-domain tables. RETIRED_ASSOCIATION_COLUMNS = frozenset({"country", "label"}) -REMOVED_SIMULATION_MODEL_NAMES = frozenset( - { - "AggregateOutput", - "AggregateType", - "BudgetSummary", - "ChangeAggregate", - "CongressionalDistrictImpact", - "ConstituencyImpact", - "DecileImpact", - "DecileType", - "HouseholdJob", - "HouseholdJobStatus", - "Inequality", - "IntraDecileImpact", - "LocalAuthorityImpact", - "OutputStatus", - "Poverty", - "ProgramStatistics", - "Report", - "ReportRun", - "ReportRunStatus", - "ReportRunTrigger", - "Simulation", - "SimulationStatus", - "SimulationType", - } -) def test_domain_models_are_grouped_into_topic_scoped_modules() -> None: @@ -102,12 +74,6 @@ def test_every_metadata_table_has_one_mapped_model() -> None: assert len(_v2_mappers()) == len(metadata_table_names) -def test_models_package_exports_no_simulation_domain_model() -> None: - exported_names = set(v2_models.__all__) | set(vars(v2_models)) - - assert REMOVED_SIMULATION_MODEL_NAMES.isdisjoint(exported_names) - - def test_kept_models_have_no_relationship_to_a_simulation_service_table() -> None: related_tables = { f"{mapper.class_.__name__}.{relationship.key}": ( diff --git a/tests/unit/v2/test_simulation_domain_boundaries.py b/tests/unit/v2/test_simulation_domain_boundaries.py new file mode 100644 index 000000000..d55c681bc --- /dev/null +++ b/tests/unit/v2/test_simulation_domain_boundaries.py @@ -0,0 +1,165 @@ +"""The simulation-domain tables and models stay out of the Public API. + +The simulation service owns household jobs, simulations, reports, report runs +and their outputs in its own ``simulation`` schema. A v2 revision dropped the +Public API's copies of those tables after their models were removed, so no +code here may import those models again and no v2 model may declare those +tables again (rule ``api.v2.simulation-tables-stay-removed``). +""" + +from __future__ import annotations + +import ast +from typing import TYPE_CHECKING + +from policyengine_api.constants import REPO +from policyengine_api.data.v2 import models as v2_models +from policyengine_api.data.v2.metadata_validation import SIMULATION_SERVICE_TABLES +from policyengine_api.data.v2.models import V2_METADATA + +if TYPE_CHECKING: + from pathlib import Path + +V2_MODELS_PACKAGE = "policyengine_api.data.v2.models" +REMOVED_MODEL_NAMES = frozenset( + { + "AggregateOutput", + "AggregateType", + "BudgetSummary", + "ChangeAggregate", + "CongressionalDistrictImpact", + "ConstituencyImpact", + "DecileImpact", + "DecileType", + "GeographicImpactBase", + "HouseholdJob", + "HouseholdJobStatus", + "Inequality", + "IntraDecileImpact", + "LocalAuthorityImpact", + "OutputStatus", + "Poverty", + "ProgramStatistics", + "Report", + "ReportRun", + "ReportRunStatus", + "ReportRunTrigger", + "Simulation", + "SimulationStatus", + "SimulationType", + } +) +REMOVED_MODULES = frozenset( + { + f"{V2_MODELS_PACKAGE}.reports", + f"{V2_MODELS_PACKAGE}.retired_tables", + f"{V2_MODELS_PACKAGE}.simulations", + "policyengine_api.data.v2.report_runs", + } +) +SCANNED_DIRECTORIES = ("policyengine_api", "scripts", "tests") +SCANNED_FILES = ("migrations/v2/env.py",) +FIXTURES = REPO / "rules/fixtures/api.v2.simulation-tables-stay-removed" + + +def _module_name(path: Path) -> str: + return ".".join(path.relative_to(REPO).with_suffix("").parts) + + +def _imported_module(node: ast.ImportFrom, importing_module: str) -> str: + if node.level == 0: + return node.module or "" + package_parts = importing_module.split(".")[: -node.level] + return ".".join([*package_parts, *([node.module] if node.module else [])]) + + +def removed_model_imports(path: Path, *, module_name: str | None = None) -> list[str]: + """Return each removed simulation-domain module or model that ``path`` imports.""" + importing_module = module_name or _module_name(path) + tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) + found: list[str] = [] + for node in ast.walk(tree): + if isinstance(node, ast.Import): + found.extend( + alias.name for alias in node.names if alias.name in REMOVED_MODULES + ) + elif isinstance(node, ast.ImportFrom): + imported_module = _imported_module(node, importing_module) + if imported_module in REMOVED_MODULES: + found.append(imported_module) + continue + imports_from_v2_models = ( + imported_module == V2_MODELS_PACKAGE + or imported_module.startswith(f"{V2_MODELS_PACKAGE}.") + ) + # A removed module can also be imported from its parent package, + # as in ``from policyengine_api.data.v2 import report_runs``. + found.extend( + f"{imported_module}.{alias.name}" + for alias in node.names + if f"{imported_module}.{alias.name}" in REMOVED_MODULES + or (imports_from_v2_models and alias.name in REMOVED_MODEL_NAMES) + ) + return found + + +def _scanned_files() -> list[Path]: + files = [REPO / relative_path for relative_path in SCANNED_FILES] + for directory in SCANNED_DIRECTORIES: + files.extend(sorted((REPO / directory).rglob("*.py"))) + return files + + +def test_no_module_imports_a_removed_simulation_model() -> None: + offending_imports = { + str(path.relative_to(REPO)): imports + for path in _scanned_files() + if (imports := removed_model_imports(path)) + } + + assert not offending_imports, ( + "The simulation service owns these models; read its API instead: " + f"{offending_imports}" + ) + + +def test_v2_metadata_declares_no_simulation_service_table() -> None: + assert SIMULATION_SERVICE_TABLES.isdisjoint(V2_METADATA.tables) + assert not any( + (REPO / "policyengine_api/data/v2" / relative_path).exists() + for relative_path in ( + "models/reports.py", + "models/retired_tables.py", + "models/simulations.py", + "report_runs.py", + ) + ) + + +def test_models_package_exports_no_simulation_domain_model() -> None: + exported_names = set(v2_models.__all__) | set(vars(v2_models)) + + assert REMOVED_MODEL_NAMES.isdisjoint(exported_names) + + +def test_rule_proof_simulation_tables_stay_removed() -> None: + """Proof for rule api.v2.simulation-tables-stay-removed (rules/registry.toml).""" + fixture_module = "policyengine_api.data.v2.example" + assert removed_model_imports( + FIXTURES / "bad_import.py", module_name=fixture_module + ) == [ + "policyengine_api.data.v2.models.retired_tables", + "policyengine_api.data.v2.report_runs", + # A removed module imported from its parent package. + "policyengine_api.data.v2.report_runs", + "policyengine_api.data.v2.models.HouseholdJob", + "policyengine_api.data.v2.models.Report", + "policyengine_api.data.v2.models.Simulation", + # The same, imported relatively from a sibling module. + "policyengine_api.data.v2.report_runs", + "policyengine_api.data.v2.models.simulations", + ] + assert ( + removed_model_imports(FIXTURES / "good_import.py", module_name=fixture_module) + == [] + ) From 38553b676278d9b96f1856add7d0b142bb485b51 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:31:38 +0400 Subject: [PATCH 3/4] Check the removal downgrade exactly and keep the Stage 12A rollback test Review of the removal revision found three gaps in the lifecycle tests: - The downgrade was checked only for the presence of the tables, enum types and columns. A new test builds the schema from empty up to the revision before the removal, upgrades to head, downgrades again, and requires every recreated table and association table to match what the earlier revisions created: columns with types, nullability and defaults, keys, unique and check constraints, indexes, and the enum types' values. Changing one check in the downgrade makes it fail. - Repurposing the previous-revision test removed the only test that ran the pre-Stage 12A release against the Stage 12A schema, which the updated rollback steps rely on after a downgrade to fbb7488ebe8a. It is back as its own test, against that revision instead of head; both tests share one helper, with each previous revision's usage in a PreviousRevisionUsage value. - The native UUID idempotency test now builds report_runs from the baseline revision, as its name says, instead of from the removal revision's downgrade. Tests only: they pass before and after, except the mutation check above. Co-Authored-By: Claude Opus 5.5 --- .../integration/test_alembic_v2_lifecycle.py | 309 +++++++++++++----- 1 file changed, 234 insertions(+), 75 deletions(-) diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index 5b43419b0..cd47ac73c 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -2,7 +2,8 @@ import itertools import os -from collections.abc import Iterator +from collections.abc import Iterable, Iterator +from dataclasses import dataclass from datetime import UTC, datetime, timedelta from uuid import UUID, uuid4 @@ -464,10 +465,44 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: engine.dispose() -# Tables the previous revision maps but never writes, which head may therefore -# give a new required column. There are none: the revision before the -# simulation-table removal writes every table it still maps. -PARENT_REVISION_READ_ONLY_TABLES: frozenset[str] = frozenset() +@dataclass(frozen=True) +class PreviousRevisionUsage: + """How a previous application revision uses the tables it maps. + + ``read_only_tables`` are tables it maps but never writes, which the newer + schema may therefore give a new required column; only newer code writes + their rows. ``nullable_columns_required_by_checks`` are nullable columns + that one of its check constraints requires a value for. + """ + + read_only_tables: frozenset[str] + nullable_columns_required_by_checks: frozenset[tuple[str, str]] + + +# The revision before Stage 12A. Its models map the two association tables and +# deleting a user cascades through them, but no code of that revision inserts +# into them, so Stage 12A may give them a new required column. +STAGE_12A_PREVIOUS_USAGE = PreviousRevisionUsage( + read_only_tables=frozenset( + {"user_simulation_associations", "user_report_associations"} + ), + nullable_columns_required_by_checks=frozenset({("simulations", "household_id")}), +) +# The revision before the simulation-table removal. It writes every table it +# still maps. Its checks require a standard user's country and one reference +# of each association. +SIMULATION_TABLE_REMOVAL_PREVIOUS_USAGE = PreviousRevisionUsage( + read_only_tables=frozenset(), + nullable_columns_required_by_checks=frozenset( + { + ("users", "primary_country"), + ("user_policies", "legacy_policy_id"), + ("user_household_associations", "legacy_household_id"), + ("user_simulation_associations", "legacy_simulation_id"), + ("user_report_associations", "legacy_report_id"), + } + ), +) USER_ASSOCIATION_TABLES = ( "user_policies", "user_household_associations", @@ -475,17 +510,6 @@ def test_empty_upgrade_check_base_downgrade_and_reupgrade() -> None: "user_report_associations", ) COUNTRY_COLUMN_NAMES = frozenset({"country", "country_id", "primary_country"}) -# Nullable columns that a parent-revision check constraint requires: a -# standard user's country, and one reference of each association. -NULLABLE_COLUMNS_REQUIRED_BY_CHECKS = frozenset( - { - ("users", "primary_country"), - ("user_policies", "legacy_policy_id"), - ("user_household_associations", "legacy_household_id"), - ("user_simulation_associations", "legacy_simulation_id"), - ("user_report_associations", "legacy_report_id"), - } -) # Required values that a parent-revision check constraint restricts. CHECKED_VALUES = {("datasets", "year"): 2026} @@ -544,6 +568,7 @@ def _row_values( table: sa.Table, stored_rows: dict[str, dict], sequence: Iterator[int], + usage: PreviousRevisionUsage, *, head_table: sa.Table | None = None, ) -> dict: @@ -556,7 +581,7 @@ def _row_values( column.name: _test_value(table, column, sequence) for column in table.columns if _is_required_without_default(column) - or (table.name, column.name) in NULLABLE_COLUMNS_REQUIRED_BY_CHECKS + or (table.name, column.name) in usage.nullable_columns_required_by_checks } if head_table is not None: for column_name in _new_required_columns(table, head_table): @@ -622,30 +647,37 @@ def _rewrite_mapped_columns(connection, table: sa.Table, row: dict) -> None: assert result.rowcount == 1, table.name +@dataclass(frozen=True) +class SchemaComparison: + """A previous revision's reflected tables, a newer schema, and their usage.""" + + parent: sa.MetaData + head: sa.MetaData + usage: PreviousRevisionUsage + + def _assert_user_deletion_cascades( connection, - parent: sa.MetaData, - head: sa.MetaData, + schemas: SchemaComparison, stored_rows: dict[str, dict], sequence: Iterator[int], ) -> None: """Delete, through the parent's users table, a user who owns Stage 12A rows.""" + parent, head, usage = schemas.parent, schemas.head, schemas.usage users = parent.tables["users"] owner = _insert_and_read_back( - connection, users, _row_values(users, stored_rows, sequence) + connection, users, _row_values(users, stored_rows, sequence, usage) ) owner_rows = {**stored_rows, "users": owner} for table_name in USER_ASSOCIATION_TABLES: table = parent.tables[table_name] head_table = ( - head.tables[table_name] - if table_name in PARENT_REVISION_READ_ONLY_TABLES - else None + head.tables[table_name] if table_name in usage.read_only_tables else None ) _insert_and_read_back( connection, table, - _row_values(table, owner_rows, sequence, head_table=head_table), + _row_values(table, owner_rows, sequence, usage, head_table=head_table), insert_table=head_table, ) identities = head.tables["user_identities"] @@ -671,6 +703,105 @@ def _assert_user_deletion_cascades( assert remaining == 0, table_name +def _assert_newer_schema_serves_previous_usage( + engine: Engine, + parent: sa.MetaData, + head: sa.MetaData, + usage: PreviousRevisionUsage, +) -> None: + """Run a previous revision's use of every table it maps against a newer schema. + + Each table of ``parent`` gets a row inserted (supplying only what the + previous revision requires), read back with every mapped column, + rewritten and deleted, and a user who owns association rows is deleted so + the cascades run. The newer schema may not drop a column ``parent`` maps, + or give a table the previous revision writes a new required column + without a server default. Every change is rolled back. + """ + parent_tables = [ + table for table in parent.sorted_tables if table.name != "alembic_version" + ] + dropped_columns = sorted( + f"{table.name}.{column.name}" + for table in parent_tables + for column in table.columns + if table.name not in head.tables + or column.name not in head.tables[table.name].columns + ) + assert dropped_columns == [] + tables_with_new_required_columns = { + table.name: sorted(_new_required_columns(table, head.tables[table.name])) + for table in parent_tables + if _new_required_columns(table, head.tables[table.name]) + } + assert set(tables_with_new_required_columns) <= usage.read_only_tables, ( + tables_with_new_required_columns + ) + + sequence = itertools.count() + with engine.connect() as connection: + transaction = connection.begin() + try: + stored_rows: dict[str, dict] = {} + for table in parent_tables: + head_table = ( + head.tables[table.name] + if table.name in usage.read_only_tables + else None + ) + stored_rows[table.name] = _insert_and_read_back( + connection, + table, + _row_values( + table, stored_rows, sequence, usage, head_table=head_table + ), + insert_table=head_table, + ) + _rewrite_mapped_columns(connection, table, stored_rows[table.name]) + + _assert_user_deletion_cascades( + connection, SchemaComparison(parent, head, usage), stored_rows, sequence + ) + + for table in reversed(parent_tables): + result = connection.execute( + table.delete().where( + _primary_key_clause(table, stored_rows[table.name]) + ) + ) + assert result.rowcount == 1, table.name + finally: + transaction.rollback() + + +def test_stage_12a_schema_serves_the_previous_revision() -> None: + """Run the pre-Stage 12A revision's table usage against the Stage 12A schema. + + A rollback to a pre-Stage 12A release after the simulation-table removal + first downgrades the schema to the Stage 12A revision, so that schema must + still serve the pre-Stage 12A release. + """ + database_url = _disposable_url() + config = _config() + engine = create_engine(database_url) + stage_12a_revision = _simulation_table_removal_parent() + + try: + command.downgrade(config, STAGE_12A_PREVIOUS_REVISION) + parent = sa.MetaData() + parent.reflect(engine) + command.upgrade(config, stage_12a_revision) + stage_12a = sa.MetaData() + stage_12a.reflect(engine) + + _assert_newer_schema_serves_previous_usage( + engine, parent, stage_12a, STAGE_12A_PREVIOUS_USAGE + ) + finally: + command.upgrade(config, "head") + engine.dispose() + + def _reflect_previous_revision_usage(engine) -> sa.MetaData: """Reflect the tables and columns the previous application revision maps. @@ -720,9 +851,6 @@ def test_simulation_table_removal_serves_the_previous_revision() -> None: command.upgrade(config, "head") head = sa.MetaData() head.reflect(engine) - parent_tables = [ - table for table in parent.sorted_tables if table.name != "alembic_version" - ] assert set(full_parent.tables) - set(head.tables) == SIMULATION_SERVICE_TABLES assert sorted( @@ -737,57 +865,86 @@ def test_simulation_table_removal_serves_the_previous_revision() -> None: for column_name in retired_columns ) - dropped_columns = sorted( - f"{table.name}.{column.name}" - for table in parent_tables - for column in table.columns - if table.name not in head.tables - or column.name not in head.tables[table.name].columns + _assert_newer_schema_serves_previous_usage( + engine, parent, head, SIMULATION_TABLE_REMOVAL_PREVIOUS_USAGE ) - assert dropped_columns == [] - tables_with_new_required_columns = { - table.name: sorted(_new_required_columns(table, head.tables[table.name])) - for table in parent_tables - if _new_required_columns(table, head.tables[table.name]) - } - assert set(tables_with_new_required_columns) <= ( - PARENT_REVISION_READ_ONLY_TABLES - ), tables_with_new_required_columns + finally: + command.upgrade(config, "head") + engine.dispose() - sequence = itertools.count() - with engine.connect() as connection: - transaction = connection.begin() - try: - stored_rows: dict[str, dict] = {} - for table in parent_tables: - head_table = ( - head.tables[table.name] - if table.name in PARENT_REVISION_READ_ONLY_TABLES - else None - ) - stored_rows[table.name] = _insert_and_read_back( - connection, - table, - _row_values( - table, stored_rows, sequence, head_table=head_table - ), - insert_table=head_table, - ) - _rewrite_mapped_columns(connection, table, stored_rows[table.name]) - _assert_user_deletion_cascades( - connection, parent, head, stored_rows, sequence +def _schema_snapshot(engine: Engine, table_names: Iterable[str]) -> dict: + """Describe tables as PostgreSQL reports them, ignoring column order.""" + inspector = inspect(engine) + snapshot: dict[str, dict] = {} + for table_name in sorted(table_names): + snapshot[table_name] = { + "columns": { + column["name"]: ( + repr(column["type"]), + column["nullable"], + column.get("default"), + ) + for column in inspector.get_columns(table_name) + }, + "primary_key": inspector.get_pk_constraint(table_name), + "foreign_keys": sorted( + ( + foreign_key["name"], + tuple(foreign_key["constrained_columns"]), + foreign_key["referred_table"], + tuple(foreign_key["referred_columns"]), + foreign_key["options"].get("ondelete"), ) + for foreign_key in inspector.get_foreign_keys(table_name) + ), + "unique_constraints": sorted( + (constraint["name"], tuple(constraint["column_names"])) + for constraint in inspector.get_unique_constraints(table_name) + ), + "check_constraints": sorted( + (constraint["name"], constraint["sqltext"]) + for constraint in inspector.get_check_constraints(table_name) + ), + "indexes": sorted( + (index["name"], tuple(index["column_names"]), index["unique"]) + for index in inspector.get_indexes(table_name) + ), + } + with engine.connect() as connection: + snapshot["enum_types"] = { + type_name: list( + connection.execute( + text( + "SELECT enumlabel FROM pg_enum JOIN pg_type " + "ON pg_enum.enumtypid = pg_type.oid " + "WHERE typname = :type_name ORDER BY enumsortorder" + ), + {"type_name": type_name}, + ).scalars() + ) + for type_name in sorted(SIMULATION_DOMAIN_ENUM_TYPES) + } + return snapshot - for table in reversed(parent_tables): - result = connection.execute( - table.delete().where( - _primary_key_clause(table, stored_rows[table.name]) - ) - ) - assert result.rowcount == 1, table.name - finally: - transaction.rollback() + +def test_simulation_table_removal_downgrade_restores_the_previous_schema() -> None: + """The downgrade recreates exactly what the earlier revisions created.""" + database_url = _disposable_url() + config = _config() + engine = create_engine(database_url) + compared_tables = SIMULATION_SERVICE_TABLES | RETIRED_ASSOCIATION_COLUMNS.keys() + + try: + command.downgrade(config, "base") + command.upgrade(config, _simulation_table_removal_parent()) + created_by_earlier_revisions = _schema_snapshot(engine, compared_tables) + command.upgrade(config, "head") + command.downgrade(config, _simulation_table_removal_parent()) + recreated_by_downgrade = _schema_snapshot(engine, compared_tables) + + assert all(created_by_earlier_revisions["enum_types"].values()) + assert recreated_by_downgrade == created_by_earlier_revisions finally: command.upgrade(config, "head") engine.dispose() @@ -838,8 +995,10 @@ def report_run_checks() -> set[str]: } try: - # The removal revision drops report_runs, so check it before that. - command.downgrade(config, _simulation_table_removal_parent()) + # The removal revision drops report_runs, so build the schema from + # empty up to the revision before it: report_runs is the baseline's. + command.downgrade(config, "base") + command.upgrade(config, _simulation_table_removal_parent()) assert isinstance(idempotency_column_type(), postgresql.UUID) assert "ck_report_runs_idempotency_key_nonblank" not in report_run_checks() From 3c5ff892458b29d79e0bc3c4e7f2d1ed1d1ea1d3 Mon Sep 17 00:00:00 2001 From: Anthony Volk <14987227+anth-volk@users.noreply.github.com> Date: Thu, 22 Oct 2026 18:32:59 +0400 Subject: [PATCH 4/4] Correct which dropped tables have replacements, and outdated passages The simulation service has its own replacements for 13 of the 15 dropped tables; aggregates and change_aggregates, v2 alpha's custom totals, were unused and have none. Several places said it owns all 15. The table set is renamed SIMULATION_DOMAIN_TABLES and its comment, the validation message, the Alembic migrations skill, the changelog fragment, the rule statement and the guard test's docstring now say so. Test names that called them simulation-service tables follow. Outdated passages found in review: the Stage 12A document's head revision and its description of the previous-revision test, which now upgrades to fbb7488ebe8a instead of head, and the comments in catalog publication and the household migration qualification. Wording, names and documentation only: no behavior changes, so no test can fail before this change. Co-Authored-By: Claude Opus 5.5 --- changelog.d/drop-simulation-tables.removed.md | 2 +- docs/engineering/skills/alembic-migrations.md | 17 ++++++++++------- docs/generated/rules.md | 2 +- docs/migration/stage-12a-user-login.md | 19 +++++++++++-------- .../data/v2/catalog/publication.py | 4 ++-- .../v2/household_migration_qualification.py | 7 ++++--- .../data/v2/metadata_validation.py | 18 ++++++++++-------- rules/registry.toml | 4 ++-- .../integration/test_alembic_v2_lifecycle.py | 12 ++++++------ tests/unit/v2/test_alembic_v2.py | 10 +++++----- tests/unit/v2/test_metadata_validation.py | 8 ++++---- tests/unit/v2/test_models.py | 8 ++++---- .../v2/test_simulation_domain_boundaries.py | 17 +++++++++-------- 13 files changed, 69 insertions(+), 59 deletions(-) diff --git a/changelog.d/drop-simulation-tables.removed.md b/changelog.d/drop-simulation-tables.removed.md index ddf8e37c5..5b96790f5 100644 --- a/changelog.d/drop-simulation-tables.removed.md +++ b/changelog.d/drop-simulation-tables.removed.md @@ -1 +1 @@ -Dropped the Public API's copies of the 15 simulation-domain v2 tables, their 8 PostgreSQL enum types and the retired `country` and `label` columns of the simulation and report associations; the simulation service owns those tables in its own schema. +Dropped the Public API's copies of the 15 simulation-domain v2 tables, their 8 PostgreSQL enum types and the retired `country` and `label` columns of the simulation and report associations. The simulation service has its own replacements for all of these tables except the unused custom-total tables `aggregates` and `change_aggregates`. diff --git a/docs/engineering/skills/alembic-migrations.md b/docs/engineering/skills/alembic-migrations.md index fd7428d3f..0d94dc7a0 100644 --- a/docs/engineering/skills/alembic-migrations.md +++ b/docs/engineering/skills/alembic-migrations.md @@ -389,10 +389,12 @@ Revision `363f34954a14` was autogenerated after the last schema-only declarations of the simulation-domain tables were deleted (an earlier release had already removed their models and stopped reading or writing them). It drops the 15 simulation-domain tables (`household_jobs`, `simulations`, -`reports`, `report_runs` and the 11 report-run output tables), which the -simulation service owns in its own `simulation` schema, and the retired +`reports`, `report_runs` and the 11 report-run output tables) and the retired `country` and `label` columns of `user_simulation_associations` and -`user_report_associations`. Its post-generation corrections: +`user_report_associations`. The simulation service has its own replacements +for all of these tables in its `simulation` schema except `aggregates` and +`change_aggregates`, v2 alpha's custom totals, which were unused and have no +replacement. Its post-generation corrections: - Alembic emits removed tables in arbitrary order. The upgrade drops each table before the tables its foreign keys reference, and the downgrade @@ -404,10 +406,11 @@ simulation service owns in its own `simulation` schema, and the retired which leaves invalid SQL in the generated downgrade. They are restored as the model declared them. -The v2 lifecycle test covers the upgrade, the downgrade to the previous -revision and the re-upgrade, and checks that head serves the previous -application revision, which no longer maps any dropped table or column. The -downgrade recreates the tables and columns empty. +The v2 lifecycle tests cover the upgrade, the downgrade to the previous +revision and the re-upgrade. They check that the downgrade recreates exactly +the tables, columns, constraints, indexes and enum types the earlier +revisions created (empty), and that head serves the previous application +revision, which no longer maps any dropped table or column. The revision must not deploy while an application revision that maps the dropped tables or columns can serve traffic, and it may run only after an diff --git a/docs/generated/rules.md b/docs/generated/rules.md index a48377314..b66d47a12 100644 --- a/docs/generated/rules.md +++ b/docs/generated/rules.md @@ -207,7 +207,7 @@ flowchart LR | `api.reports.v1-persistence-only` | The existing report route and service import only v1 persistence. | mechanical | enforced | yes | pytest `tests/unit/v2/test_report_v1_runtime_selection.py` | `pr.yml:test` | **unproven** | | `api.stage12.comparison-tables-isolated` | Serving routes and services never read the Stage 12 comparison tables, and production tables never depend on them. | mechanical | enforced | yes | pytest `tests/unit/v2/test_stage12_comparison_isolation.py` | `pr.yml:test` | **unproven** | | `api.stage12.retirement-owned` | Retiring the Stage 12 comparison schema stays owned by a named follow-up change in the Stage 12 worker contract. | mechanical | enforced | yes | pytest `tests/unit/v2/test_stage12_comparison_isolation.py` | `pr.yml:test` | **unproven** | -| `api.v2.simulation-tables-stay-removed` | Public API code never imports the removed simulation-domain models (household jobs, simulations, reports, report runs and their outputs), and the v2 metadata never declares their tables, which the simulation service owns in its own schema. | mechanical | enforced | yes | pytest `tests/unit/v2/test_metadata_validation.py`, `tests/unit/v2/test_simulation_domain_boundaries.py` | `pr.yml:test` | yes | +| `api.v2.simulation-tables-stay-removed` | Public API code never imports the removed simulation-domain models (household jobs, simulations, reports, report runs and their outputs), and the v2 metadata never declares the tables a v2 revision dropped with them. | mechanical | enforced | yes | pytest `tests/unit/v2/test_metadata_validation.py`, `tests/unit/v2/test_simulation_domain_boundaries.py` | `pr.yml:test` | yes | | `api.v2.single-orm-layer` | v2 models never create a parallel SQLAlchemy ORM layer alongside SQLModel. | mechanical | enforced | yes | pytest `tests/unit/v2/test_model_persistence.py` | `pr.yml:test` | **unproven** | ```mermaid diff --git a/docs/migration/stage-12a-user-login.md b/docs/migration/stage-12a-user-login.md index 03cac230d..791df42be 100644 --- a/docs/migration/stage-12a-user-login.md +++ b/docs/migration/stage-12a-user-login.md @@ -315,7 +315,8 @@ from the final SQLModel metadata. It has no post-generation corrections. `363f34954a14` then dropped them (see the Alembic migrations skill). The models drop the `user_associations` relationships from v2 simulations and -reports. `V2_ALEMBIC_HEAD_REVISION` is `fbb7488ebe8a`. +reports. With this revision, `V2_ALEMBIC_HEAD_REVISION` became `fbb7488ebe8a`; +revision `363f34954a14` later replaced it as head. ### Reshape safety @@ -410,7 +411,9 @@ row with a NULL `primary_country` does not fail there. The lifecycle test `test_stage_12a_schema_serves_the_previous_revision` exercises this rather than only comparing columns. It reflects the schema at -the parent revision with SQLAlchemy and upgrades to head. Then, through the +the parent revision with SQLAlchemy and upgrades to `fbb7488ebe8a`, not to +head: revision `363f34954a14` later dropped tables the parent revision maps +(see "Rollback"). Then, through the reflected parent tables, it does the following for every table the parent revision maps: @@ -420,13 +423,13 @@ revision maps: - rewrites every non-key column; - deletes the row. -Rows of the two dormant tables are inserted through head's tables, since only -new code writes them, and are then read, rewritten, and deleted through the -parent's. The test also deletes, through the parent's `users` table, a user who +Rows of the two dormant tables are inserted through the Stage 12A tables, since +only new code writes them, and are then read, rewritten, and deleted through +the parent's. The test also deletes, through the parent's `users` table, a user who owns rows in all four association tables and a `user_identities` row, and it -checks that every one of them cascades away. It fails if head drops a parent -column, or adds a required column without a server default to a table the -parent revision writes. +checks that every one of them cascades away. It fails if the Stage 12A schema +drops a parent column, or adds a required column without a server default to a +table the parent revision writes. ## Settings diff --git a/policyengine_api/data/v2/catalog/publication.py b/policyengine_api/data/v2/catalog/publication.py index a858a517d..5995beb4f 100644 --- a/policyengine_api/data/v2/catalog/publication.py +++ b/policyengine_api/data/v2/catalog/publication.py @@ -39,8 +39,8 @@ sa.column("version_num", sa.String), ) # User-created rows that publication must never change. Simulations, reports -# and report runs belong to the simulation service's own schema, and a later -# v2 revision drops the Public API's copies, so publication doesn't read them. +# and report runs belong to the simulation service's own schema, and v2 +# revision 363f34954a14 dropped the Public API's copies. PROTECTED_TABLES = (DatasetVersion.__table__,) LOGGER = logging.getLogger(__name__) diff --git a/policyengine_api/data/v2/household_migration_qualification.py b/policyengine_api/data/v2/household_migration_qualification.py index 3b2a2977f..824f06b25 100644 --- a/policyengine_api/data/v2/household_migration_qualification.py +++ b/policyengine_api/data/v2/household_migration_qualification.py @@ -27,9 +27,10 @@ HOUSEHOLD_MIGRATION_REVISION = "724b1b11a33e" V2_ALEMBIC_CONFIG = "alembic-v2.ini" -# A target before the household revision has these tables, but revision -# 363f34954a14 dropped them and their models, so the counts name them -# directly. The existing-table check skips them on a target without them. +# A target before the household revision has these tables. They have no +# models (an earlier release removed them, and revision 363f34954a14 dropped +# the tables), so the counts name them directly. The existing-table check +# skips them on a target without them. SIMULATIONS = table("simulations", column("household_id")) REPORTS = table("reports", column("household_id")) HOUSEHOLD_JOBS = table("household_jobs") diff --git a/policyengine_api/data/v2/metadata_validation.py b/policyengine_api/data/v2/metadata_validation.py index 4a7bd35b4..a54de26b2 100644 --- a/policyengine_api/data/v2/metadata_validation.py +++ b/policyengine_api/data/v2/metadata_validation.py @@ -35,10 +35,12 @@ ) -# The simulation service owns these tables in its own ``simulation`` schema. -# A v2 revision dropped the Public API's copies, and its metadata must not -# declare them again. -SIMULATION_SERVICE_TABLES = frozenset( +# The simulation-domain tables a v2 revision dropped. The simulation service +# has its own replacements for all of them in its ``simulation`` schema except +# ``aggregates`` and ``change_aggregates``, v2 alpha's custom totals, which +# were unused and have none. The Public API's metadata must not declare them +# again. +SIMULATION_DOMAIN_TABLES = frozenset( { "aggregates", "budget_summary", @@ -64,15 +66,15 @@ class V2MetadataValidationError(RuntimeError): def validate_v2_metadata_table_names(table_names: Iterable[str]) -> None: - """Reject known v1-only, simulation-service and prohibited v2 table names.""" + """Reject known v1-only, dropped simulation-domain and prohibited v2 tables.""" actual = frozenset(table_names) prohibited = sorted(actual & PROHIBITED_V2_TABLES) v1_only = sorted(actual & V1_ONLY_TABLES) - simulation_service = sorted(actual & SIMULATION_SERVICE_TABLES) - if prohibited or v1_only or simulation_service: + simulation_domain = sorted(actual & SIMULATION_DOMAIN_TABLES) + if prohibited or v1_only or simulation_domain: msg = ( "API v2-alpha metadata contains rejected tables: " f"prohibited={prohibited}, v1_only={v1_only}, " - f"simulation_service={simulation_service}" + f"simulation_domain={simulation_domain}" ) raise V2MetadataValidationError(msg) diff --git a/rules/registry.toml b/rules/registry.toml index e0cb247a0..a52b7a9f6 100644 --- a/rules/registry.toml +++ b/rules/registry.toml @@ -615,7 +615,7 @@ invocation = "make test" [[rule]] id = "api.v2.simulation-tables-stay-removed" category = "isolation" -statement = "Public API code never imports the removed simulation-domain models (household jobs, simulations, reports, report runs and their outputs), and the v2 metadata never declares their tables, which the simulation service owns in its own schema." +statement = "Public API code never imports the removed simulation-domain models (household jobs, simulations, reports, report runs and their outputs), and the v2 metadata never declares the tables a v2 revision dropped with them." rationale = "docs/engineering/skills/alembic-migrations.md#simulation-domain-table-removal" scope = ["policyengine_api/**/*.py", "scripts/**/*.py", "tests/**/*.py", "migrations/v2/env.py"] tier = "mechanical" @@ -625,7 +625,7 @@ autofix = false kind = "pytest" nodes = [ "tests/unit/v2/test_simulation_domain_boundaries.py::test_no_module_imports_a_removed_simulation_model", - "tests/unit/v2/test_simulation_domain_boundaries.py::test_v2_metadata_declares_no_simulation_service_table", + "tests/unit/v2/test_simulation_domain_boundaries.py::test_v2_metadata_declares_no_dropped_simulation_table", "tests/unit/v2/test_simulation_domain_boundaries.py::test_models_package_exports_no_simulation_domain_model", "tests/unit/v2/test_metadata_validation.py::test_known_rejected_tables_are_identified", ] diff --git a/tests/integration/test_alembic_v2_lifecycle.py b/tests/integration/test_alembic_v2_lifecycle.py index cd47ac73c..5ec99016d 100644 --- a/tests/integration/test_alembic_v2_lifecycle.py +++ b/tests/integration/test_alembic_v2_lifecycle.py @@ -18,7 +18,7 @@ from sqlalchemy.dialects import postgresql from policyengine_api.constants import REPO -from policyengine_api.data.v2.metadata_validation import SIMULATION_SERVICE_TABLES +from policyengine_api.data.v2.metadata_validation import SIMULATION_DOMAIN_TABLES from policyengine_api.data.v2.migration_target import ( V2_ALEMBIC_DISPOSABLE_TEST, V2_ALEMBIC_HEAD_REVISION, @@ -230,7 +230,7 @@ def _enum_type_names(engine) -> set[str]: def _assert_simulation_domain_schema_present(engine) -> None: assert set(inspect(engine).get_table_names(schema="public")) >= ( - SIMULATION_SERVICE_TABLES + SIMULATION_DOMAIN_TABLES ) assert _enum_type_names(engine) >= SIMULATION_DOMAIN_ENUM_TYPES for table_name, retired_columns in RETIRED_ASSOCIATION_COLUMNS.items(): @@ -247,7 +247,7 @@ def _assert_simulation_domain_schema_present(engine) -> None: def _assert_simulation_domain_schema_removed(engine) -> None: - assert SIMULATION_SERVICE_TABLES.isdisjoint( + assert SIMULATION_DOMAIN_TABLES.isdisjoint( inspect(engine).get_table_names(schema="public") ) assert SIMULATION_DOMAIN_ENUM_TYPES.isdisjoint(_enum_type_names(engine)) @@ -822,7 +822,7 @@ def _reflect_previous_revision_usage(engine) -> sa.MetaData: ) parent.reflect( engine, - only=lambda table_name, _metadata: table_name not in SIMULATION_SERVICE_TABLES, + only=lambda table_name, _metadata: table_name not in SIMULATION_DOMAIN_TABLES, ) return parent @@ -852,7 +852,7 @@ def test_simulation_table_removal_serves_the_previous_revision() -> None: head = sa.MetaData() head.reflect(engine) - assert set(full_parent.tables) - set(head.tables) == SIMULATION_SERVICE_TABLES + assert set(full_parent.tables) - set(head.tables) == SIMULATION_DOMAIN_TABLES assert sorted( f"{table.name}.{column.name}" for table in full_parent.tables.values() @@ -933,7 +933,7 @@ def test_simulation_table_removal_downgrade_restores_the_previous_schema() -> No database_url = _disposable_url() config = _config() engine = create_engine(database_url) - compared_tables = SIMULATION_SERVICE_TABLES | RETIRED_ASSOCIATION_COLUMNS.keys() + compared_tables = SIMULATION_DOMAIN_TABLES | RETIRED_ASSOCIATION_COLUMNS.keys() try: command.downgrade(config, "base") diff --git a/tests/unit/v2/test_alembic_v2.py b/tests/unit/v2/test_alembic_v2.py index 7a4c2e5d8..daaa722a6 100644 --- a/tests/unit/v2/test_alembic_v2.py +++ b/tests/unit/v2/test_alembic_v2.py @@ -16,7 +16,7 @@ from policyengine_api.constants import REPO from policyengine_api.data.v1_models import V1Base -from policyengine_api.data.v2.metadata_validation import SIMULATION_SERVICE_TABLES +from policyengine_api.data.v2.metadata_validation import SIMULATION_DOMAIN_TABLES from policyengine_api.data.v2.migration_target import ( DISPOSABLE_DATABASE_NAME, MIGRATION_ROLE, @@ -53,7 +53,7 @@ STAGE_12A_REVISION = "fbb7488ebe8a" SIMULATION_TABLE_REMOVAL_REVISION = "363f34954a14" # The output tables of a report run, which reference report_runs and simulations. -RUN_OUTPUT_TABLES = SIMULATION_SERVICE_TABLES - { +RUN_OUTPUT_TABLES = SIMULATION_DOMAIN_TABLES - { "household_jobs", "report_runs", "reports", @@ -295,7 +295,7 @@ def test_v2_revision_chain_has_generated_policy_and_user_identity_changes() -> N # The baseline also created the simulation-domain tables a later revision # dropped. baseline_table_count = len( - (V2_TABLE_NAMES | SIMULATION_SERVICE_TABLES) - POST_BASELINE_TABLE_NAMES + (V2_TABLE_NAMES | SIMULATION_DOMAIN_TABLES) - POST_BASELINE_TABLE_NAMES ) assert baseline.count("op.create_table(") == baseline_table_count assert baseline.count("op.drop_table(") == baseline_table_count @@ -796,11 +796,11 @@ def test_simulation_table_removal_revision_is_generated_and_drops_only_them() -> # The upgrade drops exactly the simulation-domain tables and the two # retired columns of each association; the downgrade recreates them. assert set(re.findall(r'op\.drop_table\("([a-z0-9_]+)"\)', upgrade)) == ( - SIMULATION_SERVICE_TABLES + SIMULATION_DOMAIN_TABLES ) assert "op.create_table(" not in upgrade assert set(re.findall(r'op\.create_table\(\s*"([a-z0-9_]+)"', downgrade)) == ( - SIMULATION_SERVICE_TABLES + SIMULATION_DOMAIN_TABLES ) assert "op.drop_table(" not in downgrade retired_columns = { diff --git a/tests/unit/v2/test_metadata_validation.py b/tests/unit/v2/test_metadata_validation.py index 8dc488c13..3455e3d12 100644 --- a/tests/unit/v2/test_metadata_validation.py +++ b/tests/unit/v2/test_metadata_validation.py @@ -4,7 +4,7 @@ from policyengine_api.data.v2.metadata_validation import ( PROHIBITED_V2_TABLES, - SIMULATION_SERVICE_TABLES, + SIMULATION_DOMAIN_TABLES, V1_ONLY_TABLES, V2MetadataValidationError, validate_v2_metadata_table_names, @@ -40,10 +40,10 @@ def test_rejection_sets_do_not_block_current_metadata() -> None: assert table_names.isdisjoint(PROHIBITED_V2_TABLES) assert table_names.isdisjoint(V1_ONLY_TABLES) - assert table_names.isdisjoint(SIMULATION_SERVICE_TABLES) + assert table_names.isdisjoint(SIMULATION_DOMAIN_TABLES) -def test_simulation_service_tables_are_the_fifteen_dropped_tables() -> None: +def test_simulation_domain_tables_are_the_fifteen_dropped_tables() -> None: dropped_tables = { "aggregates", "budget_summary", @@ -62,7 +62,7 @@ def test_simulation_service_tables_are_the_fifteen_dropped_tables() -> None: "simulations", } - assert dropped_tables == SIMULATION_SERVICE_TABLES + assert dropped_tables == SIMULATION_DOMAIN_TABLES def test_new_table_name_requires_no_allowlist_update() -> None: diff --git a/tests/unit/v2/test_models.py b/tests/unit/v2/test_models.py index dd92985ca..166583764 100644 --- a/tests/unit/v2/test_models.py +++ b/tests/unit/v2/test_models.py @@ -6,7 +6,7 @@ from sqlalchemy.schema import CreateTable from policyengine_api.data.v1_models import V1Base -from policyengine_api.data.v2.metadata_validation import SIMULATION_SERVICE_TABLES +from policyengine_api.data.v2.metadata_validation import SIMULATION_DOMAIN_TABLES from policyengine_api.data.v2.models import ( V2_METADATA, DatasetVersion, @@ -69,12 +69,12 @@ def test_every_metadata_table_has_one_mapped_model() -> None: model_table_names = {mapper.local_table.name for mapper in _v2_mappers()} assert V2_METADATA is not V1Base.metadata - assert SIMULATION_SERVICE_TABLES.isdisjoint(metadata_table_names) + assert SIMULATION_DOMAIN_TABLES.isdisjoint(metadata_table_names) assert model_table_names == metadata_table_names assert len(_v2_mappers()) == len(metadata_table_names) -def test_kept_models_have_no_relationship_to_a_simulation_service_table() -> None: +def test_kept_models_have_no_relationship_to_a_dropped_simulation_table() -> None: related_tables = { f"{mapper.class_.__name__}.{relationship.key}": ( relationship.mapper.local_table.name @@ -86,7 +86,7 @@ def test_kept_models_have_no_relationship_to_a_simulation_service_table() -> Non assert not { relationship: table_name for relationship, table_name in related_tables.items() - if table_name in SIMULATION_SERVICE_TABLES + if table_name in SIMULATION_DOMAIN_TABLES } diff --git a/tests/unit/v2/test_simulation_domain_boundaries.py b/tests/unit/v2/test_simulation_domain_boundaries.py index d55c681bc..64cab8b94 100644 --- a/tests/unit/v2/test_simulation_domain_boundaries.py +++ b/tests/unit/v2/test_simulation_domain_boundaries.py @@ -1,10 +1,11 @@ """The simulation-domain tables and models stay out of the Public API. -The simulation service owns household jobs, simulations, reports, report runs -and their outputs in its own ``simulation`` schema. A v2 revision dropped the -Public API's copies of those tables after their models were removed, so no -code here may import those models again and no v2 model may declare those -tables again (rule ``api.v2.simulation-tables-stay-removed``). +A v2 revision dropped the Public API's copies of the household job, +simulation, report, report run and report-run output tables after their models +were removed. The simulation service has its own replacements for all of them +in its ``simulation`` schema except ``aggregates`` and ``change_aggregates``, +which were unused. No code here may import those models again and no v2 model +may declare those tables again (rule ``api.v2.simulation-tables-stay-removed``). """ from __future__ import annotations @@ -14,7 +15,7 @@ from policyengine_api.constants import REPO from policyengine_api.data.v2 import models as v2_models -from policyengine_api.data.v2.metadata_validation import SIMULATION_SERVICE_TABLES +from policyengine_api.data.v2.metadata_validation import SIMULATION_DOMAIN_TABLES from policyengine_api.data.v2.models import V2_METADATA if TYPE_CHECKING: @@ -123,8 +124,8 @@ def test_no_module_imports_a_removed_simulation_model() -> None: ) -def test_v2_metadata_declares_no_simulation_service_table() -> None: - assert SIMULATION_SERVICE_TABLES.isdisjoint(V2_METADATA.tables) +def test_v2_metadata_declares_no_dropped_simulation_table() -> None: + assert SIMULATION_DOMAIN_TABLES.isdisjoint(V2_METADATA.tables) assert not any( (REPO / "policyengine_api/data/v2" / relative_path).exists() for relative_path in (