Stop using the simulation-domain v2 tables and retired association columns

changelog.d/stop-using-simulation-tables.changed.md

@@ -0,0 +1 @@
+Stopped the Public API from reading or writing its copies of the simulation-domain v2 tables and the retired `country` and `label` association columns, so a later revision can drop them while this release serves traffic.

docs/engineering/skills/testing.md

@@ -55,10 +55,10 @@ python -m pytest tests/unit/test_cloud_run_deploy_scripts.py tests/unit/test_cap
 For v2 platform foundation work, run the smallest applicable group while
 iterating, then all groups before qualification.
 
-SQLModel schema, report/run behavior, lazy configuration, and import effects:
+SQLModel schema, lazy configuration, and import effects:
 
 ```bash
-uv run pytest tests/unit/v2/test_models.py tests/unit/v2/test_model_persistence.py tests/unit/v2/test_report_runs.py tests/unit/v2/test_settings.py tests/unit/v2/test_import_side_effects.py -q
+uv run pytest tests/unit/v2/test_models.py tests/unit/v2/test_model_persistence.py tests/unit/v2/test_settings.py tests/unit/v2/test_import_side_effects.py -q
 ```
 
 The generated-only v2 Alembic chain and disposable Postgres lifecycle:

docs/migration/stage-12a-user-login.md

@@ -311,7 +311,8 @@ from the final SQLModel metadata. It has no post-generation corrections.
   kept: `country` becomes nullable and nothing reads or writes either column.
   The previous Public API revision still maps them, so dropping them would
   break it during the deploy window or an application rollback (see
-  "Rollback").
+  "Rollback"). A later release stopped mapping them: they stay declared on the
+  tables, unmapped, only until a revision drops them.
 
 The models drop the `user_associations` relationships from v2 simulations and
 reports. `V2_ALEMBIC_HEAD_REVISION` is `fbb7488ebe8a`.

docs/migration/stage-9-v2-metadata.md

@@ -23,9 +23,9 @@ Existing clients must not be redirected to the v2 resource routes.
 
 The initializer creates only reusable logical input `Dataset` rows. Each row
 has `is_output_dataset=false` and a null `storage_path`. It creates no
-package-derived `DatasetVersion` rows or simulation/report output datasets and
-does not modify `Simulation`, `Report`, or `ReportRun` rows or their dataset
-references.
+package-derived `DatasetVersion` rows or simulation/report output datasets.
+The simulation service owns simulations, reports and report runs in its own
+schema; the Public API no longer has models for them.
 
 ## Configuration and credential boundaries
 

policyengine_api/data/v2/catalog/publication.py

@@ -24,12 +24,7 @@
     PublicationEvidence,
 )
 from policyengine_api.data.v2.migration_target import V2_ALEMBIC_HEAD_REVISION
-from policyengine_api.data.v2.models import (
-    DatasetVersion,
-    Report,
-    ReportRun,
-    Simulation,
-)
+from policyengine_api.data.v2.models import DatasetVersion
 
 if TYPE_CHECKING:
     from collections.abc import Callable
@@ -43,12 +38,10 @@
     "alembic_version",
     sa.column("version_num", sa.String),
 )
-PROTECTED_TABLES = (
-    DatasetVersion.__table__,
-    Simulation.__table__,
-    Report.__table__,
-    ReportRun.__table__,
-)
+# User-created rows that publication must never change. Simulations, reports
+# and report runs belong to the simulation service's own schema, and a later
+# v2 revision drops the Public API's copies, so publication doesn't read them.
+PROTECTED_TABLES = (DatasetVersion.__table__,)
 
 LOGGER = logging.getLogger(__name__)
 
@@ -80,7 +73,7 @@ def _acquire_publication_lock(connection: Connection) -> None:
     ).scalar_one()
 
 
-def _protected_row_counts(connection: Connection) -> tuple[int, int, int, int]:
+def _protected_row_counts(connection: Connection) -> tuple[int, ...]:
     return tuple(
         connection.execute(sa.select(sa.func.count()).select_from(table)).scalar_one()
         for table in PROTECTED_TABLES
@@ -120,7 +113,7 @@ def publish_catalog(
             assert_country_matches(connection, country)
         assert_canonical_value_uniqueness(connection)
         if _protected_row_counts(connection) != before:
-            msg = "publication changed simulation, report, or dataset-version rows"
+            msg = "publication changed dataset-version rows"
             raise CatalogPublicationError(msg)
         if checkpoint is not None:
             checkpoint("after_validation", connection)

policyengine_api/data/v2/household_migration_qualification.py

@@ -10,18 +10,12 @@
 from alembic.config import Config
 from alembic.migration import MigrationContext
 from alembic.script import ScriptDirectory
-from sqlalchemy import Connection, Engine, func, inspect, select
+from sqlalchemy import Connection, Engine, column, func, inspect, select, table
 from sqlalchemy.exc import NoInspectionAvailable
 from sqlalchemy.pool import NullPool
 from sqlmodel import create_engine
 
-from policyengine_api.data.v2.models import (
-    Household,
-    HouseholdJob,
-    Report,
-    Simulation,
-    UserHouseholdAssociation,
-)
+from policyengine_api.data.v2.models import Household, UserHouseholdAssociation
 from policyengine_api.data.v2.settings import (
     V2ConfigurationError,
     V2DatabaseSettings,
@@ -33,6 +27,12 @@
 
 HOUSEHOLD_MIGRATION_REVISION = "724b1b11a33e"
 V2_ALEMBIC_CONFIG = "alembic-v2.ini"
+# A target before the household revision has these tables, but no model maps
+# them any more and a later revision drops them, so the counts name them
+# directly. The existing-table check skips them on a target without them.
+SIMULATIONS = table("simulations", column("household_id"))
+REPORTS = table("reports", column("household_id"))
+HOUSEHOLD_JOBS = table("household_jobs")
 
 
 class ScalarExecutor(Protocol):
@@ -130,19 +130,19 @@ def count(statement: object, table_name: str) -> int:
         ),
         simulation_household_references=count(
             select(func.count())
-            .select_from(Simulation)
-            .where(Simulation.household_id.is_not(None)),
-            "simulations",
+            .select_from(SIMULATIONS)
+            .where(SIMULATIONS.c.household_id.is_not(None)),
+            SIMULATIONS.name,
         ),
         report_household_references=count(
             select(func.count())
-            .select_from(Report)
-            .where(Report.household_id.is_not(None)),
-            "reports",
+            .select_from(REPORTS)
+            .where(REPORTS.c.household_id.is_not(None)),
+            REPORTS.name,
         ),
         household_jobs=count(
-            select(func.count()).select_from(HouseholdJob),
-            "household_jobs",
+            select(func.count()).select_from(HOUSEHOLD_JOBS),
+            HOUSEHOLD_JOBS.name,
         ),
     )
 

policyengine_api/data/v2/models/__init__.py

@@ -17,6 +17,11 @@
 }
 
 # Import in dependency order so every reviewed table registers exactly once.
+# retired_tables declares, without models, the tables and columns a later
+# revision drops; see its docstring.
+from policyengine_api.data.v2.models import (  # noqa: E402
+    retired_tables as retired_tables,
+)
 from policyengine_api.data.v2.models.associations import (  # noqa: E402
     UserHouseholdAssociation,
     UserPolicy,
@@ -36,8 +41,6 @@
 )
 from policyengine_api.data.v2.models.households import (  # noqa: E402
     Household,
-    HouseholdJob,
-    HouseholdJobStatus,
 )
 from policyengine_api.data.v2.models.metadata import (  # noqa: E402
     Dataset,
@@ -60,31 +63,6 @@
     LegacyUserMapping,
     LegacyUserPolicyMapping,
 )
-from policyengine_api.data.v2.models.reports import (  # noqa: E402
-    AggregateOutput,
-    AggregateType,
-    BudgetSummary,
-    ChangeAggregate,
-    CongressionalDistrictImpact,
-    ConstituencyImpact,
-    DecileImpact,
-    DecileType,
-    Inequality,
-    IntraDecileImpact,
-    LocalAuthorityImpact,
-    OutputStatus,
-    Poverty,
-    ProgramStatistics,
-    Report,
-    ReportRun,
-    ReportRunStatus,
-    ReportRunTrigger,
-)
-from policyengine_api.data.v2.models.simulations import (  # noqa: E402
-    Simulation,
-    SimulationStatus,
-    SimulationType,
-)
 from policyengine_api.data.v2.models.users import (  # noqa: E402
     User,
     UserIdentity,
@@ -96,43 +74,20 @@
 
 __all__ = [
     "V2_METADATA",
-    "AggregateOutput",
-    "AggregateType",
-    "BudgetSummary",
-    "ChangeAggregate",
-    "CongressionalDistrictImpact",
-    "ConstituencyImpact",
     "Dataset",
     "DatasetVersion",
-    "DecileImpact",
-    "DecileType",
     "Dynamic",
     "Household",
-    "HouseholdJob",
-    "HouseholdJobStatus",
-    "Inequality",
-    "IntraDecileImpact",
     "LegacyHouseholdMapping",
     "LegacyPolicyMapping",
     "LegacyUserMapping",
     "LegacyUserPolicyMapping",
-    "LocalAuthorityImpact",
-    "OutputStatus",
     "Parameter",
     "ParameterNode",
     "ParameterValue",
     "Policy",
-    "Poverty",
-    "ProgramStatistics",
     "Region",
     "RegionType",
-    "Report",
-    "ReportRun",
-    "ReportRunStatus",
-    "ReportRunTrigger",
-    "Simulation",
-    "SimulationStatus",
-    "SimulationType",
     "Stage12AggregationStatus",
     "Stage12ComparisonReport",
     "Stage12ComparisonSimulation",

policyengine_api/data/v2/models/associations.py

@@ -4,11 +4,10 @@
 nullable country-scoped v1 (legacy) reference, or both; a database check
 requires at least one.
 
-The simulation and report associations still carry the pre-Stage 12A
-``country`` and ``label`` columns. Nothing reads or writes them; they stay
-nullable only so the previous Public API revision, whose models still map
-them, can load these rows during a deploy or rollback. A later revision drops
-them once that revision can no longer be deployed.
+The simulation and report association tables still have the pre-Stage 12A
+``country`` and ``label`` columns, which a later revision drops. These models
+don't declare them, so nothing reads or writes them; ``retired_tables``
+declares them on the tables only so the metadata matches the schema.
 """
 
 from datetime import datetime
@@ -217,9 +216,6 @@ class UserSimulationAssociation(TimestampedModel, table=True):
     country_id: str = Field(max_length=2)
     name: str | None = Field(default=None, max_length=255)
     description: str | None = Field(default=None, sa_type=sa.Text)
-    # Retired; see the module docstring.
-    country: str | None = Field(default=None, max_length=16)
-    label: str | None = Field(default=None, max_length=255)
 
     user: User = Relationship(back_populates="simulation_associations")
 
@@ -299,8 +295,5 @@ class UserReportAssociation(TimestampedModel, table=True):
         default=None,
         sa_type=sa.DateTime(timezone=True),
     )
-    # Retired; see the module docstring.
-    country: str | None = Field(default=None, max_length=16)
-    label: str | None = Field(default=None, max_length=255)
 
     user: User = Relationship(back_populates="report_associations")

policyengine_api/data/v2/models/base.py

@@ -31,13 +31,13 @@
             "multi-column indexes are not expressible by one SQLModel Field."
         ),
         "ambiguous_foreign_key_relationships": (
-            "Dataset input/output, baseline/reform, and overlapping composite "
-            "region-default joins need SQLAlchemy relationship hints exposed "
-            "by SQLModel Relationship."
+            "Overlapping composite region-default joins need SQLAlchemy "
+            "relationship hints exposed by SQLModel Relationship."
         ),
-        "transaction_conflict_recovery": (
-            "Concurrent report idempotency requires a savepoint and bounded "
-            "IntegrityError recovery around the database uniqueness constraint."
+        "retired_table_declarations": (
+            "Tables and columns a later revision drops stay declared as plain "
+            "SQLAlchemy tables and columns that no model maps, so the metadata "
+            "matches the schema while nothing reads or writes them."
         ),
     }
 )